diff --git a/scripts/src/hwlab-node/source-workspace.ts b/scripts/src/hwlab-node/source-workspace.ts index cc876bc4..fee7fddb 100644 --- a/scripts/src/hwlab-node/source-workspace.ts +++ b/scripts/src/hwlab-node/source-workspace.ts @@ -82,12 +82,16 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo const dryRun = scoped.dryRun || !scoped.confirm; const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane); const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); - const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.max(scoped.timeoutSeconds, sourceWorkspace.install.timeoutSeconds + 60)); + const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.min(scoped.timeoutSeconds, 55)); const payload = parseLastJsonLineObject(statusText(result)); - const after = dryRun ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); + const wait = dryRun || result.exitCode !== 0 || payload.ok !== true + ? null + : waitForSourceWorkspaceBootstrapJob(scoped, controlPlane.ciNamespace, sourceWorkspaceJobName(scoped.spec), sourceWorkspace.install.timeoutSeconds); + const after = dryRun || wait?.ok !== true ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace); const afterOk = after === null ? null : after.ok === true; const payloadOk = payload.ok === true; - const ok = result.exitCode === 0 && payloadOk && (dryRun || afterOk === true); + const waitOk = dryRun ? true : wait?.ok === true; + const ok = result.exitCode === 0 && payloadOk && waitOk && (dryRun || afterOk === true); return { ok, command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`, @@ -102,6 +106,7 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo before, bootstrap: payload, result: compactRuntimeCommand(result), + wait, after, degradedReason: ok ? undefined : dryRun ? "source-workspace-bootstrap-dry-run-failed" : "source-workspace-bootstrap-failed", next: ok @@ -279,6 +284,72 @@ function sleepMs(ms: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } +function waitForSourceWorkspaceBootstrapJob(scoped: ScopedNodeOptions, namespace: string, jobName: string, timeoutSeconds: number): Record { + const startedAt = Date.now(); + const deadline = startedAt + (timeoutSeconds + 60) * 1000; + let polls = 0; + let lastPayload: Record = { ok: false, status: "not-polled", valuesPrinted: false }; + while (Date.now() <= deadline) { + polls += 1; + const result = runTransScript(scoped.node, sourceWorkspaceBootstrapJobStatusScript(namespace, jobName), "", Math.min(scoped.timeoutSeconds, 55)); + const payload = parseLastJsonLineObject(statusText(result)); + lastPayload = { ...payload, probe: compactRuntimeCommand(result) }; + if (result.exitCode === 0 && payload.status === "succeeded" && payload.ok === true) { + return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; + } + if (payload.status === "failed") { + return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; + } + sleepMs(5000); + } + return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false }; +} + +function sourceWorkspaceBootstrapJobStatusScript(namespace: string, jobName: string): string { + return [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `job=${shellQuote(jobName)}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "job_json=\"$tmp_dir/job.json\"", + "job_err=\"$tmp_dir/job.err\"", + "logs_file=\"$tmp_dir/logs.txt\"", + "kubectl -n \"$namespace\" get job \"$job\" -o json >\"$job_json\" 2>\"$job_err\"", + "job_exit=$?", + "kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true", + "python3 - \"$job_exit\" \"$namespace\" \"$job\" \"$job_json\" \"$job_err\" \"$logs_file\" <<'PY'", + "import json, pathlib, sys", + "job_exit = int(sys.argv[1])", + "namespace, job = sys.argv[2:4]", + "job_text = pathlib.Path(sys.argv[4]).read_text(errors='replace') if pathlib.Path(sys.argv[4]).exists() else ''", + "job_err = pathlib.Path(sys.argv[5]).read_text(errors='replace') if pathlib.Path(sys.argv[5]).exists() else ''", + "logs = pathlib.Path(sys.argv[6]).read_text(errors='replace') if pathlib.Path(sys.argv[6]).exists() else ''", + "job_obj = None", + "if job_exit == 0 and job_text.strip():", + " try:", + " job_obj = json.loads(job_text)", + " except Exception:", + " job_obj = None", + "conditions = (((job_obj or {}).get('status') or {}).get('conditions') or [])", + "complete = any((c or {}).get('type') == 'Complete' and (c or {}).get('status') == 'True' for c in conditions)", + "failed = any((c or {}).get('type') == 'Failed' and (c or {}).get('status') == 'True' for c in conditions)", + "job_payload = None", + "for line in reversed(logs.splitlines()):", + " text = line.strip()", + " if text.startswith('{') and text.endswith('}'):", + " try:", + " job_payload = json.loads(text)", + " break", + " except Exception:", + " pass", + "job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True", + "status = 'succeeded' if complete and job_ok else 'failed' if failed or (complete and not job_ok) else 'missing' if job_exit != 0 else 'running'", + "print(json.dumps({'ok': status == 'succeeded', 'status': status, 'namespace': namespace, 'job': job, 'jobExitCode': job_exit, 'complete': complete, 'failed': failed, 'active': ((job_obj or {}).get('status') or {}).get('active'), 'succeeded': ((job_obj or {}).get('status') or {}).get('succeeded'), 'failedCount': ((job_obj or {}).get('status') or {}).get('failed'), 'jobPayload': job_payload, 'jobErrorTail': job_err[-2000:], 'logsTail': logs[-4000:], 'valuesPrinted': False}, ensure_ascii=False))", + "PY", + ].join("\n"); +} + function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record { return { workspace: spec.workspace, @@ -620,7 +691,7 @@ function sourceWorkspaceBootstrapK8sScript( "logs_file=\"$tmp_dir/logs.txt\"", "printf '%s' \"$manifest_b64\" | base64 -d >\"$manifest\"", "if [ \"$dry_run\" = true ]; then", - " kubectl -n \"$namespace\" apply --dry-run=server -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"", + " kubectl -n \"$namespace\" apply --dry-run=client -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"", " apply_rc=$?", " python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'", "import json, pathlib, sys", @@ -635,33 +706,15 @@ function sourceWorkspaceBootstrapK8sScript( "kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true >/dev/null 2>&1", "kubectl -n \"$namespace\" apply -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"", "apply_rc=$?", - "wait_rc=1", - "if [ \"$apply_rc\" = 0 ]; then", - " kubectl -n \"$namespace\" wait --for=condition=complete \"job/$job\" --timeout=\"${timeout_seconds}s\" >\"$wait_out\" 2>\"$wait_err\"", - " wait_rc=$?", - "fi", - "kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true", - "python3 - \"$apply_rc\" \"$wait_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" \"$wait_out\" \"$wait_err\" \"$logs_file\" <<'PY'", + "python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'", "import json, pathlib, sys", "apply_rc = int(sys.argv[1])", - "wait_rc = int(sys.argv[2])", - "namespace, job, manifest_sha = sys.argv[3:6]", - "paths = [pathlib.Path(item) for item in sys.argv[6:]]", - "apply_out, apply_err, wait_out, wait_err, logs = [path.read_text(errors='replace') for path in paths]", - "job_payload = None", - "for line in reversed(logs.splitlines()):", - " text = line.strip()", - " if text.startswith('{') and text.endswith('}'):", - " try:", - " job_payload = json.loads(text)", - " break", - " except Exception:", - " pass", - "job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True", - "ok = apply_rc == 0 and wait_rc == 0 and job_ok", - "print(json.dumps({'ok': ok, 'status': 'succeeded' if ok else 'failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'waitExitCode': wait_rc, 'jobPayload': job_payload, 'applyOutputTail': apply_out[-2000:], 'applyErrorTail': apply_err[-2000:], 'waitOutputTail': wait_out[-2000:], 'waitErrorTail': wait_err[-2000:], 'logsTail': logs[-4000:], 'mutation': ok, 'valuesPrinted': False}, ensure_ascii=False))", + "namespace, job, manifest_sha = sys.argv[2:5]", + "apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]", + "apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]", + "print(json.dumps({'ok': apply_rc == 0, 'status': 'submitted' if apply_rc == 0 else 'submit-failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': apply_rc == 0, 'valuesPrinted': False}, ensure_ascii=False))", "PY", - "exit $([ \"$apply_rc\" = 0 ] && [ \"$wait_rc\" = 0 ] && printf 0 || printf 1)", + "exit \"$apply_rc\"", ].join("\n"); }