mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: poll source workspace bootstrap jobs

Browse Source
This commit is contained in:
Codex
2026-06-29 09:19:57 +00:00
parent 561129c873
commit c6cfe750c4
1 changed files with 81 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/source-workspace.ts
+81 -28
View File
@@ -82,12 +82,16 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo
const dryRun = scoped.dryRun || !scoped.confirm;
const controlPlane = hwlabNodeControlPlaneSourceWorkspaceBootstrap(scoped.node, scoped.lane);
const before = nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.max(scoped.timeoutSeconds, sourceWorkspace.install.timeoutSeconds + 60));
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapK8sScript(scoped.spec, sourceWorkspace, controlPlane, dryRun), "", Math.min(scoped.timeoutSeconds, 55));
const payload = parseLastJsonLineObject(statusText(result));
const after = dryRun ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
const wait = dryRun || result.exitCode !== 0 || payload.ok !== true
? null
: waitForSourceWorkspaceBootstrapJob(scoped, controlPlane.ciNamespace, sourceWorkspaceJobName(scoped.spec), sourceWorkspace.install.timeoutSeconds);
const after = dryRun || wait?.ok !== true ? null : nodeRuntimeSourceWorkspaceStatus({ ...scoped, originalArgs: ["source-workspace", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false }, sourceWorkspace);
const afterOk = after === null ? null : after.ok === true;
const payloadOk = payload.ok === true;
const ok = result.exitCode === 0 && payloadOk && (dryRun || afterOk === true);
const waitOk = dryRun ? true : wait?.ok === true;
const ok = result.exitCode === 0 && payloadOk && waitOk && (dryRun || afterOk === true);
return {
ok,
command: `hwlab nodes control-plane source-workspace bootstrap --node ${scoped.node} --lane ${scoped.lane}`,
@@ -102,6 +106,7 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo
before,
bootstrap: payload,
result: compactRuntimeCommand(result),
wait,
after,
degradedReason: ok ? undefined : dryRun ? "source-workspace-bootstrap-dry-run-failed" : "source-workspace-bootstrap-failed",
next: ok
@@ -279,6 +284,72 @@ function sleepMs(ms: number): void {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}
function waitForSourceWorkspaceBootstrapJob(scoped: ScopedNodeOptions, namespace: string, jobName: string, timeoutSeconds: number): Record<string, unknown> {
const startedAt = Date.now();
const deadline = startedAt + (timeoutSeconds + 60) * 1000;
let polls = 0;
let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
while (Date.now() <= deadline) {
polls += 1;
const result = runTransScript(scoped.node, sourceWorkspaceBootstrapJobStatusScript(namespace, jobName), "", Math.min(scoped.timeoutSeconds, 55));
const payload = parseLastJsonLineObject(statusText(result));
lastPayload = { ...payload, probe: compactRuntimeCommand(result) };
if (result.exitCode === 0 && payload.status === "succeeded" && payload.ok === true) {
return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
if (payload.status === "failed") {
return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
sleepMs(5000);
}
return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
}
function sourceWorkspaceBootstrapJobStatusScript(namespace: string, jobName: string): string {
return [
"set +e",
`namespace=${shellQuote(namespace)}`,
`job=${shellQuote(jobName)}`,
"tmp_dir=$(mktemp -d)",
"trap 'rm -rf \"$tmp_dir\"' EXIT",
"job_json=\"$tmp_dir/job.json\"",
"job_err=\"$tmp_dir/job.err\"",
"logs_file=\"$tmp_dir/logs.txt\"",
"kubectl -n \"$namespace\" get job \"$job\" -o json >\"$job_json\" 2>\"$job_err\"",
"job_exit=$?",
"kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true",
"python3 - \"$job_exit\" \"$namespace\" \"$job\" \"$job_json\" \"$job_err\" \"$logs_file\" <<'PY'",
"import json, pathlib, sys",
"job_exit = int(sys.argv[1])",
"namespace, job = sys.argv[2:4]",
"job_text = pathlib.Path(sys.argv[4]).read_text(errors='replace') if pathlib.Path(sys.argv[4]).exists() else ''",
"job_err = pathlib.Path(sys.argv[5]).read_text(errors='replace') if pathlib.Path(sys.argv[5]).exists() else ''",
"logs = pathlib.Path(sys.argv[6]).read_text(errors='replace') if pathlib.Path(sys.argv[6]).exists() else ''",
"job_obj = None",
"if job_exit == 0 and job_text.strip():",
" try:",
" job_obj = json.loads(job_text)",
" except Exception:",
" job_obj = None",
"conditions = (((job_obj or {}).get('status') or {}).get('conditions') or [])",
"complete = any((c or {}).get('type') == 'Complete' and (c or {}).get('status') == 'True' for c in conditions)",
"failed = any((c or {}).get('type') == 'Failed' and (c or {}).get('status') == 'True' for c in conditions)",
"job_payload = None",
"for line in reversed(logs.splitlines()):",
" text = line.strip()",
" if text.startswith('{') and text.endswith('}'):",
" try:",
" job_payload = json.loads(text)",
" break",
" except Exception:",
" pass",
"job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True",
"status = 'succeeded' if complete and job_ok else 'failed' if failed or (complete and not job_ok) else 'missing' if job_exit != 0 else 'running'",
"print(json.dumps({'ok': status == 'succeeded', 'status': status, 'namespace': namespace, 'job': job, 'jobExitCode': job_exit, 'complete': complete, 'failed': failed, 'active': ((job_obj or {}).get('status') or {}).get('active'), 'succeeded': ((job_obj or {}).get('status') or {}).get('succeeded'), 'failedCount': ((job_obj or {}).get('status') or {}).get('failed'), 'jobPayload': job_payload, 'jobErrorTail': job_err[-2000:], 'logsTail': logs[-4000:], 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
].join("\n");
}
function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
return {
workspace: spec.workspace,
@@ -620,7 +691,7 @@ function sourceWorkspaceBootstrapK8sScript(
"logs_file=\"$tmp_dir/logs.txt\"",
"printf '%s' \"$manifest_b64\" | base64 -d >\"$manifest\"",
"if [ \"$dry_run\" = true ]; then",
" kubectl -n \"$namespace\" apply --dry-run=server -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
" kubectl -n \"$namespace\" apply --dry-run=client -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
" apply_rc=$?",
" python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'",
"import json, pathlib, sys",
@@ -635,33 +706,15 @@ function sourceWorkspaceBootstrapK8sScript(
"kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true >/dev/null 2>&1",
"kubectl -n \"$namespace\" apply -f \"$manifest\" >\"$apply_out\" 2>\"$apply_err\"",
"apply_rc=$?",
"wait_rc=1",
"if [ \"$apply_rc\" = 0 ]; then",
" kubectl -n \"$namespace\" wait --for=condition=complete \"job/$job\" --timeout=\"${timeout_seconds}s\" >\"$wait_out\" 2>\"$wait_err\"",
" wait_rc=$?",
"fi",
"kubectl -n \"$namespace\" logs \"job/$job\" --tail=160 >\"$logs_file\" 2>&1 || true",
"python3 - \"$apply_rc\" \"$wait_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" \"$wait_out\" \"$wait_err\" \"$logs_file\" <<'PY'",
"python3 - \"$apply_rc\" \"$namespace\" \"$job\" \"$manifest_sha\" \"$apply_out\" \"$apply_err\" <<'PY'",
"import json, pathlib, sys",
"apply_rc = int(sys.argv[1])",
"wait_rc = int(sys.argv[2])",
"namespace, job, manifest_sha = sys.argv[3:6]",
"paths = [pathlib.Path(item) for item in sys.argv[6:]]",
"apply_out, apply_err, wait_out, wait_err, logs = [path.read_text(errors='replace') for path in paths]",
"job_payload = None",
"for line in reversed(logs.splitlines()):",
" text = line.strip()",
" if text.startswith('{') and text.endswith('}'):",
" try:",
" job_payload = json.loads(text)",
" break",
" except Exception:",
" pass",
"job_ok = isinstance(job_payload, dict) and job_payload.get('ok') is True",
"ok = apply_rc == 0 and wait_rc == 0 and job_ok",
"print(json.dumps({'ok': ok, 'status': 'succeeded' if ok else 'failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'waitExitCode': wait_rc, 'jobPayload': job_payload, 'applyOutputTail': apply_out[-2000:], 'applyErrorTail': apply_err[-2000:], 'waitOutputTail': wait_out[-2000:], 'waitErrorTail': wait_err[-2000:], 'logsTail': logs[-4000:], 'mutation': ok, 'valuesPrinted': False}, ensure_ascii=False))",
"namespace, job, manifest_sha = sys.argv[2:5]",
"apply_out = pathlib.Path(sys.argv[5]).read_text(errors='replace')[-2000:]",
"apply_err = pathlib.Path(sys.argv[6]).read_text(errors='replace')[-2000:]",
"print(json.dumps({'ok': apply_rc == 0, 'status': 'submitted' if apply_rc == 0 else 'submit-failed', 'namespace': namespace, 'job': job, 'manifestSha256': manifest_sha, 'applyExitCode': apply_rc, 'applyOutputTail': apply_out, 'applyErrorTail': apply_err, 'mutation': apply_rc == 0, 'valuesPrinted': False}, ensure_ascii=False))",
"PY",
"exit $([ \"$apply_rc\" = 0 ] && [ \"$wait_rc\" = 0 ] && printf 0 || printf 1)",
"exit \"$apply_rc\"",
].join("\n");
}