From 879facb93305edfe42849f0c08ca9fe49a52f5b8 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 5 Jun 2026 17:11:02 +0000 Subject: [PATCH] fix: poll v02 control-plane apply asynchronously --- scripts/hwlab-g14-contract-test.ts | 7 + scripts/src/hwlab-g14.ts | 220 ++++++++++++++++++++++++++++- 2 files changed, 225 insertions(+), 2 deletions(-) diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index ea14c6a7..666169a7 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -304,6 +304,13 @@ assertCondition( "v0.2 control-plane render must not use the fixed workspace checkout or its clean status", renderScript, ); +assertCondition( + sourceText.includes("function runG14K3sRemoteAsync") + && sourceText.includes("label: \"v02-control-plane-apply\"") + && sourceText.includes("remote async command timed out after") + && sourceText.includes("return runG14K3sRemoteAsync({"), + "v0.2 control-plane apply must use short start/poll remote-async semantics instead of one long G14:k3s apply call", +); const existingPipelineRunReuse = v02ExistingPipelineRunReuseDecision({ sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", before: { exists: true, status: "True" }, diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 35cb63f5..4527f37f 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -209,6 +209,14 @@ interface CommandJsonResult { parsed: unknown | null; } +interface RemoteAsyncCommandSpec { + script: string; + timeoutSeconds: number; + label: string; + token: string; + command: string[]; +} + interface ShellSection { stdout: string; exitCode: number | null; @@ -875,6 +883,205 @@ function g14K3sInlineScriptWithInput(script: string, input: string, timeoutMs = return commandJsonWithInput(["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "script", "--", script], input, timeoutMs); } +function remoteAsyncStatusDir(label: string): string { + return `/tmp/hwlab-${label}-async`; +} + +function remoteAsyncStatusPath(label: string, token: string): string { + return `${remoteAsyncStatusDir(label)}/${token}.json`; +} + +function remoteAsyncStartScript(spec: RemoteAsyncCommandSpec): string { + const statusPath = remoteAsyncStatusPath(spec.label, spec.token); + const donePath = `${statusPath}.done`; + const stdoutPath = `${statusPath}.stdout`; + const stderrPath = `${statusPath}.stderr`; + const commandB64 = Buffer.from(spec.script, "utf8").toString("base64"); + return [ + "set -eu", + `status_dir=${shellQuote(remoteAsyncStatusDir(spec.label))}`, + `status_path=${shellQuote(statusPath)}`, + `done_path=${shellQuote(donePath)}`, + `stdout_path=${shellQuote(stdoutPath)}`, + `stderr_path=${shellQuote(stderrPath)}`, + `command_b64=${shellQuote(commandB64)}`, + "mkdir -p \"$status_dir\"", + "rm -f \"$status_path\" \"$done_path\" \"$stdout_path\" \"$stderr_path\"", + "command_path=\"$status_path.command.sh\"", + "printf '%s' \"$command_b64\" | base64 -d > \"$command_path\"", + "chmod +x \"$command_path\"", + "node -e 'const fs=require(\"fs\"); const p=process.argv[1]; const payload={ok:null,state:\"running\",pid:Number(process.argv[2]),startedAt:new Date().toISOString(),finishedAt:null,exitCode:null,stdout:\"\",stderr:\"\"}; fs.writeFileSync(p, JSON.stringify(payload));' \"$status_path\" $$", + "(", + " set +e", + " sh \"$command_path\" >\"$stdout_path\" 2>\"$stderr_path\"", + " code=$?", + " node - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'NODE'", + "const fs = require('fs');", + "const [statusPath, stdoutPath, stderrPath, codeRaw] = process.argv.slice(2);", + "const code = Number(codeRaw);", + "const read = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };", + "fs.writeFileSync(statusPath, JSON.stringify({", + " ok: code === 0,", + " state: 'finished',", + " pid: null,", + " startedAt: null,", + " finishedAt: new Date().toISOString(),", + " exitCode: code,", + " stdout: read(stdoutPath),", + " stderr: read(stderrPath),", + "}));", + "NODE", + " touch \"$done_path\"", + ") >/dev/null 2>&1 &", + "pid=$!", + "node - \"$status_path\" \"$pid\" <<'NODE'", + "const fs = require('fs');", + "const [statusPath, pidRaw] = process.argv.slice(2);", + "const current = JSON.parse(fs.readFileSync(statusPath, 'utf8'));", + "current.pid = Number(pidRaw);", + "fs.writeFileSync(statusPath, JSON.stringify(current));", + "console.log(JSON.stringify({ ok: true, state: current.state, pid: current.pid, statusPath }));", + "NODE", + ].join("\n"); +} + +function remoteAsyncStatusScript(label: string, token: string): string { + const statusPath = remoteAsyncStatusPath(label, token); + const donePath = `${statusPath}.done`; + return [ + "set -eu", + `status_path=${shellQuote(statusPath)}`, + `done_path=${shellQuote(donePath)}`, + "if [ ! -f \"$status_path\" ]; then", + " printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing\"}'", + " exit 0", + "fi", + "if [ -f \"$done_path\" ]; then", + " cat \"$status_path\"", + " exit 0", + "fi", + "node - \"$status_path\" <<'NODE'", + "const fs = require('fs');", + "const statusPath = process.argv[2];", + "const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));", + "payload.state = payload.state || 'running';", + "payload.ok = payload.ok ?? null;", + "payload.stdout = payload.stdout || '';", + "payload.stderr = payload.stderr || '';", + "console.log(JSON.stringify(payload));", + "NODE", + ].join("\n"); +} + +function remoteAsyncKillScript(label: string, token: string): string { + const statusPath = remoteAsyncStatusPath(label, token); + const donePath = `${statusPath}.done`; + return [ + "set -eu", + `status_path=${shellQuote(statusPath)}`, + `done_path=${shellQuote(donePath)}`, + "if [ ! -f \"$status_path\" ]; then", + " printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing during kill\"}'", + " exit 0", + "fi", + "node - \"$status_path\" <<'NODE'", + "const fs = require('fs');", + "const statusPath = process.argv[2];", + "const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));", + "if (Number.isInteger(payload.pid) && payload.pid > 0) {", + " try { process.kill(payload.pid, 'TERM'); } catch {}", + "}", + "payload.ok = false;", + "payload.state = 'timeout-killed';", + "payload.exitCode = 124;", + "payload.finishedAt = new Date().toISOString();", + "payload.stderr = `${payload.stderr || ''}\\nremote async command exceeded local poll timeout and was terminated`;", + "fs.writeFileSync(statusPath, JSON.stringify(payload));", + "console.log(JSON.stringify(payload));", + "NODE", + "touch \"$done_path\"", + ].join("\n"); +} + +function parseRemoteAsyncPayload(result: CommandJsonResult): Record { + const text = result.stdout.trim(); + if (text.length === 0) return {}; + try { + return record(JSON.parse(text) as unknown); + } catch { + return {}; + } +} + +function runG14K3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult { + const startedAtMs = Date.now(); + const start = g14K3s(["script", "--", remoteAsyncStartScript(spec)], 55_000); + const startPayload = parseRemoteAsyncPayload(start); + if (!isCommandSuccess(start) || startPayload.ok !== true) return start; + const deadlineMs = startedAtMs + Math.max(1, spec.timeoutSeconds) * 1000; + let attempts = 0; + let lastStatus: CommandJsonResult | null = null; + while (Date.now() <= deadlineMs) { + attempts += 1; + const wait = runCommand(["sleep", attempts <= 1 ? "1" : "2"], repoRoot, { timeoutMs: 3_000 }); + if (wait.exitCode !== 0 && wait.timedOut) break; + const status = g14K3s(["script", "--", remoteAsyncStatusScript(spec.label, spec.token)], 55_000); + lastStatus = status; + const payload = parseRemoteAsyncPayload(status); + const state = typeof payload.state === "string" ? payload.state : null; + if (state === "finished" || payload.ok === true || payload.ok === false) { + return { + ok: payload.ok === true, + command: spec.command, + exitCode: typeof payload.exitCode === "number" ? payload.exitCode : status.exitCode, + stdout: typeof payload.stdout === "string" ? payload.stdout : status.stdout, + stderr: typeof payload.stderr === "string" ? payload.stderr : status.stderr, + parsed: { + ok: payload.ok === true, + data: { + mode: "remote-async", + label: spec.label, + token: spec.token, + statusPath: remoteAsyncStatusPath(spec.label, spec.token), + attempts, + elapsedMs: Date.now() - startedAtMs, + state, + pid: payload.pid ?? null, + }, + }, + }; + } + } + const killed = g14K3s(["script", "--", remoteAsyncKillScript(spec.label, spec.token)], 55_000); + const payload = parseRemoteAsyncPayload(killed); + return { + ok: false, + command: spec.command, + exitCode: 124, + stdout: typeof payload.stdout === "string" ? payload.stdout : "", + stderr: [ + `remote async command timed out after ${spec.timeoutSeconds}s`, + typeof payload.stderr === "string" ? payload.stderr : "", + lastStatus === null ? "" : `last status stderr: ${lastStatus.stderr.trim()}`, + killed.stderr.trim(), + ].filter(Boolean).join("\n"), + parsed: { + ok: false, + data: { + mode: "remote-async", + label: spec.label, + token: spec.token, + statusPath: remoteAsyncStatusPath(spec.label, spec.token), + attempts, + elapsedMs: Date.now() - startedAtMs, + state: typeof payload.state === "string" ? payload.state : "timeout", + pid: payload.pid ?? null, + killed: compactCommandMetadata(killed, true, false), + }, + }, + }; +} + function v02CicdRepoEnsureScript(): string { return [ `cicd_repo=${shellQuote(V02_CICD_REPO)}`, @@ -2638,7 +2845,7 @@ function cleanupV02RenderDir(renderDir: string): CommandJsonResult { } function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { - return g14K3s([ + const args = [ "kubectl", "apply", "--server-side", @@ -2653,7 +2860,16 @@ function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSe `${renderDir}/argocd/project.yaml`, "-f", `${renderDir}/argocd/application-v02.yaml`, - ], timeoutSeconds * 1000); + ]; + const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...args]; + const shellCommand = args.map(shellQuote).join(" "); + return runG14K3sRemoteAsync({ + script: `exec ${shellCommand}`, + timeoutSeconds, + label: "v02-control-plane-apply", + token: v02RenderToken(), + command, + }); } interface V02RefreshMarker {