fix: poll v02 control-plane apply asynchronously

2026-06-05 17:11:02 +00:00
parent 74dc0b8c62
commit 879facb933
2 changed files with 225 additions and 2 deletions
@@ -304,6 +304,13 @@ assertCondition(
  "v0.2 control-plane render must not use the fixed workspace checkout or its clean status",
  renderScript,
 );
+assertCondition(
+  sourceText.includes("function runG14K3sRemoteAsync")
+    && sourceText.includes("label: \"v02-control-plane-apply\"")
+    && sourceText.includes("remote async command timed out after")
+    && sourceText.includes("return runG14K3sRemoteAsync({"),
+  "v0.2 control-plane apply must use short start/poll remote-async semantics instead of one long G14:k3s apply call",
+);
 const existingPipelineRunReuse = v02ExistingPipelineRunReuseDecision({
  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
  before: { exists: true, status: "True" },
@@ -209,6 +209,14 @@ interface CommandJsonResult {
  parsed: unknown | null;
 }

+interface RemoteAsyncCommandSpec {
+  script: string;
+  timeoutSeconds: number;
+  label: string;
+  token: string;
+  command: string[];
+}
+
 interface ShellSection {
  stdout: string;
  exitCode: number | null;
@@ -875,6 +883,205 @@ function g14K3sInlineScriptWithInput(script: string, input: string, timeoutMs =
  return commandJsonWithInput(["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "script", "--", script], input, timeoutMs);
 }

+function remoteAsyncStatusDir(label: string): string {
+  return `/tmp/hwlab-${label}-async`;
+}
+
+function remoteAsyncStatusPath(label: string, token: string): string {
+  return `${remoteAsyncStatusDir(label)}/${token}.json`;
+}
+
+function remoteAsyncStartScript(spec: RemoteAsyncCommandSpec): string {
+  const statusPath = remoteAsyncStatusPath(spec.label, spec.token);
+  const donePath = `${statusPath}.done`;
+  const stdoutPath = `${statusPath}.stdout`;
+  const stderrPath = `${statusPath}.stderr`;
+  const commandB64 = Buffer.from(spec.script, "utf8").toString("base64");
+  return [
+    "set -eu",
+    `status_dir=${shellQuote(remoteAsyncStatusDir(spec.label))}`,
+    `status_path=${shellQuote(statusPath)}`,
+    `done_path=${shellQuote(donePath)}`,
+    `stdout_path=${shellQuote(stdoutPath)}`,
+    `stderr_path=${shellQuote(stderrPath)}`,
+    `command_b64=${shellQuote(commandB64)}`,
+    "mkdir -p \"$status_dir\"",
+    "rm -f \"$status_path\" \"$done_path\" \"$stdout_path\" \"$stderr_path\"",
+    "command_path=\"$status_path.command.sh\"",
+    "printf '%s' \"$command_b64\" | base64 -d > \"$command_path\"",
+    "chmod +x \"$command_path\"",
+    "node -e 'const fs=require(\"fs\"); const p=process.argv[1]; const payload={ok:null,state:\"running\",pid:Number(process.argv[2]),startedAt:new Date().toISOString(),finishedAt:null,exitCode:null,stdout:\"\",stderr:\"\"}; fs.writeFileSync(p, JSON.stringify(payload));' \"$status_path\" $$",
+    "(",
+    "  set +e",
+    "  sh \"$command_path\" >\"$stdout_path\" 2>\"$stderr_path\"",
+    "  code=$?",
+    "  node - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'NODE'",
+    "const fs = require('fs');",
+    "const [statusPath, stdoutPath, stderrPath, codeRaw] = process.argv.slice(2);",
+    "const code = Number(codeRaw);",
+    "const read = (path) => { try { return fs.readFileSync(path, 'utf8'); } catch { return ''; } };",
+    "fs.writeFileSync(statusPath, JSON.stringify({",
+    "  ok: code === 0,",
+    "  state: 'finished',",
+    "  pid: null,",
+    "  startedAt: null,",
+    "  finishedAt: new Date().toISOString(),",
+    "  exitCode: code,",
+    "  stdout: read(stdoutPath),",
+    "  stderr: read(stderrPath),",
+    "}));",
+    "NODE",
+    "  touch \"$done_path\"",
+    ") >/dev/null 2>&1 &",
+    "pid=$!",
+    "node - \"$status_path\" \"$pid\" <<'NODE'",
+    "const fs = require('fs');",
+    "const [statusPath, pidRaw] = process.argv.slice(2);",
+    "const current = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
+    "current.pid = Number(pidRaw);",
+    "fs.writeFileSync(statusPath, JSON.stringify(current));",
+    "console.log(JSON.stringify({ ok: true, state: current.state, pid: current.pid, statusPath }));",
+    "NODE",
+  ].join("\n");
+}
+
+function remoteAsyncStatusScript(label: string, token: string): string {
+  const statusPath = remoteAsyncStatusPath(label, token);
+  const donePath = `${statusPath}.done`;
+  return [
+    "set -eu",
+    `status_path=${shellQuote(statusPath)}`,
+    `done_path=${shellQuote(donePath)}`,
+    "if [ ! -f \"$status_path\" ]; then",
+    "  printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing\"}'",
+    "  exit 0",
+    "fi",
+    "if [ -f \"$done_path\" ]; then",
+    "  cat \"$status_path\"",
+    "  exit 0",
+    "fi",
+    "node - \"$status_path\" <<'NODE'",
+    "const fs = require('fs');",
+    "const statusPath = process.argv[2];",
+    "const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
+    "payload.state = payload.state || 'running';",
+    "payload.ok = payload.ok ?? null;",
+    "payload.stdout = payload.stdout || '';",
+    "payload.stderr = payload.stderr || '';",
+    "console.log(JSON.stringify(payload));",
+    "NODE",
+  ].join("\n");
+}
+
+function remoteAsyncKillScript(label: string, token: string): string {
+  const statusPath = remoteAsyncStatusPath(label, token);
+  const donePath = `${statusPath}.done`;
+  return [
+    "set -eu",
+    `status_path=${shellQuote(statusPath)}`,
+    `done_path=${shellQuote(donePath)}`,
+    "if [ ! -f \"$status_path\" ]; then",
+    "  printf '%s\\n' '{\"ok\":false,\"state\":\"missing\",\"exitCode\":null,\"stdout\":\"\",\"stderr\":\"remote async status file is missing during kill\"}'",
+    "  exit 0",
+    "fi",
+    "node - \"$status_path\" <<'NODE'",
+    "const fs = require('fs');",
+    "const statusPath = process.argv[2];",
+    "const payload = JSON.parse(fs.readFileSync(statusPath, 'utf8'));",
+    "if (Number.isInteger(payload.pid) && payload.pid > 0) {",
+    "  try { process.kill(payload.pid, 'TERM'); } catch {}",
+    "}",
+    "payload.ok = false;",
+    "payload.state = 'timeout-killed';",
+    "payload.exitCode = 124;",
+    "payload.finishedAt = new Date().toISOString();",
+    "payload.stderr = `${payload.stderr || ''}\\nremote async command exceeded local poll timeout and was terminated`;",
+    "fs.writeFileSync(statusPath, JSON.stringify(payload));",
+    "console.log(JSON.stringify(payload));",
+    "NODE",
+    "touch \"$done_path\"",
+  ].join("\n");
+}
+
+function parseRemoteAsyncPayload(result: CommandJsonResult): Record<string, unknown> {
+  const text = result.stdout.trim();
+  if (text.length === 0) return {};
+  try {
+    return record(JSON.parse(text) as unknown);
+  } catch {
+    return {};
+  }
+}
+
+function runG14K3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult {
+  const startedAtMs = Date.now();
+  const start = g14K3s(["script", "--", remoteAsyncStartScript(spec)], 55_000);
+  const startPayload = parseRemoteAsyncPayload(start);
+  if (!isCommandSuccess(start) || startPayload.ok !== true) return start;
+  const deadlineMs = startedAtMs + Math.max(1, spec.timeoutSeconds) * 1000;
+  let attempts = 0;
+  let lastStatus: CommandJsonResult | null = null;
+  while (Date.now() <= deadlineMs) {
+    attempts += 1;
+    const wait = runCommand(["sleep", attempts <= 1 ? "1" : "2"], repoRoot, { timeoutMs: 3_000 });
+    if (wait.exitCode !== 0 && wait.timedOut) break;
+    const status = g14K3s(["script", "--", remoteAsyncStatusScript(spec.label, spec.token)], 55_000);
+    lastStatus = status;
+    const payload = parseRemoteAsyncPayload(status);
+    const state = typeof payload.state === "string" ? payload.state : null;
+    if (state === "finished" || payload.ok === true || payload.ok === false) {
+      return {
+        ok: payload.ok === true,
+        command: spec.command,
+        exitCode: typeof payload.exitCode === "number" ? payload.exitCode : status.exitCode,
+        stdout: typeof payload.stdout === "string" ? payload.stdout : status.stdout,
+        stderr: typeof payload.stderr === "string" ? payload.stderr : status.stderr,
+        parsed: {
+          ok: payload.ok === true,
+          data: {
+            mode: "remote-async",
+            label: spec.label,
+            token: spec.token,
+            statusPath: remoteAsyncStatusPath(spec.label, spec.token),
+            attempts,
+            elapsedMs: Date.now() - startedAtMs,
+            state,
+            pid: payload.pid ?? null,
+          },
+        },
+      };
+    }
+  }
+  const killed = g14K3s(["script", "--", remoteAsyncKillScript(spec.label, spec.token)], 55_000);
+  const payload = parseRemoteAsyncPayload(killed);
+  return {
+    ok: false,
+    command: spec.command,
+    exitCode: 124,
+    stdout: typeof payload.stdout === "string" ? payload.stdout : "",
+    stderr: [
+      `remote async command timed out after ${spec.timeoutSeconds}s`,
+      typeof payload.stderr === "string" ? payload.stderr : "",
+      lastStatus === null ? "" : `last status stderr: ${lastStatus.stderr.trim()}`,
+      killed.stderr.trim(),
+    ].filter(Boolean).join("\n"),
+    parsed: {
+      ok: false,
+      data: {
+        mode: "remote-async",
+        label: spec.label,
+        token: spec.token,
+        statusPath: remoteAsyncStatusPath(spec.label, spec.token),
+        attempts,
+        elapsedMs: Date.now() - startedAtMs,
+        state: typeof payload.state === "string" ? payload.state : "timeout",
+        pid: payload.pid ?? null,
+        killed: compactCommandMetadata(killed, true, false),
+      },
+    },
+  };
+}
+
 function v02CicdRepoEnsureScript(): string {
  return [
    `cicd_repo=${shellQuote(V02_CICD_REPO)}`,
@@ -2638,7 +2845,7 @@ function cleanupV02RenderDir(renderDir: string): CommandJsonResult {
 }

 function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
-  return g14K3s([
+  const args = [
    "kubectl",
    "apply",
    "--server-side",
@@ -2653,7 +2860,16 @@ function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSe
    `${renderDir}/argocd/project.yaml`,
    "-f",
    `${renderDir}/argocd/application-v02.yaml`,
-  ], timeoutSeconds * 1000);
+  ];
+  const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...args];
+  const shellCommand = args.map(shellQuote).join(" ");
+  return runG14K3sRemoteAsync({
+    script: `exec ${shellCommand}`,
+    timeoutSeconds,
+    label: "v02-control-plane-apply",
+    token: v02RenderToken(),
+    command,
+  });
 }

 interface V02RefreshMarker {