feat: add YAML-first source workspace host deps

2026-06-29 09:16:16 +00:00
parent d30e5853fa
commit c9fd499fd2
3 changed files with 369 additions and 4 deletions
@@ -3,14 +3,15 @@
 import { createHash } from "node:crypto";
 import { posix as posixPath } from "node:path";
 import { hwlabNodeControlPlaneSourceWorkspaceBootstrap } from "../hwlab-node-control-plane";
-import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeSourceWorkspaceSpec } from "../hwlab-node-lanes";
+import { hwlabRuntimeLaneConfigPath, type HwlabRuntimeLaneSpec, type HwlabRuntimeSourceWorkspaceHostDependenciesSpec, type HwlabRuntimeSourceWorkspaceSpec } from "../hwlab-node-lanes";
 import { parseNodeScopedDelegatedOptions } from "./plan";
 import { runTransHostScript, runTransScript } from "./public-exposure";
 import { compactRuntimeCommand, parseLastJsonLineObject } from "./runtime-common";
 import { commaListField, keyValueLinesFromText, numericField, shellQuote, statusText } from "./utils";

 type ScopedNodeOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
-type SourceWorkspaceAction = "status" | "bootstrap";
+type SourceWorkspaceAction = "status" | "bootstrap" | "host-deps";
+type SourceWorkspaceHostDependenciesAction = "status" | "sync";

 export function nodeRuntimeSourceWorkspaceCommand(scoped: ScopedNodeOptions): Record<string, unknown> {
  const action = sourceWorkspaceAction(scoped);
@@ -27,14 +28,15 @@ export function nodeRuntimeSourceWorkspaceCommand(scoped: ScopedNodeOptions): Re
      configPath: hwlabRuntimeLaneConfigPath(),
    };
  }
+  if (action === "host-deps") return nodeRuntimeSourceWorkspaceHostDependencies(scoped, sourceWorkspace);
  if (action === "status") return nodeRuntimeSourceWorkspaceStatus(scoped, sourceWorkspace);
  return nodeRuntimeSourceWorkspaceBootstrap(scoped, sourceWorkspace);
 }

 function sourceWorkspaceAction(scoped: ScopedNodeOptions): SourceWorkspaceAction {
  const value = scoped.originalArgs[1];
-  if (value !== "status" && value !== "bootstrap") {
-    throw new Error("control-plane source-workspace usage: source-workspace status|bootstrap --node NODE --lane vNN [--dry-run|--confirm]");
+  if (value !== "status" && value !== "bootstrap" && value !== "host-deps") {
+    throw new Error("control-plane source-workspace usage: source-workspace status|bootstrap|host-deps --node NODE --lane vNN [--dry-run|--confirm]");
  }
  if (value === "status" && (scoped.confirm || scoped.dryRun)) {
    throw new Error("control-plane source-workspace status is read-only and does not accept --dry-run or --confirm");
@@ -42,6 +44,17 @@ function sourceWorkspaceAction(scoped: ScopedNodeOptions): SourceWorkspaceAction
  return value;
 }

+function sourceWorkspaceHostDependenciesAction(scoped: ScopedNodeOptions): SourceWorkspaceHostDependenciesAction {
+  const value = scoped.originalArgs[2];
+  if (value !== "status" && value !== "sync") {
+    throw new Error("control-plane source-workspace host-deps usage: source-workspace host-deps status|sync --node NODE --lane vNN [--dry-run|--confirm] [--wait]");
+  }
+  if (value === "status" && (scoped.confirm || scoped.dryRun || scoped.wait)) {
+    throw new Error("control-plane source-workspace host-deps status is read-only and does not accept --dry-run, --confirm or --wait");
+  }
+  return value;
+}
+
 function nodeRuntimeSourceWorkspaceStatus(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
  const result = runTransHostScript(scoped.node, sourceWorkspaceStatusScript(scoped.spec, sourceWorkspace), "", scoped.timeoutSeconds);
  const fields = keyValueLinesFromText(statusText(result));
@@ -97,6 +110,175 @@ function nodeRuntimeSourceWorkspaceBootstrap(scoped: ScopedNodeOptions, sourceWo
  };
 }

+function nodeRuntimeSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
+  const action = sourceWorkspaceHostDependenciesAction(scoped);
+  const hostDependencies = sourceWorkspace.hostDependencies;
+  if (hostDependencies === undefined) {
+    return {
+      ok: false,
+      command: `hwlab nodes control-plane source-workspace host-deps ${action} --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mutation: false,
+      configPath: hwlabRuntimeLaneConfigPath(),
+      degradedReason: "source-workspace-host-dependencies-yaml-missing",
+      message: "lanes.<lane>.targets.<node>.sourceWorkspace.hostDependencies must declare checkCommands, stateDir and install before this controlled entry can run.",
+    };
+  }
+  if (action === "status") return nodeRuntimeSourceWorkspaceHostDependenciesStatus(scoped, hostDependencies);
+  return nodeRuntimeSourceWorkspaceHostDependenciesSync(scoped, sourceWorkspace, hostDependencies);
+}
+
+function nodeRuntimeSourceWorkspaceHostDependenciesStatus(scoped: ScopedNodeOptions, hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
+  const result = runTransHostScript(scoped.node, sourceWorkspaceHostDependenciesStatusScript(hostDependencies), "", scoped.timeoutSeconds);
+  const payload = parseLastJsonLineObject(statusText(result));
+  const ok = result.exitCode === 0 && payload.ok === true;
+  return {
+    ok,
+    command: `hwlab nodes control-plane source-workspace host-deps status --node ${scoped.node} --lane ${scoped.lane}`,
+    node: scoped.node,
+    lane: scoped.lane,
+    mode: "read-only",
+    mutation: false,
+    configPath: hwlabRuntimeLaneConfigPath(),
+    expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
+    status: payload,
+    probe: compactRuntimeCommand(result),
+    degradedReason: ok ? undefined : "source-workspace-host-dependencies-not-ready",
+    next: ok ? undefined : { sync: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
+  };
+}
+
+function nodeRuntimeSourceWorkspaceHostDependenciesSync(
+  scoped: ScopedNodeOptions,
+  sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
+  hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec,
+): Record<string, unknown> {
+  if (scoped.confirm && scoped.dryRun) throw new Error("control-plane source-workspace host-deps sync accepts only one of --dry-run or --confirm");
+  const dryRun = scoped.dryRun || !scoped.confirm;
+  const before = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
+  if (before.ok === true) {
+    return {
+      ok: true,
+      command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mode: dryRun ? "dry-run" : "already-ready",
+      mutation: false,
+      configPath: hwlabRuntimeLaneConfigPath(),
+      expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
+      before,
+      sync: { ok: true, status: "skipped", reason: "host-dependencies-already-ready", valuesPrinted: false },
+      after: before,
+      next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` },
+    };
+  }
+  if (dryRun) {
+    return {
+      ok: true,
+      command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mode: "dry-run",
+      mutation: false,
+      configPath: hwlabRuntimeLaneConfigPath(),
+      expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
+      before,
+      sync: {
+        ok: true,
+        status: "dry-run",
+        stateDir: hostDependencies.stateDir,
+        checkCommands: hostDependencies.checkCommands,
+        installCommandSha256: createHash("sha256").update(hostDependencies.install.command).digest("hex"),
+        timeoutSeconds: hostDependencies.install.timeoutSeconds,
+        valuesPrinted: false,
+      },
+      after: null,
+      next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
+    };
+  }
+  const submit = runTransHostScript(scoped.node, sourceWorkspaceHostDependenciesSubmitScript(scoped.spec, sourceWorkspace, hostDependencies), "", Math.min(scoped.timeoutSeconds, 55));
+  const submitted = parseLastJsonLineObject(statusText(submit));
+  if (submit.exitCode !== 0 || submitted.ok !== true) {
+    return {
+      ok: false,
+      command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mode: "confirmed-submit",
+      mutation: false,
+      configPath: hwlabRuntimeLaneConfigPath(),
+      expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
+      before,
+      sync: submitted,
+      result: compactRuntimeCommand(submit),
+      after: null,
+      degradedReason: "source-workspace-host-dependencies-submit-failed",
+      next: { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
+    };
+  }
+  if (!scoped.wait) {
+    return {
+      ok: true,
+      command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
+      node: scoped.node,
+      lane: scoped.lane,
+      mode: "submitted",
+      mutation: true,
+      configPath: hwlabRuntimeLaneConfigPath(),
+      expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
+      before,
+      sync: submitted,
+      result: compactRuntimeCommand(submit),
+      after: null,
+      next: { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps status --node ${scoped.node} --lane ${scoped.lane}` },
+    };
+  }
+  const waited = waitForSourceWorkspaceHostDependencies(scoped, hostDependencies);
+  const after = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
+  const ok = waited.ok === true && after.ok === true;
+  return {
+    ok,
+    command: `hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane}`,
+    node: scoped.node,
+    lane: scoped.lane,
+    mode: "confirmed-wait",
+    mutation: true,
+    configPath: hwlabRuntimeLaneConfigPath(),
+    expected: sourceWorkspaceHostDependenciesExpected(hostDependencies),
+    before,
+    sync: submitted,
+    result: compactRuntimeCommand(submit),
+    wait: waited,
+    after,
+    degradedReason: ok ? undefined : "source-workspace-host-dependencies-sync-failed",
+    next: ok
+      ? { status: `bun scripts/cli.ts hwlab nodes control-plane source-workspace status --node ${scoped.node} --lane ${scoped.lane}` }
+      : { retry: `bun scripts/cli.ts hwlab nodes control-plane source-workspace host-deps sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
+  };
+}
+
+function waitForSourceWorkspaceHostDependencies(scoped: ScopedNodeOptions, hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
+  const startedAt = Date.now();
+  const deadline = startedAt + (hostDependencies.install.timeoutSeconds + 60) * 1000;
+  let polls = 0;
+  let lastPayload: Record<string, unknown> = { ok: false, status: "not-polled", valuesPrinted: false };
+  while (Date.now() <= deadline) {
+    polls += 1;
+    const status = nodeRuntimeSourceWorkspaceHostDependenciesStatus({ ...scoped, originalArgs: ["source-workspace", "host-deps", "status", "--node", scoped.node, "--lane", scoped.lane], confirm: false, dryRun: false, wait: false }, hostDependencies);
+    lastPayload = typeof status.status === "object" && status.status !== null ? status.status as Record<string, unknown> : lastPayload;
+    if (status.ok === true) return { ok: true, status: "succeeded", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
+    const lastInstall = typeof lastPayload.lastInstall === "object" && lastPayload.lastInstall !== null ? lastPayload.lastInstall as Record<string, unknown> : {};
+    if (lastInstall.status === "failed") return { ok: false, status: "failed", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
+    sleepMs(5000);
+  }
+  return { ok: false, status: "timeout", polls, elapsedMs: Date.now() - startedAt, lastStatus: lastPayload, valuesPrinted: false };
+}
+
+function sleepMs(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
 function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): Record<string, unknown> {
  return {
    workspace: spec.workspace,
@@ -108,6 +290,7 @@ function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: Hw
    },
    requiredCommands: sourceWorkspace.requiredCommands,
    requiredFiles: sourceWorkspace.requiredFiles,
+    hostDependencies: sourceWorkspace.hostDependencies === undefined ? null : sourceWorkspaceHostDependenciesExpected(sourceWorkspace.hostDependencies),
    install: sourceWorkspace.install,
    downloadProfile: {
      id: spec.downloadProfileId,
@@ -121,6 +304,17 @@ function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: Hw
  };
 }

+function sourceWorkspaceHostDependenciesExpected(hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): Record<string, unknown> {
+  return {
+    checkCommands: hostDependencies.checkCommands,
+    stateDir: hostDependencies.stateDir,
+    install: {
+      commandSha256: createHash("sha256").update(hostDependencies.install.command).digest("hex"),
+      timeoutSeconds: hostDependencies.install.timeoutSeconds,
+    },
+  };
+}
+
 function sourceWorkspaceStatusFromFields(fields: Record<string, string>, exitCode: number | null): Record<string, unknown> {
  const missingCommands = commaListField(fields.missingCommands);
  const missingFiles = commaListField(fields.missingFiles);
@@ -168,6 +362,116 @@ function sourceWorkspaceStatusFromFields(fields: Record<string, string>, exitCod
  };
 }

+function sourceWorkspaceHostDependenciesStatusScript(hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec): string {
+  const checkCommands = hostDependencies.checkCommands.join("\n");
+  return [
+    "set +e",
+    `state_dir=${shellQuote(hostDependencies.stateDir)}`,
+    `check_commands_b64=${shellQuote(Buffer.from(checkCommands, "utf8").toString("base64"))}`,
+    "mkdir -p \"$state_dir\" 2>/tmp/hwlab-source-workspace-host-deps-mkdir.err",
+    "mkdir_exit=$?",
+    "commands_file=$(mktemp)",
+    "trap 'rm -f \"$commands_file\"' EXIT",
+    "printf '%s' \"$check_commands_b64\" | base64 -d >\"$commands_file\"",
+    "missing_commands=",
+    "present_commands=",
+    "while IFS= read -r command_name; do",
+    "  [ -z \"$command_name\" ] && continue",
+    "  if command -v \"$command_name\" >/dev/null 2>&1; then",
+    "    present_commands=\"$present_commands${present_commands:+,}$command_name\"",
+    "  else",
+    "    missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"",
+    "  fi",
+    "done <\"$commands_file\"",
+    "status_file=\"$state_dir/install-status.json\"",
+    "last_install_json='{}'",
+    "if [ -f \"$status_file\" ]; then last_install_json=$(tail -c 20000 \"$status_file\" 2>/dev/null || printf '{}'); fi",
+    "STATE_DIR=\"$state_dir\" MKDIR_EXIT=\"$mkdir_exit\" MISSING_COMMANDS=\"$missing_commands\" PRESENT_COMMANDS=\"$present_commands\" STATUS_FILE=\"$status_file\" LAST_INSTALL_JSON=\"$last_install_json\" node <<'NODE'",
+    "const lastRaw = process.env.LAST_INSTALL_JSON || '{}';",
+    "let lastInstall = {};",
+    "try { lastInstall = JSON.parse(lastRaw); } catch { lastInstall = { ok: false, status: 'unparseable', valuesPrinted: false }; }",
+    "const missing = (process.env.MISSING_COMMANDS || '').split(',').filter(Boolean);",
+    "const present = (process.env.PRESENT_COMMANDS || '').split(',').filter(Boolean);",
+    "const mkdirExit = Number(process.env.MKDIR_EXIT || '1');",
+    "console.log(JSON.stringify({ ok: mkdirExit === 0 && missing.length === 0, status: mkdirExit === 0 && missing.length === 0 ? 'ready' : 'missing', stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, presentCommands: present, missingCommands: missing, mkdirExitCode: mkdirExit, lastInstall, valuesPrinted: false }));",
+    "NODE",
+  ].join("\n");
+}
+
+function sourceWorkspaceHostDependenciesSubmitScript(
+  spec: HwlabRuntimeLaneSpec,
+  sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec,
+  hostDependencies: HwlabRuntimeSourceWorkspaceHostDependenciesSpec,
+): string {
+  const checkCommands = hostDependencies.checkCommands.join("\n");
+  const noProxy = spec.networkProfile.proxy.noProxy.join(",");
+  const commandSha = createHash("sha256").update(hostDependencies.install.command).digest("hex");
+  const requiredCommandsSha = createHash("sha256").update(sourceWorkspace.requiredCommands.join("\n")).digest("hex");
+  return [
+    "set +e",
+    `state_dir=${shellQuote(hostDependencies.stateDir)}`,
+    `install_command_b64=${shellQuote(Buffer.from(hostDependencies.install.command, "utf8").toString("base64"))}`,
+    `check_commands_b64=${shellQuote(Buffer.from(checkCommands, "utf8").toString("base64"))}`,
+    `timeout_seconds=${String(hostDependencies.install.timeoutSeconds)}`,
+    `http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`,
+    `https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`,
+    `all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`,
+    `no_proxy_value=${shellQuote(noProxy)}`,
+    `command_sha=${shellQuote(commandSha)}`,
+    `required_commands_sha=${shellQuote(requiredCommandsSha)}`,
+    "mkdir -p \"$state_dir\"",
+    "mkdir_exit=$?",
+    "if [ \"$mkdir_exit\" -ne 0 ]; then",
+    "  MKDIR_EXIT=\"$mkdir_exit\" STATE_DIR=\"$state_dir\" node <<'NODE'",
+    "console.log(JSON.stringify({ ok: false, status: 'submit-failed', stateDir: process.env.STATE_DIR, mkdirExitCode: Number(process.env.MKDIR_EXIT || '1'), valuesPrinted: false }));",
+    "NODE",
+    "  exit \"$mkdir_exit\"",
+    "fi",
+    "job_id=\"$(date -u +%Y%m%dT%H%M%SZ)-$$\"",
+    "status_file=\"$state_dir/install-status.json\"",
+    "stdout_file=\"$state_dir/$job_id.stdout.log\"",
+    "stderr_file=\"$state_dir/$job_id.stderr.log\"",
+    "command_file=\"$state_dir/$job_id.install.sh\"",
+    "commands_file=\"$state_dir/$job_id.commands.txt\"",
+    "printf '%s' \"$install_command_b64\" | base64 -d >\"$command_file\"",
+    "printf '%s' \"$check_commands_b64\" | base64 -d >\"$commands_file\"",
+    "chmod 700 \"$command_file\"",
+    "JOB_ID=\"$job_id\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" node <<'NODE' >\"$status_file\"",
+    "console.log(JSON.stringify({ ok: false, status: 'running', jobId: process.env.JOB_ID, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, startedAt: new Date().toISOString(), valuesPrinted: false }));",
+    "NODE",
+    "(",
+    "  set +e",
+    "  export HTTP_PROXY=\"$http_proxy_value\" HTTPS_PROXY=\"$https_proxy_value\" ALL_PROXY=\"$all_proxy_value\" NO_PROXY=\"$no_proxy_value\"",
+    "  export http_proxy=\"$http_proxy_value\" https_proxy=\"$https_proxy_value\" all_proxy=\"$all_proxy_value\" no_proxy=\"$no_proxy_value\"",
+    "  timeout \"$timeout_seconds\" /bin/bash \"$command_file\" >\"$stdout_file\" 2>\"$stderr_file\"",
+    "  install_exit=$?",
+    "  missing_commands=",
+    "  present_commands=",
+    "  while IFS= read -r command_name; do",
+    "    [ -z \"$command_name\" ] && continue",
+    "    if command -v \"$command_name\" >/dev/null 2>&1; then",
+    "      present_commands=\"$present_commands${present_commands:+,}$command_name\"",
+    "    else",
+    "      missing_commands=\"$missing_commands${missing_commands:+,}$command_name\"",
+    "    fi",
+    "  done <\"$commands_file\"",
+    "  stdout_tail=$(tail -c 4000 \"$stdout_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000)",
+    "  stderr_tail=$(tail -c 4000 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000)",
+    "  JOB_ID=\"$job_id\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" INSTALL_EXIT=\"$install_exit\" PRESENT_COMMANDS=\"$present_commands\" MISSING_COMMANDS=\"$missing_commands\" STDOUT_TAIL=\"$stdout_tail\" STDERR_TAIL=\"$stderr_tail\" node <<'NODE' >\"$status_file\"",
+    "const missing = (process.env.MISSING_COMMANDS || '').split(',').filter(Boolean);",
+    "const present = (process.env.PRESENT_COMMANDS || '').split(',').filter(Boolean);",
+    "const exitCode = Number(process.env.INSTALL_EXIT || '1');",
+    "const ok = exitCode === 0 && missing.length === 0;",
+    "console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, installExitCode: exitCode, presentCommands: present, missingCommands: missing, stdoutTail: process.env.STDOUT_TAIL || '', stderrTail: process.env.STDERR_TAIL || '', finishedAt: new Date().toISOString(), valuesPrinted: false }));",
+    "NODE",
+    ") >/dev/null 2>&1 &",
+    "pid=$!",
+    "JOB_ID=\"$job_id\" PID=\"$pid\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" REQUIRED_COMMANDS_SHA=\"$required_commands_sha\" node <<'NODE'",
+    "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.PID || '0'), stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, requiredCommandsSha256: process.env.REQUIRED_COMMANDS_SHA, valuesPrinted: false }));",
+    "NODE",
+  ].join("\n");
+}
+
 function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string {
  const requiredCommands = sourceWorkspace.requiredCommands.join("\n");
  const requiredFiles = sourceWorkspace.requiredFiles.join("\n");