Merge pull request #438 from pikasTech/fix/issues-435-436-437-infra-feedback

fix: reduce infra feedback CLI friction

scripts/cli.ts

@@ -20,6 +20,7 @@ import { runArtifactRegistryCommand } from "./src/artifact-registry";
 import { runAuthBrokerCommand } from "./src/auth-broker";
 import { runGhCommand } from "./src/gh";
 import { isGhContentRoute, runGhContentRoute } from "./src/gh-route";
+import { runGitToolsCommand } from "./src/git-tools";
 import { runCommanderCommand } from "./src/commander";
 import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from "./src/help";
 import { runServerCleanupCommand } from "./src/server-cleanup";
@@ -294,6 +295,14 @@ async function main(): Promise<void> {
     return;
   }
 
+  if (top === "git") {
+    const result = runGitToolsCommand(args.slice(1));
+    const ok = (result as { ok?: unknown }).ok !== false;
+    emitJson(commandName, result, ok);
+    if (!ok) process.exitCode = 1;
+    return;
+  }
+
   if (top === "commander") {
     const result = runCommanderCommand(args.slice(1));
     const ok = (result as { ok?: unknown }).ok !== false;

scripts/src/agentrun.ts

@@ -4811,7 +4811,12 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
     throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge, httpStatus: response.status });
   }
   if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
-  if (!response.ok) throw new AgentRunRestError(response.status === 404 ? "unsupported-version" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
+  if (!response.ok) {
+    const details = response.status === 404
+      ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue)
+      : safeAgentRunEnvelope(envelope);
+    throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details });
+  }
   if (envelope.ok !== true) {
     const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), response.status);
     throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
@@ -4827,13 +4832,33 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
 function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult {
   const payload = error.toPayload(command);
   if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false);
+  const laneHint = record(record(payload.agentrun).laneAwareLookup);
+  const currentEndpoint = record(laneHint.currentEndpoint);
+  const nextCommands = Array.isArray(laneHint.nextCommands)
+    ? laneHint.nextCommands.filter((value): value is string => typeof value === "string" && value.length > 0).slice(0, 4)
+    : [];
+  const hintLines = Object.keys(laneHint).length === 0
+    ? []
+    : [
+        "",
+        "Lane lookup hint:",
+        `  Current baseUrl: ${String(currentEndpoint.baseUrl ?? "(unknown)")}`,
+        `  Config: ${String(currentEndpoint.configPath ?? "(unknown)")}`,
+        `  ${String(laneHint.summary ?? "The resource was not found on the currently configured AgentRun manager endpoint.")}`,
+      ];
+  const nextLines = nextCommands.length > 0
+    ? nextCommands
+    : [
+        "Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.",
+        "Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.",
+      ];
   return renderedCliResult(false, command, [
     `Error: ${payload.failureKind}`,
     String(payload.message ?? ""),
+    ...hintLines,
     "",
     "Next:",
-    "  Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.",
+    ...nextLines.map((line) => `  ${line}`),
-    "  Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.",
   ].join("\n"));
 }
 
@@ -5059,6 +5084,81 @@ function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRun
   };
 }
 
+function addAgentRunNotFoundLookupHint(details: Record<string, unknown>, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string): Record<string, unknown> {
+  const resource = agentRunResourceFromPath(pathValue);
+  const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath);
+  const candidateLanes = laneConfig?.lanes ?? [];
+  const nextCommands = candidateLanes
+    .filter((lane) => typeof lane.node === "string" && lane.node.length > 0 && typeof lane.lane === "string" && lane.lane.length > 0)
+    .slice(0, 4)
+    .map((lane) => `bun scripts/cli.ts agentrun control-plane status --node ${lane.node} --lane ${lane.lane}`);
+  return {
+    ...details,
+    laneAwareLookup: {
+      kind: "agentrun-resource-not-found-on-current-endpoint",
+      summary: resource === null
+        ? "The request returned 404 on the currently configured AgentRun manager endpoint; a resource from another lane will not be visible through this baseUrl."
+        : `${resource.kind}/${resource.id} returned 404 on the currently configured AgentRun manager endpoint; if it was created on another lane, inspect that lane before treating the resource as lost.`,
+      request: {
+        method,
+        path: pathValue,
+        ...(resource === null ? {} : { resource }),
+      },
+      currentEndpoint: {
+        transport: "direct-http",
+        baseUrl: config.manager.baseUrl,
+        configPath: config.sourcePath,
+        defaultNode: laneConfig?.defaultNode ?? null,
+        defaultLane: laneConfig?.defaultLane ?? null,
+      },
+      candidateLanes,
+      nextCommands,
+      note: "Resource commands use manager.baseUrl from the client config; --node/--lane currently belongs to control-plane inspection commands.",
+      valuesPrinted: false,
+    },
+  };
+}
+
+function agentRunResourceFromPath(pathValue: string): Record<string, string> | null {
+  const run = pathValue.match(/\/runs\/([^/?#]+)/u);
+  if (run?.[1] !== undefined) return { kind: "run", id: decodeURIComponent(run[1]) };
+  const session = pathValue.match(/\/sessions\/([^/?#]+)/u);
+  if (session?.[1] !== undefined) return { kind: "session", id: decodeURIComponent(session[1]) };
+  const command = pathValue.match(/\/commands\/([^/?#]+)/u);
+  if (command?.[1] !== undefined) return { kind: "command", id: decodeURIComponent(command[1]) };
+  const task = pathValue.match(/\/queue\/tasks\/([^/?#]+)/u);
+  if (task?.[1] !== undefined) return { kind: "task", id: decodeURIComponent(task[1]) };
+  return null;
+}
+
+function readAgentRunLaneLookupConfig(configPath: string): { defaultNode: string | null; defaultLane: string | null; lanes: Record<string, unknown>[] } | null {
+  try {
+    const raw = readFileSync(configPath, "utf8");
+    const parsed = record(Bun.YAML.parse(raw) as unknown);
+    const controlPlane = record(parsed.controlPlane);
+    const defaultTarget = record(controlPlane.default);
+    const lanes = record(controlPlane.lanes);
+    return {
+      defaultNode: stringOrNull(defaultTarget.node),
+      defaultLane: stringOrNull(defaultTarget.lane),
+      lanes: Object.entries(lanes).map(([laneName, rawLane]) => {
+        const lane = record(rawLane);
+        const runtime = record(lane.runtime);
+        return {
+          lane: laneName,
+          node: stringOrNull(lane.node) ?? stringOrNull(lane.nodeId),
+          version: stringOrNull(lane.version),
+          namespace: stringOrNull(runtime.namespace),
+          internalBaseUrl: stringOrNull(runtime.internalBaseUrl),
+          isDefault: laneName === stringOrNull(defaultTarget.lane),
+        };
+      }),
+    };
+  } catch {
+    return null;
+  }
+}
+
 function agentRunDryRunPlan(action: string, pathValue: string, body: Record<string, unknown>, confirmCommand: string, method: AgentRunHttpMethod = "POST", extra: Record<string, unknown> = {}): Record<string, unknown> {
   return {
     ok: true,
@@ -5293,9 +5393,9 @@ function safeAgentRunEnvelope(envelope: Record<string, unknown>): Record<string,
 }
 
 function normalizeAgentRunFailureKind(raw: string | null, httpStatus: number): AgentRunFailureKind {
-  if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed") return raw;
+  if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
   if (httpStatus === 401 || httpStatus === 403) return "auth-failed";
-  if (httpStatus === 404) return "unsupported-version";
+  if (httpStatus === 404) return "not-found";
   return raw === "schema-invalid" ? "validation-failed" : "validation-failed";
 }
 
@@ -5383,7 +5483,7 @@ interface AgentRunResolvedAuth {
 }
 
 type AgentRunHttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
-type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed";
+type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
 
 type AgentRunRestCompatGroup = "queue" | "sessions" | "aipod-specs" | "aipods" | "runs" | "commands" | "runner";
 type AgentRunResourceVerb = "get" | "describe" | "events" | "logs" | "result" | "ack" | "cancel" | "dispatch" | "create" | "apply" | "send" | "explain";

scripts/src/git-tools.ts

@@ -0,0 +1,266 @@
+import { spawnSync } from "node:child_process";
+import { resolve } from "node:path";
+import { repoRoot } from "./config";
+
+interface GitHubPushFallbackOptions {
+  repo: string | null;
+  branch: string | null;
+  cwd: string;
+  hostName: string;
+  port: number;
+  confirm: boolean;
+}
+
+export function runGitToolsCommand(args: string[]): Record<string, unknown> {
+  const [action] = args;
+  if (action === undefined || action === "help" || action === "--help" || action === "-h") return gitToolsHelp();
+  if (action !== "github-push-fallback") throw new Error(`unsupported git command: ${action}`);
+  const options = parseGitHubPushFallbackOptions(args.slice(1));
+  const repo = options.repo ?? deriveGitHubRepo(options.cwd);
+  const branch = options.branch ?? currentGitBranch(options.cwd);
+  if (repo === null) throw new Error("git github-push-fallback requires --repo owner/name, or an origin remote that points at GitHub");
+  if (branch === null) throw new Error("git github-push-fallback requires --branch <branch>, or a checked-out branch");
+  validateGitHubRepo(repo);
+  validatePushRef(branch);
+
+  const remoteUrl = `ssh://git@ssh.github.com:${options.port}/${repo}.git`;
+  const sshCommand = [
+    "ssh",
+    "-o", `HostName=${options.hostName}`,
+    "-o", `Port=${options.port}`,
+    "-o", "HostKeyAlias=ssh.github.com",
+    "-o", "StrictHostKeyChecking=accept-new",
+  ].join(" ");
+  const argv = ["git", "push", remoteUrl, branch];
+  const base = {
+    ok: true,
+    command: "git github-push-fallback",
+    cwd: options.cwd,
+    mutation: options.confirm,
+    repository: repo,
+    branch,
+    remoteUrl,
+    ssh: {
+      hostName: options.hostName,
+      port: options.port,
+      hostKeyAlias: "ssh.github.com",
+      strictHostKeyChecking: "accept-new",
+    },
+    env: {
+      GIT_SSH_COMMAND: sshCommand,
+      valuesPrinted: false,
+    },
+    argv,
+    boundary: "does not edit git remotes; intended for GitHub DNS/port-22 reachability fallback only",
+  };
+
+  if (!options.confirm) {
+    return {
+      ...base,
+      executed: false,
+      dryRun: true,
+      next: [
+        "rerun with --confirm to execute this one push without changing origin",
+        "use --host-name <ip> only when both github.com and ssh.github.com DNS are broken on the target host",
+      ],
+    };
+  }
+
+  const result = spawnSync("git", ["push", remoteUrl, branch], {
+    cwd: options.cwd,
+    env: { ...process.env, GIT_SSH_COMMAND: sshCommand },
+    encoding: "utf8",
+    timeout: 120_000,
+  });
+  const stdout = result.stdout ?? "";
+  const stderr = result.stderr ?? "";
+  const ok = result.status === 0;
+  return {
+    ...base,
+    ok,
+    executed: true,
+    dryRun: false,
+    exitCode: result.status,
+    signal: result.signal,
+    stdoutTail: tail(stdout, 2000),
+    stderrTail: tail(stderr, 4000),
+    diagnostic: classifyGitPushFailure(`${stdout}\n${stderr}`),
+  };
+}
+
+function gitToolsHelp(): Record<string, unknown> {
+  return {
+    ok: true,
+    command: "git",
+    usage: [
+      "bun scripts/cli.ts git github-push-fallback --repo pikasTech/HWLAB --branch fix/name",
+      "bun scripts/cli.ts git github-push-fallback --repo pikasTech/HWLAB --branch fix/name --host-name 140.82.116.36 --confirm",
+    ],
+    behavior: [
+      "Plans or executes a one-shot GitHub push through ssh.github.com:443 without editing git remotes.",
+      "Default output is a dry-run plan. Add --confirm to execute the push.",
+      "Use --host-name <ip> only when target-host DNS cannot resolve GitHub SSH names; HostKeyAlias remains ssh.github.com.",
+    ],
+    options: {
+      "--repo <owner/name>": "GitHub repository. Defaults to parsing origin.",
+      "--branch <branch>": "Local branch/ref to push. Defaults to the checked-out branch.",
+      "--cwd <path>": "Git worktree. Defaults to the UniDesk repo root.",
+      "--host-name <host-or-ip>": "SSH HostName override. Defaults to ssh.github.com.",
+      "--port <n>": "SSH port. Defaults to 443.",
+      "--confirm": "Execute the push. Without this flag, only prints a plan.",
+    },
+  };
+}
+
+function parseGitHubPushFallbackOptions(args: string[]): GitHubPushFallbackOptions {
+  const options: GitHubPushFallbackOptions = {
+    repo: null,
+    branch: null,
+    cwd: repoRoot,
+    hostName: "ssh.github.com",
+    port: 443,
+    confirm: false,
+  };
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index] ?? "";
+    const next = args[index + 1];
+    if (arg === "--repo") {
+      options.repo = requireValue(arg, next);
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--repo=")) {
+      options.repo = requireValue("--repo", arg.slice("--repo=".length));
+      continue;
+    }
+    if (arg === "--branch") {
+      options.branch = requireValue(arg, next);
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--branch=")) {
+      options.branch = requireValue("--branch", arg.slice("--branch=".length));
+      continue;
+    }
+    if (arg === "--cwd") {
+      options.cwd = resolve(requireValue(arg, next));
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--cwd=")) {
+      options.cwd = resolve(requireValue("--cwd", arg.slice("--cwd=".length)));
+      continue;
+    }
+    if (arg === "--host-name" || arg === "--host") {
+      options.hostName = requireValue(arg, next);
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--host-name=")) {
+      options.hostName = requireValue("--host-name", arg.slice("--host-name=".length));
+      continue;
+    }
+    if (arg === "--host-ip") {
+      options.hostName = requireValue(arg, next);
+      index += 1;
+      continue;
+    }
+    if (arg === "--port") {
+      options.port = parsePort(requireValue(arg, next));
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--port=")) {
+      options.port = parsePort(requireValue("--port", arg.slice("--port=".length)));
+      continue;
+    }
+    if (arg === "--confirm") {
+      options.confirm = true;
+      continue;
+    }
+    if (arg === "--dry-run") continue;
+    throw new Error(`unsupported git github-push-fallback option: ${arg}`);
+  }
+  validateHostName(options.hostName);
+  return options;
+}
+
+function deriveGitHubRepo(cwd: string): string | null {
+  const result = gitCapture(cwd, ["remote", "get-url", "origin"]);
+  if (!result.ok) return null;
+  const remote = result.stdout.trim();
+  const patterns = [
+    /^git@github\.com:([^/\s]+\/[^/\s]+?)(?:\.git)?$/u,
+    /^https:\/\/github\.com\/([^/\s]+\/[^/\s]+?)(?:\.git)?$/u,
+    /^ssh:\/\/git@(?:github\.com|ssh\.github\.com)(?::\d+)?\/([^/\s]+\/[^/\s]+?)(?:\.git)?$/u,
+  ];
+  for (const pattern of patterns) {
+    const match = remote.match(pattern);
+    if (match?.[1] !== undefined) return match[1];
+  }
+  return null;
+}
+
+function currentGitBranch(cwd: string): string | null {
+  const result = gitCapture(cwd, ["branch", "--show-current"]);
+  const branch = result.ok ? result.stdout.trim() : "";
+  return branch.length > 0 ? branch : null;
+}
+
+function gitCapture(cwd: string, args: string[]): { ok: boolean; stdout: string; stderr: string } {
+  const result = spawnSync("git", args, { cwd, encoding: "utf8", timeout: 10_000 });
+  return {
+    ok: result.status === 0,
+    stdout: result.stdout ?? "",
+    stderr: result.stderr ?? "",
+  };
+}
+
+function classifyGitPushFailure(output: string): Record<string, unknown> | null {
+  if (/Could not resolve hostname github\.com|Temporary failure in name resolution|Name or service not known/iu.test(output)) {
+    return {
+      kind: "github-dns-or-name-resolution",
+      next: "retry with git github-push-fallback; if ssh.github.com DNS also fails, pass --host-name <known GitHub SSH IP>",
+    };
+  }
+  if (/connect to host github\.com port 22|Connection timed out|Network is unreachable/iu.test(output)) {
+    return {
+      kind: "github-port-22-or-network-blocked",
+      next: "retry through ssh.github.com:443 with git github-push-fallback",
+    };
+  }
+  if (/Permission denied \(publickey\)|Authentication failed|Repository not found/iu.test(output)) {
+    return {
+      kind: "auth-or-repository-access",
+      next: "do not keep retrying network fallbacks; inspect SSH key, deploy key, repository name, and GitHub access",
+    };
+  }
+  return null;
+}
+
+function requireValue(name: string, value: string | undefined): string {
+  if (value === undefined || value.length === 0) throw new Error(`${name} requires a non-empty value`);
+  return value;
+}
+
+function parsePort(value: string): number {
+  const port = Number(value);
+  if (!Number.isInteger(port) || port <= 0 || port > 65535) throw new Error("--port must be an integer from 1 to 65535");
+  return port;
+}
+
+function validateGitHubRepo(value: string): void {
+  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)) throw new Error("--repo must be owner/name");
+}
+
+function validatePushRef(value: string): void {
+  if (!/^[A-Za-z0-9_./:-]+$/u.test(value) || value.includes("..") || value.startsWith("-")) throw new Error("--branch must be a simple git ref without whitespace or shell characters");
+}
+
+function validateHostName(value: string): void {
+  if (!/^[A-Za-z0-9_.:-]+$/u.test(value) || value.startsWith("-")) throw new Error("--host-name must be a hostname or IP literal");
+}
+
+function tail(value: string, maxChars: number): string {
+  return value.length > maxChars ? value.slice(-maxChars) : value;
+}

scripts/src/help.ts

@@ -57,6 +57,7 @@ export function rootHelp(): unknown {
       { command: "artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service", description: "Manage the D601 host-managed CNCF Distribution registry and run pull-only artifact CD for supported services, including D601 direct, k3s-managed, and code-queue dev-only consumers." },
       { command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
       { command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, issue/comment apply_patch body patching, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
+      { command: "git github-push-fallback [--repo owner/name] [--branch branch] [--host-name host-or-ip] [--confirm]", description: "Plan or execute a one-shot GitHub push through ssh.github.com:443 without editing remotes; use only for reviewed DNS/port-22 push fallback." },
       { command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run", description: "Host Codex commander skeleton contract, no-daemon smoke plan, and dry-run approval preview without live bridges or message sends." },
       { command: "hwlab nodes control-plane|git-mirror|secret|test-accounts|web-probe --node <node> --lane <lane>", description: "Manage HWLAB node/lane runtime prerequisites, including D601 YAML-declared infra/tools-image/Argo bootstrap, redacted test-account preparation, Web DOM probe credential injection, and G14 v0.3+ runtime lanes, with the node identity passed as data." },
       { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." },

scripts/src/ssh-playwright.ts

@@ -66,8 +66,20 @@ export async function runSshPlaywrightOperation(
   const localDir = resolve(options.localDir);
   const submitCommand = builders.buildRouteCommand(invocation.route, ["sh", "-c", remotePlaywrightSubmitScript(remoteDir), "unidesk-playwright-submit"], { stdin: true });
   const submit = await executor.runRemoteCommand(submitCommand, userScript);
-  if (submit.exitCode !== 0) {
+  const submitRecovery = submit.exitCode !== 0 && isRecoverableSubmitTimeout(submit)
-    throw new Error(`ssh playwright submit failed: exitCode=${submit.exitCode}; stdoutTail=${JSON.stringify(submit.stdout.slice(-1000))}; stderrTail=${JSON.stringify(submit.stderr.slice(-1000))}`);
+    ? {
+        recovered: true,
+        reason: "submit-short-connection-timeout",
+        remoteDir,
+        runId,
+        exitCode: submit.exitCode,
+        stdoutTail: submit.stdout.slice(-1000),
+        stderrTail: submit.stderr.slice(-1000),
+        next: "submit may have been cut by the 60s trans runtime limit after the background job was launched; polling remote status by remoteDir/runId",
+      }
+    : null;
+  if (submit.exitCode !== 0 && submitRecovery === null) {
+    throw new Error(`ssh playwright submit failed: exitCode=${submit.exitCode}; remoteDir=${remoteDir}; runId=${runId}; stdoutTail=${JSON.stringify(submit.stdout.slice(-1000))}; stderrTail=${JSON.stringify(submit.stderr.slice(-1000))}`);
   }
   const manifest = await pollRemoteManifest(invocation.route, executor, builders, remoteDir, runId, options);
   const artifacts: Array<SshVerifiedDownloadResult & { manifestBytes: number | null; manifestSha256: string | null }> = [];
@@ -124,6 +136,8 @@ export async function runSshPlaywrightOperation(
     remoteCommand: {
       exitCode: manifest.exitCode,
       submitExitCode: submit.exitCode,
+      submitRecovered: submitRecovery !== null,
+      submitRecovery,
       submitStdoutBytes: Buffer.byteLength(submit.stdout, "utf8"),
       submitStderrBytes: Buffer.byteLength(submit.stderr, "utf8"),
     },
@@ -272,17 +286,23 @@ async function pollRemoteManifest(
     lastStatus = status;
     if (status.exitCode === 0 && status.stdout.includes(manifestEnd)) return parseRemoteManifest(status, remoteDir, runId);
     if (status.exitCode !== 0 && !/status\t(?:pending|running)/u.test(status.stdout)) {
-      throw new Error(`ssh playwright status failed: exitCode=${status.exitCode}; stdoutTail=${JSON.stringify(status.stdout.slice(-1000))}; stderrTail=${JSON.stringify(status.stderr.slice(-1000))}`);
+      throw new Error(`ssh playwright status failed: exitCode=${status.exitCode}; remoteDir=${remoteDir}; runId=${runId}; stdoutTail=${JSON.stringify(status.stdout.slice(-1000))}; stderrTail=${JSON.stringify(status.stderr.slice(-1000))}`);
     }
     await sleep(options.pollIntervalMs);
   }
-  throw new Error(`ssh playwright timed out waiting for remote job after ${options.waitTimeoutMs}ms; lastStdoutTail=${JSON.stringify(lastStatus?.stdout.slice(-1000) ?? "")}; lastStderrTail=${JSON.stringify(lastStatus?.stderr.slice(-1000) ?? "")}`);
+  throw new Error(`ssh playwright timed out waiting for remote job after ${options.waitTimeoutMs}ms; remoteDir=${remoteDir}; runId=${runId}; lastStdoutTail=${JSON.stringify(lastStatus?.stdout.slice(-1000) ?? "")}; lastStderrTail=${JSON.stringify(lastStatus?.stderr.slice(-1000) ?? "")}`);
 }
 
 function sleep(ms: number): Promise<void> {
   return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
 }
 
+function isRecoverableSubmitTimeout(submit: SshCaptureResult): boolean {
+  if (submit.exitCode === 124) return true;
+  const combined = `${submit.stdout}\n${submit.stderr}`;
+  return /(?:timed?\s*out|timeout|60s|exitCode=124|signal\s+TERM|signal\s+KILL)/iu.test(combined);
+}
+
 function remotePlaywrightSubmitScript(remoteDir: string): string {
   const runner = Buffer.from(remotePlaywrightRunnerScript(remoteDir), "utf8").toString("base64");
   return [