Merge pull request #438 from pikasTech/fix/issues-435-436-437-infra-feedback

fix: reduce infra feedback CLI friction
This commit is contained in:
Lyon
2026-06-16 06:49:30 +08:00
committed by GitHub
5 changed files with 406 additions and 10 deletions
+9
View File
@@ -20,6 +20,7 @@ import { runArtifactRegistryCommand } from "./src/artifact-registry";
import { runAuthBrokerCommand } from "./src/auth-broker";
import { runGhCommand } from "./src/gh";
import { isGhContentRoute, runGhContentRoute } from "./src/gh-route";
import { runGitToolsCommand } from "./src/git-tools";
import { runCommanderCommand } from "./src/commander";
import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from "./src/help";
import { runServerCleanupCommand } from "./src/server-cleanup";
@@ -294,6 +295,14 @@ async function main(): Promise<void> {
return;
}
if (top === "git") {
const result = runGitToolsCommand(args.slice(1));
const ok = (result as { ok?: unknown }).ok !== false;
emitJson(commandName, result, ok);
if (!ok) process.exitCode = 1;
return;
}
if (top === "commander") {
const result = runCommanderCommand(args.slice(1));
const ok = (result as { ok?: unknown }).ok !== false;
+106 -6
View File
@@ -4811,7 +4811,12 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge, httpStatus: response.status });
}
if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
if (!response.ok) throw new AgentRunRestError(response.status === 404 ? "unsupported-version" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
if (!response.ok) {
const details = response.status === 404
? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue)
: safeAgentRunEnvelope(envelope);
throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details });
}
if (envelope.ok !== true) {
const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), response.status);
throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
@@ -4827,13 +4832,33 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult {
const payload = error.toPayload(command);
if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false);
const laneHint = record(record(payload.agentrun).laneAwareLookup);
const currentEndpoint = record(laneHint.currentEndpoint);
const nextCommands = Array.isArray(laneHint.nextCommands)
? laneHint.nextCommands.filter((value): value is string => typeof value === "string" && value.length > 0).slice(0, 4)
: [];
const hintLines = Object.keys(laneHint).length === 0
? []
: [
"",
"Lane lookup hint:",
` Current baseUrl: ${String(currentEndpoint.baseUrl ?? "(unknown)")}`,
` Config: ${String(currentEndpoint.configPath ?? "(unknown)")}`,
` ${String(laneHint.summary ?? "The resource was not found on the currently configured AgentRun manager endpoint.")}`,
];
const nextLines = nextCommands.length > 0
? nextCommands
: [
"Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.",
"Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.",
];
return renderedCliResult(false, command, [
`Error: ${payload.failureKind}`,
String(payload.message ?? ""),
...hintLines,
"",
"Next:",
" Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.",
" Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.",
...nextLines.map((line) => ` ${line}`),
].join("\n"));
}
@@ -5059,6 +5084,81 @@ function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRun
};
}
function addAgentRunNotFoundLookupHint(details: Record<string, unknown>, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string): Record<string, unknown> {
const resource = agentRunResourceFromPath(pathValue);
const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath);
const candidateLanes = laneConfig?.lanes ?? [];
const nextCommands = candidateLanes
.filter((lane) => typeof lane.node === "string" && lane.node.length > 0 && typeof lane.lane === "string" && lane.lane.length > 0)
.slice(0, 4)
.map((lane) => `bun scripts/cli.ts agentrun control-plane status --node ${lane.node} --lane ${lane.lane}`);
return {
...details,
laneAwareLookup: {
kind: "agentrun-resource-not-found-on-current-endpoint",
summary: resource === null
? "The request returned 404 on the currently configured AgentRun manager endpoint; a resource from another lane will not be visible through this baseUrl."
: `${resource.kind}/${resource.id} returned 404 on the currently configured AgentRun manager endpoint; if it was created on another lane, inspect that lane before treating the resource as lost.`,
request: {
method,
path: pathValue,
...(resource === null ? {} : { resource }),
},
currentEndpoint: {
transport: "direct-http",
baseUrl: config.manager.baseUrl,
configPath: config.sourcePath,
defaultNode: laneConfig?.defaultNode ?? null,
defaultLane: laneConfig?.defaultLane ?? null,
},
candidateLanes,
nextCommands,
note: "Resource commands use manager.baseUrl from the client config; --node/--lane currently belongs to control-plane inspection commands.",
valuesPrinted: false,
},
};
}
function agentRunResourceFromPath(pathValue: string): Record<string, string> | null {
const run = pathValue.match(/\/runs\/([^/?#]+)/u);
if (run?.[1] !== undefined) return { kind: "run", id: decodeURIComponent(run[1]) };
const session = pathValue.match(/\/sessions\/([^/?#]+)/u);
if (session?.[1] !== undefined) return { kind: "session", id: decodeURIComponent(session[1]) };
const command = pathValue.match(/\/commands\/([^/?#]+)/u);
if (command?.[1] !== undefined) return { kind: "command", id: decodeURIComponent(command[1]) };
const task = pathValue.match(/\/queue\/tasks\/([^/?#]+)/u);
if (task?.[1] !== undefined) return { kind: "task", id: decodeURIComponent(task[1]) };
return null;
}
function readAgentRunLaneLookupConfig(configPath: string): { defaultNode: string | null; defaultLane: string | null; lanes: Record<string, unknown>[] } | null {
try {
const raw = readFileSync(configPath, "utf8");
const parsed = record(Bun.YAML.parse(raw) as unknown);
const controlPlane = record(parsed.controlPlane);
const defaultTarget = record(controlPlane.default);
const lanes = record(controlPlane.lanes);
return {
defaultNode: stringOrNull(defaultTarget.node),
defaultLane: stringOrNull(defaultTarget.lane),
lanes: Object.entries(lanes).map(([laneName, rawLane]) => {
const lane = record(rawLane);
const runtime = record(lane.runtime);
return {
lane: laneName,
node: stringOrNull(lane.node) ?? stringOrNull(lane.nodeId),
version: stringOrNull(lane.version),
namespace: stringOrNull(runtime.namespace),
internalBaseUrl: stringOrNull(runtime.internalBaseUrl),
isDefault: laneName === stringOrNull(defaultTarget.lane),
};
}),
};
} catch {
return null;
}
}
function agentRunDryRunPlan(action: string, pathValue: string, body: Record<string, unknown>, confirmCommand: string, method: AgentRunHttpMethod = "POST", extra: Record<string, unknown> = {}): Record<string, unknown> {
return {
ok: true,
@@ -5293,9 +5393,9 @@ function safeAgentRunEnvelope(envelope: Record<string, unknown>): Record<string,
}
function normalizeAgentRunFailureKind(raw: string | null, httpStatus: number): AgentRunFailureKind {
if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed") return raw;
if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
if (httpStatus === 401 || httpStatus === 403) return "auth-failed";
if (httpStatus === 404) return "unsupported-version";
if (httpStatus === 404) return "not-found";
return raw === "schema-invalid" ? "validation-failed" : "validation-failed";
}
@@ -5383,7 +5483,7 @@ interface AgentRunResolvedAuth {
}
type AgentRunHttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed";
type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
type AgentRunRestCompatGroup = "queue" | "sessions" | "aipod-specs" | "aipods" | "runs" | "commands" | "runner";
type AgentRunResourceVerb = "get" | "describe" | "events" | "logs" | "result" | "ack" | "cancel" | "dispatch" | "create" | "apply" | "send" | "explain";
+266
View File
@@ -0,0 +1,266 @@
import { spawnSync } from "node:child_process";
import { resolve } from "node:path";
import { repoRoot } from "./config";
interface GitHubPushFallbackOptions {
repo: string | null;
branch: string | null;
cwd: string;
hostName: string;
port: number;
confirm: boolean;
}
export function runGitToolsCommand(args: string[]): Record<string, unknown> {
const [action] = args;
if (action === undefined || action === "help" || action === "--help" || action === "-h") return gitToolsHelp();
if (action !== "github-push-fallback") throw new Error(`unsupported git command: ${action}`);
const options = parseGitHubPushFallbackOptions(args.slice(1));
const repo = options.repo ?? deriveGitHubRepo(options.cwd);
const branch = options.branch ?? currentGitBranch(options.cwd);
if (repo === null) throw new Error("git github-push-fallback requires --repo owner/name, or an origin remote that points at GitHub");
if (branch === null) throw new Error("git github-push-fallback requires --branch <branch>, or a checked-out branch");
validateGitHubRepo(repo);
validatePushRef(branch);
const remoteUrl = `ssh://git@ssh.github.com:${options.port}/${repo}.git`;
const sshCommand = [
"ssh",
"-o", `HostName=${options.hostName}`,
"-o", `Port=${options.port}`,
"-o", "HostKeyAlias=ssh.github.com",
"-o", "StrictHostKeyChecking=accept-new",
].join(" ");
const argv = ["git", "push", remoteUrl, branch];
const base = {
ok: true,
command: "git github-push-fallback",
cwd: options.cwd,
mutation: options.confirm,
repository: repo,
branch,
remoteUrl,
ssh: {
hostName: options.hostName,
port: options.port,
hostKeyAlias: "ssh.github.com",
strictHostKeyChecking: "accept-new",
},
env: {
GIT_SSH_COMMAND: sshCommand,
valuesPrinted: false,
},
argv,
boundary: "does not edit git remotes; intended for GitHub DNS/port-22 reachability fallback only",
};
if (!options.confirm) {
return {
...base,
executed: false,
dryRun: true,
next: [
"rerun with --confirm to execute this one push without changing origin",
"use --host-name <ip> only when both github.com and ssh.github.com DNS are broken on the target host",
],
};
}
const result = spawnSync("git", ["push", remoteUrl, branch], {
cwd: options.cwd,
env: { ...process.env, GIT_SSH_COMMAND: sshCommand },
encoding: "utf8",
timeout: 120_000,
});
const stdout = result.stdout ?? "";
const stderr = result.stderr ?? "";
const ok = result.status === 0;
return {
...base,
ok,
executed: true,
dryRun: false,
exitCode: result.status,
signal: result.signal,
stdoutTail: tail(stdout, 2000),
stderrTail: tail(stderr, 4000),
diagnostic: classifyGitPushFailure(`${stdout}\n${stderr}`),
};
}
function gitToolsHelp(): Record<string, unknown> {
return {
ok: true,
command: "git",
usage: [
"bun scripts/cli.ts git github-push-fallback --repo pikasTech/HWLAB --branch fix/name",
"bun scripts/cli.ts git github-push-fallback --repo pikasTech/HWLAB --branch fix/name --host-name 140.82.116.36 --confirm",
],
behavior: [
"Plans or executes a one-shot GitHub push through ssh.github.com:443 without editing git remotes.",
"Default output is a dry-run plan. Add --confirm to execute the push.",
"Use --host-name <ip> only when target-host DNS cannot resolve GitHub SSH names; HostKeyAlias remains ssh.github.com.",
],
options: {
"--repo <owner/name>": "GitHub repository. Defaults to parsing origin.",
"--branch <branch>": "Local branch/ref to push. Defaults to the checked-out branch.",
"--cwd <path>": "Git worktree. Defaults to the UniDesk repo root.",
"--host-name <host-or-ip>": "SSH HostName override. Defaults to ssh.github.com.",
"--port <n>": "SSH port. Defaults to 443.",
"--confirm": "Execute the push. Without this flag, only prints a plan.",
},
};
}
function parseGitHubPushFallbackOptions(args: string[]): GitHubPushFallbackOptions {
const options: GitHubPushFallbackOptions = {
repo: null,
branch: null,
cwd: repoRoot,
hostName: "ssh.github.com",
port: 443,
confirm: false,
};
for (let index = 0; index < args.length; index += 1) {
const arg = args[index] ?? "";
const next = args[index + 1];
if (arg === "--repo") {
options.repo = requireValue(arg, next);
index += 1;
continue;
}
if (arg.startsWith("--repo=")) {
options.repo = requireValue("--repo", arg.slice("--repo=".length));
continue;
}
if (arg === "--branch") {
options.branch = requireValue(arg, next);
index += 1;
continue;
}
if (arg.startsWith("--branch=")) {
options.branch = requireValue("--branch", arg.slice("--branch=".length));
continue;
}
if (arg === "--cwd") {
options.cwd = resolve(requireValue(arg, next));
index += 1;
continue;
}
if (arg.startsWith("--cwd=")) {
options.cwd = resolve(requireValue("--cwd", arg.slice("--cwd=".length)));
continue;
}
if (arg === "--host-name" || arg === "--host") {
options.hostName = requireValue(arg, next);
index += 1;
continue;
}
if (arg.startsWith("--host-name=")) {
options.hostName = requireValue("--host-name", arg.slice("--host-name=".length));
continue;
}
if (arg === "--host-ip") {
options.hostName = requireValue(arg, next);
index += 1;
continue;
}
if (arg === "--port") {
options.port = parsePort(requireValue(arg, next));
index += 1;
continue;
}
if (arg.startsWith("--port=")) {
options.port = parsePort(requireValue("--port", arg.slice("--port=".length)));
continue;
}
if (arg === "--confirm") {
options.confirm = true;
continue;
}
if (arg === "--dry-run") continue;
throw new Error(`unsupported git github-push-fallback option: ${arg}`);
}
validateHostName(options.hostName);
return options;
}
function deriveGitHubRepo(cwd: string): string | null {
const result = gitCapture(cwd, ["remote", "get-url", "origin"]);
if (!result.ok) return null;
const remote = result.stdout.trim();
const patterns = [
/^git@github\.com:([^/\s]+\/[^/\s]+?)(?:\.git)?$/u,
/^https:\/\/github\.com\/([^/\s]+\/[^/\s]+?)(?:\.git)?$/u,
/^ssh:\/\/git@(?:github\.com|ssh\.github\.com)(?::\d+)?\/([^/\s]+\/[^/\s]+?)(?:\.git)?$/u,
];
for (const pattern of patterns) {
const match = remote.match(pattern);
if (match?.[1] !== undefined) return match[1];
}
return null;
}
function currentGitBranch(cwd: string): string | null {
const result = gitCapture(cwd, ["branch", "--show-current"]);
const branch = result.ok ? result.stdout.trim() : "";
return branch.length > 0 ? branch : null;
}
function gitCapture(cwd: string, args: string[]): { ok: boolean; stdout: string; stderr: string } {
const result = spawnSync("git", args, { cwd, encoding: "utf8", timeout: 10_000 });
return {
ok: result.status === 0,
stdout: result.stdout ?? "",
stderr: result.stderr ?? "",
};
}
function classifyGitPushFailure(output: string): Record<string, unknown> | null {
if (/Could not resolve hostname github\.com|Temporary failure in name resolution|Name or service not known/iu.test(output)) {
return {
kind: "github-dns-or-name-resolution",
next: "retry with git github-push-fallback; if ssh.github.com DNS also fails, pass --host-name <known GitHub SSH IP>",
};
}
if (/connect to host github\.com port 22|Connection timed out|Network is unreachable/iu.test(output)) {
return {
kind: "github-port-22-or-network-blocked",
next: "retry through ssh.github.com:443 with git github-push-fallback",
};
}
if (/Permission denied \(publickey\)|Authentication failed|Repository not found/iu.test(output)) {
return {
kind: "auth-or-repository-access",
next: "do not keep retrying network fallbacks; inspect SSH key, deploy key, repository name, and GitHub access",
};
}
return null;
}
function requireValue(name: string, value: string | undefined): string {
if (value === undefined || value.length === 0) throw new Error(`${name} requires a non-empty value`);
return value;
}
function parsePort(value: string): number {
const port = Number(value);
if (!Number.isInteger(port) || port <= 0 || port > 65535) throw new Error("--port must be an integer from 1 to 65535");
return port;
}
function validateGitHubRepo(value: string): void {
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)) throw new Error("--repo must be owner/name");
}
function validatePushRef(value: string): void {
if (!/^[A-Za-z0-9_./:-]+$/u.test(value) || value.includes("..") || value.startsWith("-")) throw new Error("--branch must be a simple git ref without whitespace or shell characters");
}
function validateHostName(value: string): void {
if (!/^[A-Za-z0-9_.:-]+$/u.test(value) || value.startsWith("-")) throw new Error("--host-name must be a hostname or IP literal");
}
function tail(value: string, maxChars: number): string {
return value.length > maxChars ? value.slice(-maxChars) : value;
}
+1
View File
@@ -57,6 +57,7 @@ export function rootHelp(): unknown {
{ command: "artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service", description: "Manage the D601 host-managed CNCF Distribution registry and run pull-only artifact CD for supported services, including D601 direct, k3s-managed, and code-queue dev-only consumers." },
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, issue/comment apply_patch body patching, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
{ command: "git github-push-fallback [--repo owner/name] [--branch branch] [--host-name host-or-ip] [--confirm]", description: "Plan or execute a one-shot GitHub push through ssh.github.com:443 without editing remotes; use only for reviewed DNS/port-22 push fallback." },
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run", description: "Host Codex commander skeleton contract, no-daemon smoke plan, and dry-run approval preview without live bridges or message sends." },
{ command: "hwlab nodes control-plane|git-mirror|secret|test-accounts|web-probe --node <node> --lane <lane>", description: "Manage HWLAB node/lane runtime prerequisites, including D601 YAML-declared infra/tools-image/Argo bootstrap, redacted test-account preparation, Web DOM probe credential injection, and G14 v0.3+ runtime lanes, with the node identity passed as data." },
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." },
+24 -4
View File
@@ -66,8 +66,20 @@ export async function runSshPlaywrightOperation(
const localDir = resolve(options.localDir);
const submitCommand = builders.buildRouteCommand(invocation.route, ["sh", "-c", remotePlaywrightSubmitScript(remoteDir), "unidesk-playwright-submit"], { stdin: true });
const submit = await executor.runRemoteCommand(submitCommand, userScript);
if (submit.exitCode !== 0) {
throw new Error(`ssh playwright submit failed: exitCode=${submit.exitCode}; stdoutTail=${JSON.stringify(submit.stdout.slice(-1000))}; stderrTail=${JSON.stringify(submit.stderr.slice(-1000))}`);
const submitRecovery = submit.exitCode !== 0 && isRecoverableSubmitTimeout(submit)
? {
recovered: true,
reason: "submit-short-connection-timeout",
remoteDir,
runId,
exitCode: submit.exitCode,
stdoutTail: submit.stdout.slice(-1000),
stderrTail: submit.stderr.slice(-1000),
next: "submit may have been cut by the 60s trans runtime limit after the background job was launched; polling remote status by remoteDir/runId",
}
: null;
if (submit.exitCode !== 0 && submitRecovery === null) {
throw new Error(`ssh playwright submit failed: exitCode=${submit.exitCode}; remoteDir=${remoteDir}; runId=${runId}; stdoutTail=${JSON.stringify(submit.stdout.slice(-1000))}; stderrTail=${JSON.stringify(submit.stderr.slice(-1000))}`);
}
const manifest = await pollRemoteManifest(invocation.route, executor, builders, remoteDir, runId, options);
const artifacts: Array<SshVerifiedDownloadResult & { manifestBytes: number | null; manifestSha256: string | null }> = [];
@@ -124,6 +136,8 @@ export async function runSshPlaywrightOperation(
remoteCommand: {
exitCode: manifest.exitCode,
submitExitCode: submit.exitCode,
submitRecovered: submitRecovery !== null,
submitRecovery,
submitStdoutBytes: Buffer.byteLength(submit.stdout, "utf8"),
submitStderrBytes: Buffer.byteLength(submit.stderr, "utf8"),
},
@@ -272,17 +286,23 @@ async function pollRemoteManifest(
lastStatus = status;
if (status.exitCode === 0 && status.stdout.includes(manifestEnd)) return parseRemoteManifest(status, remoteDir, runId);
if (status.exitCode !== 0 && !/status\t(?:pending|running)/u.test(status.stdout)) {
throw new Error(`ssh playwright status failed: exitCode=${status.exitCode}; stdoutTail=${JSON.stringify(status.stdout.slice(-1000))}; stderrTail=${JSON.stringify(status.stderr.slice(-1000))}`);
throw new Error(`ssh playwright status failed: exitCode=${status.exitCode}; remoteDir=${remoteDir}; runId=${runId}; stdoutTail=${JSON.stringify(status.stdout.slice(-1000))}; stderrTail=${JSON.stringify(status.stderr.slice(-1000))}`);
}
await sleep(options.pollIntervalMs);
}
throw new Error(`ssh playwright timed out waiting for remote job after ${options.waitTimeoutMs}ms; lastStdoutTail=${JSON.stringify(lastStatus?.stdout.slice(-1000) ?? "")}; lastStderrTail=${JSON.stringify(lastStatus?.stderr.slice(-1000) ?? "")}`);
throw new Error(`ssh playwright timed out waiting for remote job after ${options.waitTimeoutMs}ms; remoteDir=${remoteDir}; runId=${runId}; lastStdoutTail=${JSON.stringify(lastStatus?.stdout.slice(-1000) ?? "")}; lastStderrTail=${JSON.stringify(lastStatus?.stderr.slice(-1000) ?? "")}`);
}
function sleep(ms: number): Promise<void> {
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
}
function isRecoverableSubmitTimeout(submit: SshCaptureResult): boolean {
if (submit.exitCode === 124) return true;
const combined = `${submit.stdout}\n${submit.stderr}`;
return /(?:timed?\s*out|timeout|60s|exitCode=124|signal\s+TERM|signal\s+KILL)/iu.test(combined);
}
function remotePlaywrightSubmitScript(remoteDir: string): string {
const runner = Buffer.from(remotePlaywrightRunnerScript(remoteDir), "utf8").toString("base64");
return [