diff --git a/scripts/cli.ts b/scripts/cli.ts index c5899456..8c055510 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -20,6 +20,7 @@ import { runArtifactRegistryCommand } from "./src/artifact-registry"; import { runAuthBrokerCommand } from "./src/auth-broker"; import { runGhCommand } from "./src/gh"; import { isGhContentRoute, runGhContentRoute } from "./src/gh-route"; +import { runGitToolsCommand } from "./src/git-tools"; import { runCommanderCommand } from "./src/commander"; import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from "./src/help"; import { runServerCleanupCommand } from "./src/server-cleanup"; @@ -294,6 +295,14 @@ async function main(): Promise { return; } + if (top === "git") { + const result = runGitToolsCommand(args.slice(1)); + const ok = (result as { ok?: unknown }).ok !== false; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } + if (top === "commander") { const result = runCommanderCommand(args.slice(1)); const ok = (result as { ok?: unknown }).ok !== false; diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 30c8e98a..bfbd1136 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -4811,7 +4811,12 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge, httpStatus: response.status }); } if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) }); - if (!response.ok) throw new AgentRunRestError(response.status === 404 ? "unsupported-version" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) }); + if (!response.ok) { + const details = response.status === 404 + ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue) + : safeAgentRunEnvelope(envelope); + throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details }); + } if (envelope.ok !== true) { const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), response.status); throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) }); @@ -4827,13 +4832,33 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult { const payload = error.toPayload(command); if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false); + const laneHint = record(record(payload.agentrun).laneAwareLookup); + const currentEndpoint = record(laneHint.currentEndpoint); + const nextCommands = Array.isArray(laneHint.nextCommands) + ? laneHint.nextCommands.filter((value): value is string => typeof value === "string" && value.length > 0).slice(0, 4) + : []; + const hintLines = Object.keys(laneHint).length === 0 + ? [] + : [ + "", + "Lane lookup hint:", + ` Current baseUrl: ${String(currentEndpoint.baseUrl ?? "(unknown)")}`, + ` Config: ${String(currentEndpoint.configPath ?? "(unknown)")}`, + ` ${String(laneHint.summary ?? "The resource was not found on the currently configured AgentRun manager endpoint.")}`, + ]; + const nextLines = nextCommands.length > 0 + ? nextCommands + : [ + "Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.", + "Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.", + ]; return renderedCliResult(false, command, [ `Error: ${payload.failureKind}`, String(payload.message ?? ""), + ...hintLines, "", "Next:", - " Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.", - " Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.", + ...nextLines.map((line) => ` ${line}`), ].join("\n")); } @@ -5059,6 +5084,81 @@ function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRun }; } +function addAgentRunNotFoundLookupHint(details: Record, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string): Record { + const resource = agentRunResourceFromPath(pathValue); + const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath); + const candidateLanes = laneConfig?.lanes ?? []; + const nextCommands = candidateLanes + .filter((lane) => typeof lane.node === "string" && lane.node.length > 0 && typeof lane.lane === "string" && lane.lane.length > 0) + .slice(0, 4) + .map((lane) => `bun scripts/cli.ts agentrun control-plane status --node ${lane.node} --lane ${lane.lane}`); + return { + ...details, + laneAwareLookup: { + kind: "agentrun-resource-not-found-on-current-endpoint", + summary: resource === null + ? "The request returned 404 on the currently configured AgentRun manager endpoint; a resource from another lane will not be visible through this baseUrl." + : `${resource.kind}/${resource.id} returned 404 on the currently configured AgentRun manager endpoint; if it was created on another lane, inspect that lane before treating the resource as lost.`, + request: { + method, + path: pathValue, + ...(resource === null ? {} : { resource }), + }, + currentEndpoint: { + transport: "direct-http", + baseUrl: config.manager.baseUrl, + configPath: config.sourcePath, + defaultNode: laneConfig?.defaultNode ?? null, + defaultLane: laneConfig?.defaultLane ?? null, + }, + candidateLanes, + nextCommands, + note: "Resource commands use manager.baseUrl from the client config; --node/--lane currently belongs to control-plane inspection commands.", + valuesPrinted: false, + }, + }; +} + +function agentRunResourceFromPath(pathValue: string): Record | null { + const run = pathValue.match(/\/runs\/([^/?#]+)/u); + if (run?.[1] !== undefined) return { kind: "run", id: decodeURIComponent(run[1]) }; + const session = pathValue.match(/\/sessions\/([^/?#]+)/u); + if (session?.[1] !== undefined) return { kind: "session", id: decodeURIComponent(session[1]) }; + const command = pathValue.match(/\/commands\/([^/?#]+)/u); + if (command?.[1] !== undefined) return { kind: "command", id: decodeURIComponent(command[1]) }; + const task = pathValue.match(/\/queue\/tasks\/([^/?#]+)/u); + if (task?.[1] !== undefined) return { kind: "task", id: decodeURIComponent(task[1]) }; + return null; +} + +function readAgentRunLaneLookupConfig(configPath: string): { defaultNode: string | null; defaultLane: string | null; lanes: Record[] } | null { + try { + const raw = readFileSync(configPath, "utf8"); + const parsed = record(Bun.YAML.parse(raw) as unknown); + const controlPlane = record(parsed.controlPlane); + const defaultTarget = record(controlPlane.default); + const lanes = record(controlPlane.lanes); + return { + defaultNode: stringOrNull(defaultTarget.node), + defaultLane: stringOrNull(defaultTarget.lane), + lanes: Object.entries(lanes).map(([laneName, rawLane]) => { + const lane = record(rawLane); + const runtime = record(lane.runtime); + return { + lane: laneName, + node: stringOrNull(lane.node) ?? stringOrNull(lane.nodeId), + version: stringOrNull(lane.version), + namespace: stringOrNull(runtime.namespace), + internalBaseUrl: stringOrNull(runtime.internalBaseUrl), + isDefault: laneName === stringOrNull(defaultTarget.lane), + }; + }), + }; + } catch { + return null; + } +} + function agentRunDryRunPlan(action: string, pathValue: string, body: Record, confirmCommand: string, method: AgentRunHttpMethod = "POST", extra: Record = {}): Record { return { ok: true, @@ -5293,9 +5393,9 @@ function safeAgentRunEnvelope(envelope: Record): Record { + const [action] = args; + if (action === undefined || action === "help" || action === "--help" || action === "-h") return gitToolsHelp(); + if (action !== "github-push-fallback") throw new Error(`unsupported git command: ${action}`); + const options = parseGitHubPushFallbackOptions(args.slice(1)); + const repo = options.repo ?? deriveGitHubRepo(options.cwd); + const branch = options.branch ?? currentGitBranch(options.cwd); + if (repo === null) throw new Error("git github-push-fallback requires --repo owner/name, or an origin remote that points at GitHub"); + if (branch === null) throw new Error("git github-push-fallback requires --branch , or a checked-out branch"); + validateGitHubRepo(repo); + validatePushRef(branch); + + const remoteUrl = `ssh://git@ssh.github.com:${options.port}/${repo}.git`; + const sshCommand = [ + "ssh", + "-o", `HostName=${options.hostName}`, + "-o", `Port=${options.port}`, + "-o", "HostKeyAlias=ssh.github.com", + "-o", "StrictHostKeyChecking=accept-new", + ].join(" "); + const argv = ["git", "push", remoteUrl, branch]; + const base = { + ok: true, + command: "git github-push-fallback", + cwd: options.cwd, + mutation: options.confirm, + repository: repo, + branch, + remoteUrl, + ssh: { + hostName: options.hostName, + port: options.port, + hostKeyAlias: "ssh.github.com", + strictHostKeyChecking: "accept-new", + }, + env: { + GIT_SSH_COMMAND: sshCommand, + valuesPrinted: false, + }, + argv, + boundary: "does not edit git remotes; intended for GitHub DNS/port-22 reachability fallback only", + }; + + if (!options.confirm) { + return { + ...base, + executed: false, + dryRun: true, + next: [ + "rerun with --confirm to execute this one push without changing origin", + "use --host-name only when both github.com and ssh.github.com DNS are broken on the target host", + ], + }; + } + + const result = spawnSync("git", ["push", remoteUrl, branch], { + cwd: options.cwd, + env: { ...process.env, GIT_SSH_COMMAND: sshCommand }, + encoding: "utf8", + timeout: 120_000, + }); + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + const ok = result.status === 0; + return { + ...base, + ok, + executed: true, + dryRun: false, + exitCode: result.status, + signal: result.signal, + stdoutTail: tail(stdout, 2000), + stderrTail: tail(stderr, 4000), + diagnostic: classifyGitPushFailure(`${stdout}\n${stderr}`), + }; +} + +function gitToolsHelp(): Record { + return { + ok: true, + command: "git", + usage: [ + "bun scripts/cli.ts git github-push-fallback --repo pikasTech/HWLAB --branch fix/name", + "bun scripts/cli.ts git github-push-fallback --repo pikasTech/HWLAB --branch fix/name --host-name 140.82.116.36 --confirm", + ], + behavior: [ + "Plans or executes a one-shot GitHub push through ssh.github.com:443 without editing git remotes.", + "Default output is a dry-run plan. Add --confirm to execute the push.", + "Use --host-name only when target-host DNS cannot resolve GitHub SSH names; HostKeyAlias remains ssh.github.com.", + ], + options: { + "--repo ": "GitHub repository. Defaults to parsing origin.", + "--branch ": "Local branch/ref to push. Defaults to the checked-out branch.", + "--cwd ": "Git worktree. Defaults to the UniDesk repo root.", + "--host-name ": "SSH HostName override. Defaults to ssh.github.com.", + "--port ": "SSH port. Defaults to 443.", + "--confirm": "Execute the push. Without this flag, only prints a plan.", + }, + }; +} + +function parseGitHubPushFallbackOptions(args: string[]): GitHubPushFallbackOptions { + const options: GitHubPushFallbackOptions = { + repo: null, + branch: null, + cwd: repoRoot, + hostName: "ssh.github.com", + port: 443, + confirm: false, + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + const next = args[index + 1]; + if (arg === "--repo") { + options.repo = requireValue(arg, next); + index += 1; + continue; + } + if (arg.startsWith("--repo=")) { + options.repo = requireValue("--repo", arg.slice("--repo=".length)); + continue; + } + if (arg === "--branch") { + options.branch = requireValue(arg, next); + index += 1; + continue; + } + if (arg.startsWith("--branch=")) { + options.branch = requireValue("--branch", arg.slice("--branch=".length)); + continue; + } + if (arg === "--cwd") { + options.cwd = resolve(requireValue(arg, next)); + index += 1; + continue; + } + if (arg.startsWith("--cwd=")) { + options.cwd = resolve(requireValue("--cwd", arg.slice("--cwd=".length))); + continue; + } + if (arg === "--host-name" || arg === "--host") { + options.hostName = requireValue(arg, next); + index += 1; + continue; + } + if (arg.startsWith("--host-name=")) { + options.hostName = requireValue("--host-name", arg.slice("--host-name=".length)); + continue; + } + if (arg === "--host-ip") { + options.hostName = requireValue(arg, next); + index += 1; + continue; + } + if (arg === "--port") { + options.port = parsePort(requireValue(arg, next)); + index += 1; + continue; + } + if (arg.startsWith("--port=")) { + options.port = parsePort(requireValue("--port", arg.slice("--port=".length))); + continue; + } + if (arg === "--confirm") { + options.confirm = true; + continue; + } + if (arg === "--dry-run") continue; + throw new Error(`unsupported git github-push-fallback option: ${arg}`); + } + validateHostName(options.hostName); + return options; +} + +function deriveGitHubRepo(cwd: string): string | null { + const result = gitCapture(cwd, ["remote", "get-url", "origin"]); + if (!result.ok) return null; + const remote = result.stdout.trim(); + const patterns = [ + /^git@github\.com:([^/\s]+\/[^/\s]+?)(?:\.git)?$/u, + /^https:\/\/github\.com\/([^/\s]+\/[^/\s]+?)(?:\.git)?$/u, + /^ssh:\/\/git@(?:github\.com|ssh\.github\.com)(?::\d+)?\/([^/\s]+\/[^/\s]+?)(?:\.git)?$/u, + ]; + for (const pattern of patterns) { + const match = remote.match(pattern); + if (match?.[1] !== undefined) return match[1]; + } + return null; +} + +function currentGitBranch(cwd: string): string | null { + const result = gitCapture(cwd, ["branch", "--show-current"]); + const branch = result.ok ? result.stdout.trim() : ""; + return branch.length > 0 ? branch : null; +} + +function gitCapture(cwd: string, args: string[]): { ok: boolean; stdout: string; stderr: string } { + const result = spawnSync("git", args, { cwd, encoding: "utf8", timeout: 10_000 }); + return { + ok: result.status === 0, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; +} + +function classifyGitPushFailure(output: string): Record | null { + if (/Could not resolve hostname github\.com|Temporary failure in name resolution|Name or service not known/iu.test(output)) { + return { + kind: "github-dns-or-name-resolution", + next: "retry with git github-push-fallback; if ssh.github.com DNS also fails, pass --host-name ", + }; + } + if (/connect to host github\.com port 22|Connection timed out|Network is unreachable/iu.test(output)) { + return { + kind: "github-port-22-or-network-blocked", + next: "retry through ssh.github.com:443 with git github-push-fallback", + }; + } + if (/Permission denied \(publickey\)|Authentication failed|Repository not found/iu.test(output)) { + return { + kind: "auth-or-repository-access", + next: "do not keep retrying network fallbacks; inspect SSH key, deploy key, repository name, and GitHub access", + }; + } + return null; +} + +function requireValue(name: string, value: string | undefined): string { + if (value === undefined || value.length === 0) throw new Error(`${name} requires a non-empty value`); + return value; +} + +function parsePort(value: string): number { + const port = Number(value); + if (!Number.isInteger(port) || port <= 0 || port > 65535) throw new Error("--port must be an integer from 1 to 65535"); + return port; +} + +function validateGitHubRepo(value: string): void { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)) throw new Error("--repo must be owner/name"); +} + +function validatePushRef(value: string): void { + if (!/^[A-Za-z0-9_./:-]+$/u.test(value) || value.includes("..") || value.startsWith("-")) throw new Error("--branch must be a simple git ref without whitespace or shell characters"); +} + +function validateHostName(value: string): void { + if (!/^[A-Za-z0-9_.:-]+$/u.test(value) || value.startsWith("-")) throw new Error("--host-name must be a hostname or IP literal"); +} + +function tail(value: string, maxChars: number): string { + return value.length > maxChars ? value.slice(-maxChars) : value; +} diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 15d17a52..04d921e7 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -57,6 +57,7 @@ export function rootHelp(): unknown { { command: "artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service", description: "Manage the D601 host-managed CNCF Distribution registry and run pull-only artifact CD for supported services, including D601 direct, k3s-managed, and code-queue dev-only consumers." }, { command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." }, { command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, issue/comment apply_patch body patching, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." }, + { command: "git github-push-fallback [--repo owner/name] [--branch branch] [--host-name host-or-ip] [--confirm]", description: "Plan or execute a one-shot GitHub push through ssh.github.com:443 without editing remotes; use only for reviewed DNS/port-22 push fallback." }, { command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run", description: "Host Codex commander skeleton contract, no-daemon smoke plan, and dry-run approval preview without live bridges or message sends." }, { command: "hwlab nodes control-plane|git-mirror|secret|test-accounts|web-probe --node --lane ", description: "Manage HWLAB node/lane runtime prerequisites, including D601 YAML-declared infra/tools-image/Argo bootstrap, redacted test-account preparation, Web DOM probe credential injection, and G14 v0.3+ runtime lanes, with the node identity passed as data." }, { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, diff --git a/scripts/src/ssh-playwright.ts b/scripts/src/ssh-playwright.ts index c0f6438c..55571bee 100644 --- a/scripts/src/ssh-playwright.ts +++ b/scripts/src/ssh-playwright.ts @@ -66,8 +66,20 @@ export async function runSshPlaywrightOperation( const localDir = resolve(options.localDir); const submitCommand = builders.buildRouteCommand(invocation.route, ["sh", "-c", remotePlaywrightSubmitScript(remoteDir), "unidesk-playwright-submit"], { stdin: true }); const submit = await executor.runRemoteCommand(submitCommand, userScript); - if (submit.exitCode !== 0) { - throw new Error(`ssh playwright submit failed: exitCode=${submit.exitCode}; stdoutTail=${JSON.stringify(submit.stdout.slice(-1000))}; stderrTail=${JSON.stringify(submit.stderr.slice(-1000))}`); + const submitRecovery = submit.exitCode !== 0 && isRecoverableSubmitTimeout(submit) + ? { + recovered: true, + reason: "submit-short-connection-timeout", + remoteDir, + runId, + exitCode: submit.exitCode, + stdoutTail: submit.stdout.slice(-1000), + stderrTail: submit.stderr.slice(-1000), + next: "submit may have been cut by the 60s trans runtime limit after the background job was launched; polling remote status by remoteDir/runId", + } + : null; + if (submit.exitCode !== 0 && submitRecovery === null) { + throw new Error(`ssh playwright submit failed: exitCode=${submit.exitCode}; remoteDir=${remoteDir}; runId=${runId}; stdoutTail=${JSON.stringify(submit.stdout.slice(-1000))}; stderrTail=${JSON.stringify(submit.stderr.slice(-1000))}`); } const manifest = await pollRemoteManifest(invocation.route, executor, builders, remoteDir, runId, options); const artifacts: Array = []; @@ -124,6 +136,8 @@ export async function runSshPlaywrightOperation( remoteCommand: { exitCode: manifest.exitCode, submitExitCode: submit.exitCode, + submitRecovered: submitRecovery !== null, + submitRecovery, submitStdoutBytes: Buffer.byteLength(submit.stdout, "utf8"), submitStderrBytes: Buffer.byteLength(submit.stderr, "utf8"), }, @@ -272,17 +286,23 @@ async function pollRemoteManifest( lastStatus = status; if (status.exitCode === 0 && status.stdout.includes(manifestEnd)) return parseRemoteManifest(status, remoteDir, runId); if (status.exitCode !== 0 && !/status\t(?:pending|running)/u.test(status.stdout)) { - throw new Error(`ssh playwright status failed: exitCode=${status.exitCode}; stdoutTail=${JSON.stringify(status.stdout.slice(-1000))}; stderrTail=${JSON.stringify(status.stderr.slice(-1000))}`); + throw new Error(`ssh playwright status failed: exitCode=${status.exitCode}; remoteDir=${remoteDir}; runId=${runId}; stdoutTail=${JSON.stringify(status.stdout.slice(-1000))}; stderrTail=${JSON.stringify(status.stderr.slice(-1000))}`); } await sleep(options.pollIntervalMs); } - throw new Error(`ssh playwright timed out waiting for remote job after ${options.waitTimeoutMs}ms; lastStdoutTail=${JSON.stringify(lastStatus?.stdout.slice(-1000) ?? "")}; lastStderrTail=${JSON.stringify(lastStatus?.stderr.slice(-1000) ?? "")}`); + throw new Error(`ssh playwright timed out waiting for remote job after ${options.waitTimeoutMs}ms; remoteDir=${remoteDir}; runId=${runId}; lastStdoutTail=${JSON.stringify(lastStatus?.stdout.slice(-1000) ?? "")}; lastStderrTail=${JSON.stringify(lastStatus?.stderr.slice(-1000) ?? "")}`); } function sleep(ms: number): Promise { return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); } +function isRecoverableSubmitTimeout(submit: SshCaptureResult): boolean { + if (submit.exitCode === 124) return true; + const combined = `${submit.stdout}\n${submit.stderr}`; + return /(?:timed?\s*out|timeout|60s|exitCode=124|signal\s+TERM|signal\s+KILL)/iu.test(combined); +} + function remotePlaywrightSubmitScript(remoteDir: string): string { const runner = Buffer.from(remotePlaywrightRunnerScript(remoteDir), "utf8").toString("base64"); return [