import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { rootPath, type UniDeskConfig } from "./config"; import type { RenderedCliResult } from "./output"; import { applyLocalCaddyManagedSite } from "./pk01-caddy"; import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; import { runRemoteSshCommandCapture } from "./remote"; import { startJob } from "./jobs"; import { AGENTRUN_CONFIG_PATH, agentRunLaneSummary, agentRunPipelineRunName, agentRunProviderCredentialRefs, resolveAgentRunLaneTarget, type AgentRunLaneSpec, } from "./agentrun-lanes"; import { agentRunImageArtifact, placeholderAgentRunImage, renderedFilesDigest, renderedObjectsDigest, renderAgentRunControlPlaneManifests, renderAgentRunGitopsFiles, type AgentRunArtifactService, } from "./agentrun-manifests"; import { sha256Fingerprint } from "./platform-infra-ops-library"; export function agentRunHelp(): unknown { return { command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|explain", output: "human by default; use -o json|yaml or --raw for machine/debug output", usage: [ "bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", "bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", "bun scripts/cli.ts agentrun get tasks -o wide", "bun scripts/cli.ts agentrun get tasks -o json", "bun scripts/cli.ts agentrun describe task/", "bun scripts/cli.ts agentrun events run/ --after-seq 0 --limit 100", "bun scripts/cli.ts agentrun logs session/ --tail 100", "bun scripts/cli.ts agentrun result run/ --command ", "bun scripts/cli.ts agentrun ack task/ --reader-id cli", "bun scripts/cli.ts agentrun cancel task/ --reason --dry-run", "bun scripts/cli.ts agentrun dispatch task/", "bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin --idempotency-key ", "bun scripts/cli.ts agentrun apply -f - --dry-run", "bun scripts/cli.ts agentrun send session/ --aipod Artificer --prompt-stdin", "bun scripts/cli.ts agentrun explain task", "bun scripts/cli.ts agentrun explain session-policy", "bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", "bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", "bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane status", "bun scripts/cli.ts agentrun control-plane status --full", "bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-vNN-ci-", "bun scripts/cli.ts agentrun control-plane status --source-commit ", "bun scripts/cli.ts agentrun control-plane expose --dry-run", "bun scripts/cli.ts agentrun control-plane expose --confirm", "bun scripts/cli.ts agentrun control-plane trigger-current --dry-run", "bun scripts/cli.ts agentrun control-plane trigger-current --confirm", "bun scripts/cli.ts agentrun control-plane refresh --dry-run", "bun scripts/cli.ts agentrun control-plane refresh --confirm", "bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run", "bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm", "bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --dry-run", "bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --confirm", "bun scripts/cli.ts agentrun git-mirror status", "bun scripts/cli.ts agentrun git-mirror status --full", "bun scripts/cli.ts agentrun git-mirror sync --confirm", "bun scripts/cli.ts agentrun git-mirror flush --confirm", ], resources: ["task/qt", "run", "command/cmd", "runnerjob/rjob", "session/ses", "aipodspec/aps"], description: "Operate AgentRun through Kubernetes-style resource verbs. Human output is compact by default; -o json|yaml returns the UniDesk render-only client schema, --raw exposes the REST envelope, and --node/--lane targets a YAML-declared runtime lane.", legacyCompatibility: "queue/runs/commands/runner/sessions/aipod-specs remain as compatibility groups backed by direct HTTP; new commander work should use get/describe/events/logs/result/ack/cancel/dispatch/create/apply/send. sessions turn/steer are removed; use send only.", }; } export async function runAgentRunCommand(config: UniDeskConfig | null, args: string[]): Promise | RenderedCliResult> { const route = normalizeAgentRunCommandArgs(args); const { group, action, actionArgs } = route; if (group === undefined || isHelpArg(group)) return renderAgentRunHelp(route.canonicalArgs); if (isHelpArg(action) || actionArgs.some(isHelpArg)) return renderAgentRunHelp(route.canonicalArgs); if (isResourceVerb(group)) return await runAgentRunResourceCommand(config, group, action, actionArgs, route.canonicalArgs); if (isAgentRunRestCompatGroup(group)) { return await runAgentRunRestCompatCommand(config, group, route.forwardArgs, route.canonicalArgs); } if (config === null) throw new Error("agentrun control-plane and git-mirror commands require UniDesk config"); if (group === "control-plane") { if (action === "plan") return await controlPlanePlan(config, parseStatusOptions(actionArgs)); if (action === "apply") return await controlPlaneApply(config, parseLaneConfirmOptions(actionArgs)); if (action === "status") { const options = parseStatusOptions(actionArgs); const result = await status(config, options); return options.full || options.raw ? result : renderAgentRunControlPlaneStatusSummary(result); } if (action === "secret-sync") return await secretSync(config, parseSecretSyncOptions(actionArgs)); if (action === "restart") return await restartYamlLane(config, parseLaneConfirmOptions(actionArgs)); if (action === "expose") return await exposeAgentRun(config, parseConfirmOptions(actionArgs)); if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(actionArgs)); if (action === "refresh") return await refresh(config, parseRefreshOptions(actionArgs)); if (action === "cleanup-runners") return await cleanupRunners(config, parseCleanupRunnersOptions(actionArgs)); if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(actionArgs)); if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs)); } if (group === "git-mirror") { if (action === "status") return await gitMirrorStatus(config, parseGitMirrorStatusOptions(actionArgs)); if (action === "sync" || action === "flush") { const options = parseGitMirrorOptions(actionArgs); const { spec } = resolveAgentRunLaneTarget(options); if (options.confirm && !options.wait) { return startAsyncAgentRunJob( `agentrun_${spec.lane}_git_mirror_${action}`, ["bun", "scripts/cli.ts", "agentrun", "git-mirror", action, "--node", spec.nodeId, "--lane", spec.lane, "--confirm", "--wait", "--timeout-seconds", String(options.timeoutSeconds)], `Run AgentRun ${spec.version} git mirror ${action} on ${spec.nodeId}`, ); } return await runGitMirrorJob(config, action, options); } } return unsupported(route.canonicalArgs); } function isAgentRunRestCompatGroup(group: string | undefined): group is AgentRunRestCompatGroup { return group === "queue" || group === "sessions" || group === "aipod-specs" || group === "aipods" || group === "runs" || group === "commands" || group === "runner"; } interface AgentRunCommandRoute { canonicalArgs: string[]; group: string | undefined; action: string | undefined; actionArgs: string[]; forwardArgs: string[]; } function normalizeAgentRunCommandArgs(args: string[]): AgentRunCommandRoute { const canonicalArgs = args[0] === "v01" ? args.slice(1) : args; const [group, action, ...actionArgs] = canonicalArgs; return { canonicalArgs, group, action, actionArgs, forwardArgs: canonicalArgs.slice(1), }; } function isHelpArg(value: string | undefined): boolean { return value === "help" || value === "--help" || value === "-h"; } function isResourceVerb(value: string | undefined): value is AgentRunResourceVerb { return value === "get" || value === "describe" || value === "events" || value === "logs" || value === "result" || value === "ack" || value === "cancel" || value === "dispatch" || value === "create" || value === "apply" || value === "send" || value === "explain"; } function renderAgentRunHelp(args: string[]): RenderedCliResult { const text = agentRunHelpText(args); return renderedCliResult(true, "agentrun help", text); } function agentRunHelpText(args: string[]): string { const [verb, kind] = args.filter((arg) => !isHelpArg(arg)); if (verb === "get" && kind !== undefined) return agentRunGetKindHelp(kind); if (verb === "get") { return [ "Usage: bun scripts/cli.ts agentrun get [options]", "", "Resources: tasks|sessions|runs|commands|runnerjobs|aipodspecs", "Output: table by default; use -o wide|name|json|yaml.", "", "Examples:", " bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", " bun scripts/cli.ts agentrun get sessions --limit 20", " bun scripts/cli.ts agentrun get tasks -o json", ].join("\n"); } if (verb === "describe") { return [ "Usage: bun scripts/cli.ts agentrun describe [--node --lane ] [--full] [-o json|yaml] [--raw]", "", "Kinds: task|run|command|runnerjob|session|aipodspec", "Examples:", " bun scripts/cli.ts agentrun describe task/qt_...", " bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", " bun scripts/cli.ts agentrun describe command/cmd_... --run ", " bun scripts/cli.ts agentrun describe session/ --full", ].join("\n"); } if (verb === "events") { return "Usage: bun scripts/cli.ts agentrun events run/ [--after-seq N] [--limit 100] [-o json|yaml] [--raw]"; } if (verb === "logs") { return "Usage: bun scripts/cli.ts agentrun logs session/ [--tail 100|--after-seq N] [--limit 100] [--full-text] [-o json|yaml] [--raw]"; } if (verb === "result") { return "Usage: bun scripts/cli.ts agentrun result run/ --command | agentrun result command/ --run "; } if (verb === "ack") { return "Usage: bun scripts/cli.ts agentrun ack task/|session/ [--reader-id cli]"; } if (verb === "cancel") { return "Usage: bun scripts/cli.ts agentrun cancel task/|session/|run/|command/ --reason [--run ] [--dry-run]"; } if (verb === "dispatch") { return "Usage: bun scripts/cli.ts agentrun dispatch task/"; } if (verb === "create") { return "Usage: bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin [--idempotency-key ] [--dry-run]"; } if (verb === "apply") { return "Usage: bun scripts/cli.ts agentrun apply -f task.yaml|json|- [--dry-run]\nTask manifests use kind: Task and spec: ."; } if (verb === "send") { return "Usage: bun scripts/cli.ts agentrun send session/ --aipod Artificer --prompt-stdin\nThe server decides whether this becomes an internal steer or a new turn from session state."; } if (verb === "explain") return agentRunExplain(kind ?? "task"); if (verb === "control-plane") { return [ "Usage: bun scripts/cli.ts agentrun control-plane [options]", "", "Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-released-pvs", "Examples:", " bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", " bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", " bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane status", " bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-vNN-ci-", " bun scripts/cli.ts agentrun control-plane expose --dry-run", " bun scripts/cli.ts agentrun control-plane trigger-current --dry-run", " bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run", ].join("\n"); } if (verb === "git-mirror") { return [ "Usage: bun scripts/cli.ts agentrun git-mirror [--full|--raw|--confirm]", "", "Confirmed sync/flush returns an async job unless --wait is set.", ].join("\n"); } if (verb !== undefined && isAgentRunRestCompatGroup(verb)) { return [ `Compatibility group: agentrun ${verb} ...`, "", "Use resource primitives for daily commander work:", " bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", " bun scripts/cli.ts agentrun describe task/", " bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", " bun scripts/cli.ts agentrun events run/ --after-seq 0 --limit 100", " bun scripts/cli.ts agentrun logs session/ --tail 100", "", "Use --raw on a resource command when you need the direct AgentRun REST envelope.", ].join("\n"); } return [ "Usage: bun scripts/cli.ts agentrun [options]", "", "Verbs: get, describe, events, logs, result, ack, cancel, dispatch, create, apply, send, explain", "Resources: task/qt, run, command/cmd, runnerjob/rjob, session/ses, aipodspec/aps", "", "Common:", " bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", " bun scripts/cli.ts agentrun describe task/", " bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", " bun scripts/cli.ts agentrun events run/ --after-seq 0 --limit 100", " bun scripts/cli.ts agentrun logs session/ --tail 100", " bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin", "", "Machine/debug output:", " -o json|yaml emits a stable UniDesk resource object without the global JSON envelope.", " --raw emits the direct AgentRun REST envelope.", " --node/--lane targets a YAML-declared AgentRun runtime lane; default remains the configured public manager.", ].join("\n"); } function agentRunGetKindHelp(kindRaw: string): string { const kind = parseResourceKind(kindRaw); if (kind === "task") return "Usage: bun scripts/cli.ts agentrun get tasks [--queue commander] [--state running,pending,completed,failed] [--unread] [--limit 20] [-o wide|name|json|yaml]"; if (kind === "session") return "Usage: bun scripts/cli.ts agentrun get sessions [--limit 20] [-o wide|name|json|yaml]"; if (kind === "run") return "Usage: bun scripts/cli.ts agentrun get runs --task [--limit 20] [-o wide|name|json|yaml]"; if (kind === "command") return "Usage: bun scripts/cli.ts agentrun get commands --run [--command ] [-o wide|name|json|yaml]"; if (kind === "runnerjob") return "Usage: bun scripts/cli.ts agentrun get runnerjobs --run --command [-o wide|name|json|yaml]"; if (kind === "aipodspec") return "Usage: bun scripts/cli.ts agentrun get aipodspecs [-o wide|name|json|yaml]"; return "Unknown resource. Supported: tasks, sessions, runs, commands, runnerjobs, aipodspecs."; } async function runAgentRunResourceCommand(config: UniDeskConfig | null, verb: AgentRunResourceVerb, action: string | undefined, actionArgs: string[], canonicalArgs: string[]): Promise { if (isHelpArg(action) || actionArgs.some(isHelpArg)) return renderAgentRunHelp(canonicalArgs); if (verb === "explain") return renderedCliResult(true, "agentrun explain", agentRunExplain(action ?? "task")); const resourceArgs = action === undefined ? actionArgs : [action, ...actionArgs]; const options = parseResourceOptions(resourceArgs); const bridgeActionArgs = stripAgentRunResourceWrapperArgs(actionArgs); const command = `agentrun ${canonicalArgs.join(" ")}`.trim(); try { return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, options), async () => { if (verb === "get") return await resourceGet(config, command, action, bridgeActionArgs, options); if (verb === "describe") return await resourceDescribe(config, command, action, bridgeActionArgs, options); if (verb === "events") return await resourceEvents(config, command, action, bridgeActionArgs, options); if (verb === "logs") return await resourceLogs(config, command, action, bridgeActionArgs, options); if (verb === "result") return await resourceResult(config, command, action, bridgeActionArgs, options); if (verb === "ack") return await resourceAck(config, command, action, bridgeActionArgs, options); if (verb === "cancel") return await resourceCancel(config, command, action, bridgeActionArgs, options); if (verb === "dispatch") return await resourceDispatch(config, command, action, bridgeActionArgs, options); if (verb === "create") return await resourceCreate(config, command, action, bridgeActionArgs, options); if (verb === "apply") return await resourceApply(config, command, bridgeActionArgs, options); if (verb === "send") return await resourceSessionPromptCommand(config, command, action, bridgeActionArgs, options); return renderedCliResult(false, command, `Unsupported AgentRun resource command. Try: bun scripts/cli.ts agentrun --help`); }); } catch (error) { if (error instanceof AgentRunRestError) return renderAgentRunRestError(command, error, options); return renderedCliResult(false, command, `Error: ${error instanceof Error ? error.message : String(error)}`); } return renderedCliResult(false, command, `Unsupported AgentRun resource command. Try: bun scripts/cli.ts agentrun --help`); } export function stripAgentRunResourceWrapperArgs(args: string[]): string[] { const result: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (arg === "-o" || arg === "--output") { index += 1; continue; } if (arg === "--node" || arg === "--lane") { index += 1; continue; } if (arg.startsWith("--output=") || arg.startsWith("-o=")) continue; if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue; if (arg === "--full" || arg === "--raw" || arg === "--debug" || arg === "--full-text") continue; result.push(arg); } return result; } function parseResourceOptions(args: string[]): AgentRunResourceOptions { const options: AgentRunResourceOptions = { output: "human", full: false, raw: false, debug: false, limit: 20, queue: null, state: null, unread: false, readerId: "cli", taskId: null, runId: null, commandId: null, sessionId: null, afterSeq: null, tail: null, fullText: false, reason: null, dryRun: false, file: null, aipod: null, idempotencyKey: null, promptStdin: false, node: null, lane: null, passthroughArgs: [], }; const valueFlags = new Set(["-o", "--output", "--limit", "--queue", "--state", "--reader-id", "--task", "--task-id", "--run", "--run-id", "--command", "--command-id", "--session", "--session-id", "--after-seq", "--tail", "--reason", "-f", "--file", "--filename", "--aipod", "--idempotency-key", "--node", "--lane"]); const booleanFlags = new Set(["--full", "--raw", "--debug", "--unread", "--dry-run", "--full-text", "--prompt-stdin", "--stdin"]); for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (!arg.startsWith("-")) continue; if (arg.startsWith("--") && arg.includes("=")) { const [flag, ...rest] = arg.split("="); applyResourceOption(options, flag, rest.join("=")); continue; } if (booleanFlags.has(arg)) { applyResourceOption(options, arg, null); continue; } if (valueFlags.has(arg)) { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); applyResourceOption(options, arg, value); index += 1; continue; } } return options; } function applyResourceOption(options: AgentRunResourceOptions, flag: string, value: string | null): void { if (flag === "-o" || flag === "--output") { if (value !== "human" && value !== "wide" && value !== "name" && value !== "json" && value !== "yaml") throw new Error(`${flag} must be one of human,wide,name,json,yaml`); options.output = value; return; } if (flag === "--full") options.full = true; else if (flag === "--raw") options.raw = true; else if (flag === "--debug") options.debug = true; else if (flag === "--unread") options.unread = true; else if (flag === "--dry-run") options.dryRun = true; else if (flag === "--full-text") options.fullText = true; else if (flag === "--prompt-stdin" || flag === "--stdin") options.promptStdin = true; else if (flag === "--limit") options.limit = parseNonNegativeInt(value, "--limit", 20, 500); else if (flag === "--queue") options.queue = requiredValue(value, flag); else if (flag === "--state") options.state = requiredValue(value, flag); else if (flag === "--reader-id") options.readerId = requiredValue(value, flag); else if (flag === "--task" || flag === "--task-id") options.taskId = requiredValue(value, flag); else if (flag === "--run" || flag === "--run-id") options.runId = requiredValue(value, flag); else if (flag === "--command" || flag === "--command-id") options.commandId = requiredValue(value, flag); else if (flag === "--session" || flag === "--session-id") options.sessionId = requiredValue(value, flag); else if (flag === "--after-seq") options.afterSeq = parseNonNegativeInt(value, "--after-seq", 0, Number.MAX_SAFE_INTEGER); else if (flag === "--tail") options.tail = parseNonNegativeInt(value, "--tail", 100, 1000); else if (flag === "--reason") options.reason = requiredValue(value, flag); else if (flag === "-f" || flag === "--file" || flag === "--filename") options.file = requiredValue(value, flag); else if (flag === "--aipod") options.aipod = requiredValue(value, flag); else if (flag === "--idempotency-key") options.idempotencyKey = requiredValue(value, flag); else if (flag === "--node") options.node = requiredValue(value, flag); else if (flag === "--lane") options.lane = requiredValue(value, flag); } function parseNonNegativeInt(raw: string | null, flag: string, defaultValue: number, maxValue: number): number { if (raw === null || raw.length === 0) return defaultValue; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${flag} must be a non-negative integer`); return Math.min(value, maxValue); } function requiredValue(value: string | null, flag: string): string { if (value === null || value.length === 0) throw new Error(`${flag} requires a value`); return value; } async function resourceGet(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const kind = parseResourceKind(action); if (kind === null) throw new Error("get requires a resource: tasks|sessions|runs|commands|runnerjobs|aipodspecs"); let result: Record; if (kind === "task") { const taskListArgs = options.unread ? ["commander", "--reader-id", options.readerId, "--limit", String(options.limit)] : [ "list", "--state", taskListState(options), "--limit", String(options.queue === null ? options.limit : Math.max(options.limit, 100)), ...(options.queue === null ? [] : ["--queue", options.queue]), ]; result = await runAgentRunRestCommand(config, "queue", taskListArgs); return renderResourceResult(command, result, options, "Task", normalizeTaskItems(innerData(result), options).slice(0, options.limit)); } if (kind === "session") { result = await runAgentRunRestCommand(config, "sessions", ["ps", "--limit", String(options.limit)]); return renderResourceResult(command, result, options, "Session", normalizeSessionItems(innerData(result)).slice(0, options.limit)); } if (kind === "aipodspec") { result = await runAgentRunRestCommand(config, "aipod-specs", ["list"]); return renderResourceResult(command, result, options, "AipodSpec", normalizeAipodSpecItems(innerData(result)).slice(0, options.limit)); } if (kind === "run" && options.taskId !== null) { result = await runAgentRunRestCommand(config, "queue", ["show", options.taskId]); const task = record(innerData(result)); return renderResourceResult(command, result, options, "Run", normalizeAttemptResources(task, "run")); } if (kind === "command" && options.runId !== null && options.commandId !== null) { result = await runAgentRunRestCommand(config, "commands", ["show", options.commandId, "--run-id", options.runId]); return renderResourceResult(command, result, options, "Command", normalizeSingleCommand(innerData(result))); } if (kind === "runnerjob" && options.runId !== null && options.commandId !== null) { result = await runAgentRunRestCommand(config, "runner", ["jobs", "--run-id", options.runId, "--command-id", options.commandId]); return renderResourceResult(command, result, options, "RunnerJob", normalizeRunnerJobItems(innerData(result))); } throw new Error(`get ${kind}s requires more context; use --task, --run/--command, or describe /`); } async function resourceDescribe(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args); if (ref.kind === "task") { const result = await runAgentRunRestCommand(config, "queue", ["show", ref.name, ...(options.full ? ["--full"] : [])]); const data = record(innerData(result)); const task = unwrapTaskDetail(data); if (options.raw) return renderMachine(command, result, "json", result.ok !== false); if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: ref.kind, name: ref.name, resource: task }, options.output, result.ok !== false); return renderedCliResult(result.ok !== false, command, renderTaskDescription(task, options)); } if (ref.kind === "run") { const result = await runAgentRunRestCommand(config, "runs", ["show", ref.name, ...(options.full ? ["--full"] : [])]); return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result))); } if (ref.kind === "command") { const runId = options.runId ?? requiredContext("command describe", "--run "); const result = await runAgentRunRestCommand(config, "commands", ["show", ref.name, "--run-id", runId, ...(options.full ? ["--full"] : [])]); return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result))); } if (ref.kind === "runnerjob") { const runId = options.runId ?? requiredContext("runnerjob describe", "--run "); const result = await runAgentRunRestCommand(config, "runner", ["job-status", ref.name, "--run-id", runId]); return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result))); } if (ref.kind === "session") { const result = await runAgentRunRestCommand(config, "sessions", ["show", ref.name, ...(options.full ? ["--full"] : [])]); return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result))); } if (ref.kind === "aipodspec") { const result = await runAgentRunRestCommand(config, "aipod-specs", ["show", ref.name, ...(options.full ? ["--full"] : [])]); return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result))); } throw new Error(`unsupported describe kind: ${ref.kind}`); } async function resourceEvents(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "run"); const runId = ref.kind === "run" ? ref.name : options.runId ?? requiredContext("events", "--run "); const eventArgs = ["events", runId, "--after-seq", String(options.afterSeq ?? 0), "--limit", String(options.limit), "--tail-summary"]; const result = await runAgentRunRestCommand(config, "runs", eventArgs); return renderEventLike(command, result, options, "Event", normalizeEventItems(innerData(result)), runId); } async function resourceLogs(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "session"); if (ref.kind !== "session") throw new Error("logs currently requires session/"); const effectiveLimit = options.tail ?? options.limit; const logArgs = ["output", ref.name, "--limit", String(effectiveLimit)]; if (options.afterSeq !== null) logArgs.push("--after-seq", String(options.afterSeq)); if (options.fullText) logArgs.push("--full-text"); let result: Record; if (options.afterSeq === null && options.tail !== null) { result = await resourceLogsTailResult(config, ref.name, effectiveLimit, options.fullText); } else { result = await runAgentRunRestCommand(config, "sessions", logArgs); } return renderEventLike(command, result, { ...options, limit: effectiveLimit }, "Log", normalizeLogItems(innerData(result)), ref.name); } async function resourceLogsTailResult(config: UniDeskConfig | null, sessionId: string, limit: number, fullText: boolean): Promise> { const session = record(innerData(await runAgentRunRestCommand(config, "sessions", ["show", sessionId]))); const lastSeq = nonNegativeIntegerOrNull(session.lastEventSeq) ?? 0; let window = Math.max(limit * 4, 100); let result: Record | null = null; let data: Record = {}; for (let attempt = 0; attempt < 6; attempt += 1) { const afterSeq = Math.max(0, lastSeq - window); const queryLimit = Math.max(limit, Math.min(1000, window)); const args = ["output", sessionId, "--limit", String(queryLimit), "--after-seq", String(afterSeq)]; if (fullText) args.push("--full-text"); result = await runAgentRunRestCommand(config, "sessions", args); data = record(innerData(result)); const items = arrayRecords(data.items); if (items.length >= limit || afterSeq === 0) break; window *= 2; } if (result === null) return await runAgentRunRestCommand(config, "sessions", ["output", sessionId, "--limit", String(limit), "--after-seq", "0"]); const items = arrayRecords(data.items); const tailedItems = items.slice(-limit); const lastReturnedSeq = tailedItems.length > 0 ? nonNegativeIntegerOrNull(tailedItems[tailedItems.length - 1]?.seq) : null; return { ...result, data: { ...data, items: tailedItems, count: tailedItems.length, cursor: lastReturnedSeq === null ? data.cursor : String(lastReturnedSeq), clientTail: { requested: limit, sourceLastEventSeq: lastSeq, scanned: items.length, valuesPrinted: false, }, }, }; } async function resourceResult(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args); if (ref.kind === "run") { const commandId = options.commandId ?? requiredContext("run result", "--command "); const result = await runAgentRunRestCommand(config, "runs", ["result", ref.name, "--command-id", commandId]); return renderResultSummary(command, result, options, ref); } if (ref.kind === "command") { const runId = options.runId ?? requiredContext("command result", "--run "); const result = await runAgentRunRestCommand(config, "commands", ["result", ref.name, "--run-id", runId]); return renderResultSummary(command, result, options, ref); } throw new Error("result supports run/ or command/"); } async function resourceAck(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "task"); const result = ref.kind === "task" ? await runAgentRunRestCommand(config, "queue", ["read", ref.name, "--reader-id", options.readerId]) : ref.kind === "session" ? await runAgentRunRestCommand(config, "sessions", ["read", ref.name, "--reader-id", options.readerId]) : null; if (result === null) throw new Error("ack supports task/ or session/"); return renderMutationSummary(command, result, options, `Acknowledged ${ref.kind}/${shortId(ref.name)}`); } async function resourceCancel(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "task"); const cancelArgs = ["cancel", ref.name]; if (options.reason !== null) cancelArgs.push("--reason", options.reason); if (ref.kind === "command") cancelArgs.push("--run-id", options.runId ?? requiredContext("command cancel", "--run ")); if (options.dryRun) { const result = agentRunResourceCancelDryRunPlan(ref, options, rerunWithoutDryRun(command)); return renderMutationSummary(command, result, options, `Planned cancel ${ref.kind}/${shortId(ref.name)}`, [rerunWithoutDryRun(command)]); } const result = ref.kind === "task" ? await runAgentRunRestCommand(config, "queue", cancelArgs) : ref.kind === "session" ? await runAgentRunRestCommand(config, "sessions", cancelArgs) : ref.kind === "run" ? await runAgentRunRestCommand(config, "runs", cancelArgs) : ref.kind === "command" ? await runAgentRunRestCommand(config, "commands", cancelArgs) : null; if (result === null) throw new Error("cancel supports task/, session/, run/, or command/"); return renderMutationSummary(command, result, options, `${options.dryRun ? "Planned cancel" : "Cancel requested"} ${ref.kind}/${shortId(ref.name)}`, options.dryRun ? [rerunWithoutDryRun(command)] : undefined); } function agentRunResourceCancelDryRunPlan(ref: AgentRunResourceRef, options: AgentRunResourceOptions, confirmCommand: string): Record { const body: Record = {}; if (options.reason !== null) body.reason = options.reason; if (ref.kind === "task") return agentRunDryRunPlan("task-cancel", `/api/v1/queue/tasks/${encodeURIComponent(ref.name)}/cancel`, body, confirmCommand); if (ref.kind === "session") return agentRunDryRunPlan("session-cancel", `/api/v1/sessions/${encodeURIComponent(ref.name)}/control`, { action: "cancel", ...body }, confirmCommand); if (ref.kind === "run") return agentRunDryRunPlan("run-cancel", `/api/v1/runs/${encodeURIComponent(ref.name)}/cancel`, body, confirmCommand); if (ref.kind === "command") { const runId = options.runId ?? requiredContext("command cancel", "--run "); return agentRunDryRunPlan("command-cancel", `/api/v1/commands/${encodeURIComponent(ref.name)}/cancel`, body, confirmCommand, "POST", { commandRef: { runId, commandId: ref.name, valuesPrinted: false }, }); } throw new Error("cancel supports task/, session/, run/, or command/"); } async function resourceDispatch(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "task"); if (ref.kind !== "task") throw new Error("dispatch supports task/"); const result = await runAgentRunRestCommand(config, "queue", ["dispatch", ref.name, ...stripLeadingResource(args, ref.name)]); return renderMutationSummary(command, result, options, `Task dispatch submitted ${shortId(ref.name)}`, [ `bun scripts/cli.ts agentrun describe task/${ref.name}`, ]); } async function resourceCreate(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const kind = parseResourceKind(action); if (kind !== "task") throw new Error("create currently supports: create task"); const submitArgs = ["submit", ...stripLeadingResource(args, "task")]; const result = await runAgentRunRestCommand(config, "queue", submitArgs); return renderMutationSummary(command, result, options, "Task create submitted", options.dryRun ? [rerunWithoutDryRun(command)] : undefined); } async function resourceApply(config: UniDeskConfig | null, command: string, args: string[], options: AgentRunResourceOptions): Promise { void config; const file = options.file ?? requiredContext("apply", "-f |-"); const raw = file === "-" ? readFileSync(0, "utf8") : readFileSync(file, "utf8"); const parsed = parseTaskManifest(raw, file); const result = options.dryRun ? agentRunDryRunPlan("queue-submit", "/api/v1/queue/tasks", parsed, `bun scripts/cli.ts ${command.replace(/\s+--dry-run\b/gu, "").trim()}`) : await agentRunRestRequest("agentrun queue submit", "POST", "/api/v1/queue/tasks", parsed); return renderMutationSummary(command, result, options, `${options.dryRun ? "Dry-run applied" : "Applied"} task manifest`, options.dryRun ? [rerunWithoutDryRun(command)] : undefined); } async function resourceSessionPromptCommand(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "session"); if (ref.kind !== "session") throw new Error("send requires session/"); const sessionArgs = ["send", ref.name, ...stripLeadingResource(args, ref.name)]; const result = await runAgentRunRestCommand(config, "sessions", sessionArgs); return renderMutationSummary(command, result, options, options.dryRun ? "Session send plan" : "Session send submitted"); } function renderedCliResult(ok: boolean, command: string, renderedText: string, contentType: RenderedCliResult["contentType"] = "text/plain"): RenderedCliResult { return { ok, command, renderedText, contentType }; } function renderResourceResult(command: string, raw: Record, options: AgentRunResourceOptions, kindLabel: string, items: Record[]): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); const payload = { kind: `${kindLabel}List`, count: items.length, items }; if (options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output); if (options.output === "name") return renderedCliResult(raw.ok !== false, command, items.map((item) => `${String(item.kind ?? kindLabel).toLowerCase()}/${String(item.name ?? "")}`).join("\n")); if (items.length === 0) return renderedCliResult(raw.ok !== false, command, `No ${kindLabel.toLowerCase()} resources found.`); return renderedCliResult(raw.ok !== false, command, renderResourceTable(items, options.output === "wide")); } function renderDescribe(command: string, raw: Record, options: AgentRunResourceOptions, ref: AgentRunResourceRef, text: string): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: ref.kind, name: ref.name, resource: innerData(raw) }, options.output, raw.ok !== false); return renderedCliResult(raw.ok !== false, command, text); } function renderEventLike(command: string, raw: Record, options: AgentRunResourceOptions, kind: string, items: Record[], sourceName: string): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); const data = record(innerData(raw)); const payload = { kind: `${kind}List`, source: sourceName, nextAfterSeq: data.nextAfterSeq ?? null, count: items.length, items }; if (options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output, raw.ok !== false); const lines = [ `${kind}s for ${sourceName}`, renderTable(["SEQ", "TYPE", "STATUS", "COMMAND", "SUMMARY"], items.map((item) => [ String(item.seq ?? "-"), String(item.type ?? "-"), String(item.status ?? item.phase ?? "-"), shortId(String(item.commandId ?? "-")), truncateOneLine(String(item.summary ?? item.text ?? "") || "-", 96), ])), ]; if (data.nextAfterSeq !== undefined && data.nextAfterSeq !== null) lines.push(`Next: bun scripts/cli.ts ${nextPagedResourceCommand(command, String(data.nextAfterSeq), options.limit)}`); return renderedCliResult(raw.ok !== false, command, lines.join("\n")); } function renderResultSummary(command: string, raw: Record, options: AgentRunResourceOptions, ref: AgentRunResourceRef): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); const data = record(innerData(raw)); if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: "Result", ref, result: data }, options.output, raw.ok !== false); const executionOk = typeof data.executionOk === "boolean" ? data.executionOk : typeof data.ok === "boolean" ? data.ok : raw.ok !== false; const lines = [ `Result: ${ref.kind}/${shortId(ref.name)}`, `State: ${displayValue(data.state ?? data.status ?? data.terminalStatus ?? "-")}`, `OK: ${String(executionOk)}`, ]; const providerTerminalFailure = typeof data.providerTerminalFailure === "boolean" ? data.providerTerminalFailure : null; const authority = stringOrNull(data.finalResponseAuthority); const fallback = typeof data.finalResponseFallback === "boolean" ? data.finalResponseFallback : null; const needsContinuation = typeof data.needsContinuation === "boolean" ? data.needsContinuation : null; if (providerTerminalFailure !== null) lines.push(`ProviderTerminalFailure: ${String(providerTerminalFailure)}`); if (authority !== null) lines.push(`Authority: ${authority}`); if (fallback !== null) lines.push(`Fallback: ${String(fallback)}`); if (needsContinuation !== null) lines.push(`NeedsContinuation: ${String(needsContinuation)}`); const finalRecord = record(data.finalResponse); const final = stringOrNull(data.finalResponse) ?? stringOrNull(finalRecord.content) ?? stringOrNull(finalRecord.text) ?? stringOrNull(finalRecord.message) ?? stringOrNull(data.output) ?? stringOrNull(data.summary) ?? stringOrNull(data.result); if (final !== null) lines.push("", truncateMultiline(final, options.fullText ? 8000 : 1600)); else { const failure = renderFailureLines(data); lines.push("", ...(failure.length > 0 ? failure : ["No final response is available yet."])); } return renderedCliResult(raw.ok !== false, command, lines.join("\n")); } function renderMutationSummary(command: string, raw: Record, options: AgentRunResourceOptions, headline: string, overrideNextLines?: string[]): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); if (options.output === "json" || options.output === "yaml") return renderMachine(command, raw, options.output, raw.ok !== false); const data = record(innerData(raw)); const id = stringOrNull(data.id) ?? stringOrNull(data.taskId) ?? stringOrNull(data.sessionId); const lines = [ headline, `OK: ${String(raw.ok !== false)}`, ]; if (id !== null) lines.push(`Name: ${id}`); const decision = stringOrNull(data.decision); const internalCommandType = stringOrNull(data.internalCommandType); if (data.dryRun !== undefined) lines.push(`DryRun: ${String(data.dryRun)}`); if (data.mutation !== undefined) lines.push(`Mutation: ${String(data.mutation)}`); if (decision !== null) lines.push(`Decision: ${decision}`); if (internalCommandType !== null) lines.push(`InternalCommandType: ${internalCommandType}`); const next = record(raw.next ?? data.next); const nextLines = (overrideNextLines ?? Object.values(next).map(String)).filter((line) => line.length > 0).slice(0, 5); if (nextLines.length > 0) lines.push("", "Next:", ...nextLines.map((line) => ` ${line}`)); return renderedCliResult(raw.ok !== false, command, lines.join("\n")); } function rerunWithoutDryRun(command: string): string { return `bun scripts/cli.ts ${command.replace(/\s+--dry-run\b/gu, "").trim()}`; } function renderMachine(command: string, value: unknown, mode: "json" | "yaml", ok = true): RenderedCliResult { const text = mode === "json" ? `${JSON.stringify(value, null, 2)}\n` : `${Bun.YAML.stringify(value)}\n`; return renderedCliResult(ok, command, text, mode === "json" ? "application/json" : "application/yaml"); } function parseResourceKind(raw: string | undefined): AgentRunResourceKind | null { if (raw === undefined) return null; const value = raw.toLowerCase().replace(/s$/u, ""); if (value === "task" || value === "qt" || value === "queue" || value === "queuetask") return "task"; if (value === "run") return "run"; if (value === "command" || value === "cmd") return "command"; if (value === "runnerjob" || value === "runner-job" || value === "rjob" || value === "job") return "runnerjob"; if (value === "session" || value === "ses") return "session"; if (value === "aipodspec" || value === "aipod-spec" || value === "aipod-specs" || value === "aps") return "aipodspec"; return null; } function parseResourceRef(action: string | undefined, args: string[], defaultKind: AgentRunResourceKind | null = null): AgentRunResourceRef { const raw = action ?? ""; if (raw.includes("/")) { const [kindRaw, ...nameParts] = raw.split("/"); const kind = parseResourceKind(kindRaw); const name = nameParts.join("/"); if (kind === null || name.length === 0) throw new Error(`invalid resource reference: ${raw}`); return { kind, name }; } const kind = parseResourceKind(raw); if (kind !== null) { const name = args.find((arg) => !arg.startsWith("-") && arg !== raw); if (name === undefined) throw new Error(`${kind} reference requires a name`); return { kind, name }; } if (defaultKind !== null && raw.length > 0) return { kind: defaultKind, name: raw }; throw new Error("resource reference required, for example task/ or session/"); } function stripLeadingResource(args: string[], resourceName: string): string[] { let skipped = false; const result: string[] = []; for (const arg of args) { if (!skipped && (arg === resourceName || arg.endsWith(`/${resourceName}`) || parseResourceKind(arg) !== null)) { skipped = true; continue; } result.push(arg); } return result; } function requiredContext(context: string, flag: string): never { throw new Error(`${context} requires ${flag}`); } function innerData(raw: Record): unknown { const data = raw.data; if (typeof data === "object" && data !== null && "data" in data && Object.keys(raw).length <= 3) return record(data).data; return data ?? raw; } function normalizeTaskItems(data: unknown, options: AgentRunResourceOptions): Record[] { const items = arrayRecords(record(data).items ?? data); return items .filter((item) => options.queue === null || String(item.queue ?? "") === options.queue) .filter((item) => options.state === null || options.state.split(",").includes(String(item.state ?? ""))) .filter((item) => !options.unread || item.unread === true || item.attention === true) .map((item) => { const attempt = record(item.latestAttempt); return { kind: "task", name: item.id, state: item.state, queue: item.queue, lane: item.lane, profile: item.backendProfile, run: attempt.runId, command: attempt.commandId, runnerjob: attempt.runnerJobId, session: attempt.sessionId ?? stringOrNull(record(item.sessionRef).sessionId), age: relativeAge(stringOrNull(item.updatedAt) ?? stringOrNull(item.createdAt)), title: item.title, updatedAt: item.updatedAt, attention: item.unread === true || item.attention === true ? "unread" : "", }; }); } function taskListState(options: AgentRunResourceOptions): string { const requested = options.state?.split(",").map((item) => item.trim()).filter((item) => item.length > 0); return requested?.[0] ?? "running"; } function normalizeSessionItems(data: unknown): Record[] { const items = arrayRecords(record(data).items ?? data); return items.map((item) => ({ kind: "session", name: item.id ?? item.sessionId ?? item.name, state: item.state ?? item.status ?? item.phase ?? "-", queue: item.queue ?? "", lane: item.lane ?? "", profile: item.backendProfile ?? item.profile ?? "", run: item.runId ?? "", command: item.commandId ?? "", runnerjob: item.runnerJobId ?? "", session: item.id ?? item.sessionId ?? item.name, age: relativeAge(stringOrNull(item.updatedAt) ?? stringOrNull(item.createdAt)), title: item.title ?? "", })); } function normalizeAipodSpecItems(data: unknown): Record[] { const items = arrayRecords(record(data).items ?? data); return items.map((item) => ({ kind: "aipodspec", name: item.name ?? item.id ?? item.metadata?.toString(), state: item.state ?? "available", queue: "", lane: item.lane ?? "", profile: item.backendProfile ?? item.profile ?? "", title: item.description ?? item.title ?? "", })); } function normalizeAttemptResources(task: Record, kind: "run" | "command" | "runnerjob"): Record[] { const attempt = record(task.latestAttempt); const key = kind === "run" ? "runId" : kind === "command" ? "commandId" : "runnerJobId"; const name = stringOrNull(attempt[key]); if (name === null) return []; return [{ kind, name, state: attempt.state ?? task.state, queue: task.queue, lane: task.lane, run: attempt.runId, command: attempt.commandId, runnerjob: attempt.runnerJobId, session: attempt.sessionId, title: task.title, }]; } function normalizeSingleCommand(data: unknown): Record[] { const item = record(data); return [{ kind: "command", name: item.id ?? item.commandId ?? "", state: item.state ?? item.status ?? "", run: item.runId ?? "", command: item.id ?? item.commandId ?? "", session: item.sessionId ?? "", title: item.summary ?? item.commandType ?? "", }]; } function normalizeRunnerJobItems(data: unknown): Record[] { const items = arrayRecords(record(data).items ?? data); return items.map((item) => ({ kind: "runnerjob", name: item.id ?? item.runnerJobId ?? item.name ?? item.jobName, state: item.phase ?? item.state ?? item.status ?? "", run: item.runId ?? "", command: item.commandId ?? "", runnerjob: item.id ?? item.runnerJobId ?? item.name ?? item.jobName, session: item.sessionId ?? "", title: item.image ?? item.namespace ?? "", })); } function normalizeEventItems(data: unknown): Record[] { return arrayRecords(record(data).items ?? data).map((item) => { const payload = record(item.payload); return { seq: item.seq, type: item.type, status: agentRunEventStatus(item, payload), phase: item.phase ?? payload.phase, commandId: agentRunEventCommandId(item, payload), summary: agentRunEventSummary(item, payload), }; }); } function normalizeLogItems(data: unknown): Record[] { return arrayRecords(record(data).items ?? data).map((item) => { const payload = record(item.payload); return { seq: item.seq, type: item.type ?? item.role ?? "output", status: agentRunEventStatus(item, payload), commandId: agentRunEventCommandId(item, payload), summary: agentRunEventSummary(item, payload), }; }); } function agentRunEventCommandId(item: Record, payload: Record): string { return stringOrNull(item.commandId) ?? stringOrNull(payload.commandId) ?? stringOrNull(record(payload.summary).commandId) ?? "-"; } function agentRunEventStatus(item: Record, payload: Record): string { return stringOrNull(item.status) ?? stringOrNull(item.phase) ?? stringOrNull(payload.status) ?? stringOrNull(payload.phase) ?? stringOrNull(payload.failureKind) ?? (item.type === "error" ? "error" : ""); } function agentRunEventSummary(item: Record, payload: Record): string { const type = stringOrNull(item.type); const parts: string[] = []; const push = (value: unknown): void => { const text = eventText(value); if (text !== null && !parts.includes(text)) parts.push(text); }; if (type === "error") { push(payload.failureKind); const error = record(payload.error); push(error.message); push(error.additionalDetails); if (payload.willRetry === true) push("willRetry=true"); } if (type === "backend_status") { push(payload.phase); push(formatCountField("tools", record(payload.tools).count)); push(formatCountField("bundles", record(payload.bundles).count)); push(formatCountField("promptRefs", record(payload.promptRefs).count)); push(payload.jobName); push(payload.namespace); } if (type === "tool_call") { push(payload.summary); push(payload.toolName); push(payload.command); const exitCode = payload.exitCode; if (exitCode !== undefined && exitCode !== null) push(`exit=${String(exitCode)}`); push(payload.outputSummary); } if (type === "command_output") { push(commandOutputSummary(payload)); } if (type === "assistant_message") { push(payload.summary); push(payload.text); } push(item.summary); push(item.outputSummary); push(item.text); if (type !== "command_output") { push(payload.summary); push(payload.outputSummary); push(payload.text); } push(payload.message); push(payload.reason); return parts.join("; "); } function commandOutputSummary(payload: Record): string | null { const raw = commandOutputText(payload); if (raw === null) return null; const structured = parseJsonRecordFromText(raw); if (structured !== null) { const summary = summarizeStructuredCliOutput(structured); if (summary !== null) return summary; } return summarizePartialJsonCommandOutput(raw) ?? summarizeTextCommandOutput(raw); } function commandOutputText(payload: Record): string | null { const summary = record(payload.summary); const summaryText = stringOrNull(summary.text); const payloadText = stringOrNull(payload.text); if (summary.textTruncated === true && payloadText !== null) return payloadText; return summaryText ?? payloadText ?? stringOrNull(payload.outputSummary) ?? eventText(payload.summary); } function parseJsonRecordFromText(raw: string): Record | null { const trimmed = raw.trim(); const direct = tryParseJsonRecord(trimmed); if (direct !== null) return direct; const firstBrace = trimmed.indexOf("{"); const lastBrace = trimmed.lastIndexOf("}"); if (firstBrace >= 0 && lastBrace > firstBrace) { const sliced = tryParseJsonRecord(trimmed.slice(firstBrace, lastBrace + 1)); if (sliced !== null) return sliced; } const line = trimmed .split(/\r?\n/u) .map((item) => item.trim()) .reverse() .find((item) => item.startsWith("{") && item.endsWith("}")); return line === undefined ? null : tryParseJsonRecord(line); } function tryParseJsonRecord(raw: string): Record | null { try { const parsed = JSON.parse(raw) as unknown; return isRecord(parsed) ? parsed : null; } catch { return null; } } function summarizeStructuredCliOutput(value: Record): string | null { const parts: string[] = []; const action = firstPathText(value, ["action", "operation", "command", "kind"]); const status = firstPathText(value, ["status", "state", "phase", "body.status"]); const failure = firstPathText(value, ["failureKind", "error.failureKind", "body.failureKind"]); const message = firstPathText(value, ["error.message", "error.additionalDetails", "message", "reason", "body.message", "body.error", "result.message"]); const primary = [ action, status, labeledSummaryValue("http", firstPathText(value, ["httpStatus", "request.httpStatus"])), ].filter((item): item is string => item !== null); if (primary.length > 0) pushSummaryPart(parts, primary.join(" ")); pushLabeledSummaryPart(parts, "failure", failure); if (failure !== null || isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message); pushLabeledSummaryPart(parts, "trace", firstPathText(value, ["traceId", "lastTraceId", "trace.traceId", "trace.id", "body.traceId", "body.trace.id", "providerTrace.traceId", "runnerTrace.traceId"]), true); pushLabeledSummaryPart(parts, "session", firstPathText(value, ["sessionId", "session.sessionId", "body.sessionId", "workspace.selectedAgentSessionId", "workspace.selectedConversation.sessionId"]), true); pushLabeledSummaryPart(parts, "conversation", firstPathText(value, ["conversationId", "session.conversationId", "body.conversationId", "workspace.selectedConversationId", "workspace.selectedConversation.conversationId"]), true); pushLabeledSummaryPart(parts, "thread", firstPathText(value, ["threadId", "session.threadId", "body.threadId", "workspace.selectedConversation.threadId"]), true); pushLabeledSummaryPart(parts, "provider", firstPathText(value, ["providerProfile", "backendProfile", "profile", "session.providerProfile", "workspace.providerProfile"])); pushLabeledSummaryPart(parts, "route", commandOutputRouteSummary(value)); pushLabeledSummaryPart(parts, "pipeline", firstPathText(value, ["pipelineRun", "pipelineRunName", "pipelineRun.name", "pipeline.runName", "pipeline.name"]), true); pushLabeledSummaryPart(parts, "run", firstPathText(value, ["runId", "agentRun.runId"]), true); pushLabeledSummaryPart(parts, "cmd", firstPathText(value, ["commandId", "agentRun.commandId", "providerTrace.commandId", "runnerTrace.commandId"]), true); pushLabeledSummaryPart(parts, "job", firstPathText(value, ["jobName", "agentRun.jobName", "runnerJobId"]), true); pushLabeledSummaryPart(parts, "lane", firstPathText(value, ["runtimeEndpoint.lane", "lane"])); pushLabeledSummaryPart(parts, "ns", firstPathText(value, ["runtimeEndpoint.namespace", "namespace"])); pushLabeledSummaryPart(parts, "actor", firstPathText(value, ["actor.username", "body.actor.username", "actor.displayName", "body.actor.displayName"])); pushLabeledSummaryPart(parts, "auth", firstPathText(value, ["authMethod", "auth.authMethod", "body.authMethod"])); pushLabeledSummaryPart(parts, "reply", firstPathText(value, ["reply.content", "result.reply.content", "body.reply.content", "answer", "content", "body.content"])); if (failure === null && !isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message); const itemsCount = arrayRecords(value.items).length; if (itemsCount > 0) pushSummaryPart(parts, `items=${itemsCount}`); const eventsCount = arrayRecords(value.events).length; if (eventsCount > 0) pushSummaryPart(parts, `events=${eventsCount}`); if (value.sessionUsable === false) pushSummaryPart(parts, "sessionUsable=false"); if (record(value.summary).textTruncated === true || value.textTruncated === true || value.outputTruncated === true) pushSummaryPart(parts, "truncated=true"); return parts.length > 0 ? parts.slice(0, 10).join("; ") : null; } function commandOutputRouteSummary(value: Record): string | null { const route = record(value.route); const request = record(value.request); const method = stringOrNull(route.method) ?? stringOrNull(request.method); const path = stringOrNull(route.path) ?? stringOrNull(request.path); if (method !== null && path !== null) return `${method} ${path}`; return stringOrNull(request.url) ?? stringOrNull(value.baseUrl); } function firstPathText(value: Record, paths: string[]): string | null { for (const path of paths) { const text = summaryScalar(pathValue(value, path)); if (text !== null) return text; } return null; } function pathValue(value: Record, path: string): unknown { let current: unknown = value; for (const key of path.split(".")) { if (!isRecord(current)) return undefined; current = current[key]; } return current; } function summaryScalar(value: unknown): string | null { if (typeof value === "string") { const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : null; } if (typeof value === "number" && Number.isFinite(value)) return String(value); if (typeof value === "boolean") return String(value); return null; } function isFailureLikeStatus(value: string | null): boolean { return value !== null && /(?:fail|error|timeout|cancel|interrupt|denied|rejected)/iu.test(value); } function pushLabeledSummaryPart(parts: string[], label: string, value: string | null, shorten = false): void { const text = labeledSummaryValue(label, value, shorten); if (text !== null) pushSummaryPart(parts, text); } function labeledSummaryValue(label: string, value: string | null, shorten = false): string | null { if (value === null) return null; const rendered = shorten ? shortId(value) : truncateOneLine(value, 80); return `${label}=${rendered}`; } function pushSummaryPart(parts: string[], value: string): void { const text = truncateOneLine(value, 160); if (text.length > 0 && !parts.includes(text)) parts.push(text); } function summarizeTextCommandOutput(raw: string): string | null { const lines = raw .split(/\r?\n/u) .map((line) => line.trim()) .filter((line) => line.length > 0 && !isLowSignalCommandOutputLine(line)); const signal = lines.find((line) => /(?:trace|session|conversation|thread|status|failure|error|route|http|pipeline|provider|commit|succeeded|failed|created)/iu.test(line)) ?? lines[0] ?? null; if (signal === null) return null; return truncateOneLine(cleanCommandOutputLine(signal), 220); } function summarizePartialJsonCommandOutput(raw: string): string | null { const fields = new Map(); for (const line of raw.split(/\r?\n/u)) { const field = jsonLineField(line); if (field === null || isLowSignalJsonField(field.key)) continue; if (!fields.has(field.key) && field.value.length > 0) fields.set(field.key, field.value); } const parts: string[] = []; const action = fields.get("action") ?? fields.get("operation") ?? fields.get("command") ?? fields.get("kind") ?? null; const status = fields.get("status") ?? fields.get("state") ?? fields.get("phase") ?? null; const http = fields.get("httpStatus") ?? null; const failure = fields.get("failureKind") ?? null; const message = fields.get("message") ?? fields.get("reason") ?? fields.get("error") ?? null; const primary = [action, status, labeledSummaryValue("http", http)].filter((item): item is string => item !== null); if (primary.length > 0) pushSummaryPart(parts, primary.join(" ")); pushLabeledSummaryPart(parts, "failure", failure); if (failure !== null || isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message); pushLabeledSummaryPart(parts, "trace", fields.get("traceId") ?? fields.get("lastTraceId") ?? null, true); pushLabeledSummaryPart(parts, "session", fields.get("sessionId") ?? fields.get("selectedAgentSessionId") ?? null, true); pushLabeledSummaryPart(parts, "conversation", fields.get("conversationId") ?? fields.get("selectedConversationId") ?? null, true); pushLabeledSummaryPart(parts, "thread", fields.get("threadId") ?? null, true); pushLabeledSummaryPart(parts, "provider", fields.get("providerProfile") ?? fields.get("backendProfile") ?? fields.get("profile") ?? null); pushLabeledSummaryPart(parts, "route", partialRouteSummary(fields)); pushLabeledSummaryPart(parts, "pipeline", fields.get("pipelineRun") ?? fields.get("pipelineRunName") ?? null, true); pushLabeledSummaryPart(parts, "run", fields.get("runId") ?? null, true); pushLabeledSummaryPart(parts, "cmd", fields.get("commandId") ?? null, true); pushLabeledSummaryPart(parts, "job", fields.get("jobName") ?? fields.get("runnerJobId") ?? null, true); pushLabeledSummaryPart(parts, "lane", fields.get("lane") ?? null); pushLabeledSummaryPart(parts, "ns", fields.get("namespace") ?? null); pushLabeledSummaryPart(parts, "actor", fields.get("username") ?? fields.get("displayName") ?? null); pushLabeledSummaryPart(parts, "auth", fields.get("authMethod") ?? null); if (failure === null && !isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message); return parts.length > 0 ? parts.slice(0, 10).join("; ") : null; } function partialRouteSummary(fields: Map): string | null { const method = fields.get("method") ?? null; const path = fields.get("path") ?? null; if (method !== null && path !== null) return `${method} ${path}`; return fields.get("url") ?? fields.get("baseUrl") ?? null; } function jsonLineField(line: string): { key: string; value: string } | null { const match = line.trim().replace(/,$/u, "").match(/^"([^"]+)":\s*(.+)$/u); if (match === null) return null; return { key: match[1], value: cleanJsonScalar(match[2]) }; } function cleanJsonScalar(raw: string): string { const trimmed = raw.trim(); if (trimmed === "null" || trimmed === "{" || trimmed === "[" || trimmed === "}" || trimmed === "]") return ""; if (trimmed.startsWith('"') && trimmed.endsWith('"')) { try { const parsed = JSON.parse(trimmed) as unknown; return typeof parsed === "string" ? parsed.trim() : String(parsed); } catch { return trimmed.slice(1, -1).trim(); } } return trimmed; } function cleanCommandOutputLine(line: string): string { const trimmed = line.replace(/,$/u, "").trim(); const jsonField = trimmed.match(/^"([^"]+)":\s*(.+)$/u); if (jsonField === null) return trimmed; const key = jsonField[1]; const value = jsonField[2].replace(/^"|"$/gu, "").trim(); return `${key}=${value}`; } function isLowSignalCommandOutputLine(line: string): boolean { const trimmed = line.replace(/,$/u, "").trim(); if (trimmed === "{" || trimmed === "}" || trimmed === "}," || trimmed === "];" || trimmed === "[") return true; const field = jsonLineField(trimmed); return field !== null && isLowSignalJsonField(field.key); } function isLowSignalJsonField(key: string): boolean { return key === "generatedAt" || key === "cli" || key === "version" || key === "valuesRedacted" || key === "secretMaterialStored"; } function eventText(value: unknown): string | null { const direct = stringOrNull(value); if (direct !== null && direct.trim().length > 0) return direct.trim(); const nested = record(value); return stringOrNull(nested.text) ?? stringOrNull(nested.message) ?? stringOrNull(nested.reason) ?? stringOrNull(nested.summary); } function formatCountField(label: string, value: unknown): string | null { if (typeof value === "number" && Number.isFinite(value)) return `${label}=${value}`; if (typeof value === "string" && value.length > 0) return `${label}=${value}`; return null; } function renderResourceTable(items: Record[], wide: boolean): string { const headers = wide ? ["NAME", "STATE", "QUEUE", "LANE", "PROFILE", "RUN", "CMD", "RJOB", "SESSION", "AGE", "TITLE"] : ["NAME", "STATE", "QUEUE", "LANE", "RUN", "CMD", "RJOB", "SESSION", "AGE", "ATTENTION"]; const rows = items.map((item) => { if (wide) { return [ resourceName(item), displayValue(item.state), displayValue(item.queue), displayValue(item.lane), displayValue(item.profile), shortId(stringOrDash(item.run)), shortId(stringOrDash(item.command)), shortId(stringOrDash(item.runnerjob)), shortId(stringOrDash(item.session)), displayValue(item.age), truncateOneLine(displayValue(item.title), 48), ]; } return [ resourceName(item), displayValue(item.state), displayValue(item.queue), displayValue(item.lane), shortId(stringOrDash(item.run)), shortId(stringOrDash(item.command)), shortId(stringOrDash(item.runnerjob)), shortId(stringOrDash(item.session)), displayValue(item.age), displayValue(item.attention), ]; }); return renderTable(headers, rows); } function renderTaskDescription(task: Record, options: AgentRunResourceOptions): string { const attempt = record(task.latestAttempt); const taskId = displayValue(task.id); const runId = stringOrNull(attempt.runId); const commandId = stringOrNull(attempt.commandId); const runnerJobId = stringOrNull(attempt.runnerJobId); const sessionId = stringOrNull(attempt.sessionId) ?? stringOrNull(record(task.sessionRef).sessionId); const lines = [ `Name: task/${taskId}`, `State: ${displayValue(task.state)} Attention: ${displayValue(task.unread === true ? "unread" : "")}`, `Queue: ${displayValue(task.queue)}`, `Lane: ${displayValue(task.lane)} Profile: ${displayValue(task.backendProfile)} Provider: ${displayValue(task.providerId)}`, `Title: ${displayValue(task.title)}`, "", "Latest Attempt:", ` Attempt: ${displayValue(attempt.attemptId)} State: ${displayValue(attempt.state)}`, ` Run: ${runId === null ? "-" : `run/${runId}`}`, ` Command: ${commandId === null ? "-" : `command/${commandId}`}`, ` Runner: ${runnerJobId === null ? "-" : `runnerjob/${runnerJobId}`}`, ` Session: ${sessionId === null ? "-" : `session/${sessionId}`}`, "", "Conditions:", ` Terminal: ${displayValue(task.state)}`, ` Failure: ${displayValue(task.failureKind ?? task.degradedReason ?? "none")}`, " Values: redacted", "", "Next:", ]; const next = [ runId === null ? null : `bun scripts/cli.ts agentrun events run/${runId} --after-seq 0 --limit 100`, sessionId === null ? null : `bun scripts/cli.ts agentrun logs session/${sessionId} --tail 100`, runId === null || commandId === null ? null : `bun scripts/cli.ts agentrun result run/${runId} --command ${commandId}`, `bun scripts/cli.ts agentrun ack task/${taskId}`, options.full ? null : `bun scripts/cli.ts agentrun describe task/${taskId} --full`, ].filter((item): item is string => item !== null).slice(0, 5); lines.push(...next.map((item) => ` ${item}`)); return lines.join("\n"); } function unwrapTaskDetail(data: Record): Record { const task = record(data.task); if (Object.keys(task).length > 0) return task; return data; } function renderGenericDescription(ref: AgentRunResourceRef, data: unknown): string { const value = record(data); const lines = [ `Name: ${ref.kind}/${ref.name}`, `State: ${displayValue(value.state ?? value.status ?? value.phase ?? value.executionState ?? value.terminalStatus ?? "-")}`, ]; const identity = renderResourceIdentityLines(ref, value); if (identity.length > 0) lines.push("", ...identity); const supervisor = renderSupervisorLines(ref, value); if (supervisor.length > 0) lines.push("", "Supervisor:", ...supervisor.map((line) => ` ${line}`)); const failure = renderFailureLines(value); if (failure.length > 0) lines.push("", ...failure); const next = renderResourceNextLines(ref, value); if (next.length > 0) lines.push("", "Next:", ...next.map((line) => ` ${line}`)); if (identity.length === 0 && failure.length === 0 && next.length === 0) { lines.push( "", "Summary:", JSON.stringify(pickCompact(value, ["id", "name", "runId", "commandId", "runnerJobId", "sessionId", "state", "status", "phase", "reason", "failureKind", "valuesPrinted"]), null, 2), ); } return lines.join("\n"); } function renderResourceIdentityLines(ref: AgentRunResourceRef, value: Record): string[] { const supervisor = record(value.supervisor); const fields: [string, unknown][] = [ ["Run", firstKnownString(value.runId, value.activeRunId, value.lastRunId, supervisor.runId, supervisor.activeRunId, supervisor.lastRunId)], ["Command", firstKnownString(value.commandId, value.activeCommandId, value.lastCommandId, supervisor.commandId, supervisor.activeCommandId, supervisor.lastCommandId)], ["RunnerJob", value.runnerJobId ?? (ref.kind === "runnerjob" ? value.id : null)], ["Session", firstKnownString(value.sessionId, supervisor.sessionId)], ["Namespace", value.namespace], ["Job", value.jobName], ["Image", value.image], ["LogPath", value.logPath], ["Started", value.startedAt], ["Finished", value.finishedAt], ]; if (ref.kind !== "runnerjob") { return fields .filter(([label]) => label === "Run" || label === "Command" || label === "RunnerJob" || label === "Session") .map(([label, item]) => formatResourceField(label, item)) .filter((line): line is string => line !== null); } return fields.map(([label, item]) => formatResourceField(label, item)).filter((line): line is string => line !== null); } function firstKnownString(...values: unknown[]): string | null { for (const value of values) { const text = stringOrNull(value); if (text !== null && text.length > 0) return text; } return null; } function formatResourceField(label: string, value: unknown): string | null { const text = stringOrNull(value); if (text === null || text.length === 0) return null; return `${label}: ${text}`; } function renderSupervisorLines(ref: AgentRunResourceRef, value: Record): string[] { if (ref.kind !== "session" && ref.kind !== "task") return []; const supervisor = record(value.supervisor); const liveness = record(value.liveness); const source = Object.keys(supervisor).length > 0 ? supervisor : liveness; if (Object.keys(source).length === 0) return []; const timeoutBudget = record(source.timeoutBudget); const terminalClassification = record(source.terminalClassification); const runId = firstKnownString(source.activeRunId, source.runId, source.lastRunId, value.activeRunId, value.runId, value.lastRunId); const commandId = firstKnownString(source.activeCommandId, source.commandId, source.lastCommandId, value.activeCommandId, value.commandId, value.lastCommandId); const lines: string[] = []; const executionState = firstKnownString(source.executionState, value.executionState); if (executionState !== null) lines.push(`Execution: ${executionState}`); if (typeof source.active === "boolean") lines.push(`Active: ${String(source.active)}`); if (runId !== null) lines.push(`Run: run/${runId}`); if (commandId !== null) lines.push(`Command: command/${commandId}`); const phase = firstKnownString(source.phase, liveness.phase); const lastEventAgeMs = typeof source.lastEventAgeMs === "number" ? source.lastEventAgeMs : typeof liveness.lastEventAgeMs === "number" ? liveness.lastEventAgeMs : null; if (phase !== null || lastEventAgeMs !== null) lines.push(`Liveness: ${displayValue(phase)}${lastEventAgeMs === null ? "" : ` ageMs=${String(lastEventAgeMs)}`}`); const timeoutKind = firstKnownString(timeoutBudget.timeoutKind); const timeoutState = firstKnownString(timeoutBudget.state); const remainingMs = typeof timeoutBudget.remainingMs === "number" ? timeoutBudget.remainingMs : null; if (timeoutKind !== null || timeoutState !== null || remainingMs !== null) lines.push(`Timeout: ${displayValue(timeoutKind)} / ${displayValue(timeoutState)}${remainingMs === null ? "" : ` remainingMs=${String(remainingMs)}`}`); const category = firstKnownString(terminalClassification.category); const reason = firstKnownString(terminalClassification.reason); if (category !== null || reason !== null) lines.push(`Classification: ${displayValue(category)}${reason === null ? "" : `; ${reason}`}`); return lines; } function renderFailureLines(value: Record): string[] { const terminalClassification = record(value.terminalClassification); const nestedTerminalClassification = Object.keys(terminalClassification).length > 0 ? terminalClassification : record(record(value.liveness).terminalClassification); const failureKind = stringOrNull(value.failureKind) ?? stringOrNull(nestedTerminalClassification.failureKind) ?? stringOrNull(value.degradedReason); const failureMessage = stringOrNull(value.failureMessage) ?? stringOrNull(nestedTerminalClassification.failureMessage) ?? stringOrNull(value.message) ?? stringOrNull(value.reason); const category = stringOrNull(nestedTerminalClassification.category); const timeoutKind = stringOrNull(nestedTerminalClassification.timeoutKind); const timeoutState = stringOrNull(nestedTerminalClassification.timeoutState); const providerInterruption = stringOrNull(nestedTerminalClassification.providerInterruption); const lastActivity = stringOrNull(nestedTerminalClassification.lastActivityKind); const lastActivitySeq = nestedTerminalClassification.lastActivitySeq; const lines: string[] = []; if (failureKind !== null) lines.push(`Failure: ${failureKind}`); if (failureMessage !== null) lines.push(`Message: ${failureMessage}`); if (category !== null) lines.push(`Category: ${category}`); if (timeoutKind !== null || timeoutState !== null) lines.push(`Timeout: ${displayValue(timeoutKind)} / ${displayValue(timeoutState)}`); if (providerInterruption !== null) lines.push(`Provider: ${providerInterruption}`); if (lastActivity !== null) lines.push(`LastActivity: ${lastActivity}${lastActivitySeq === undefined ? "" : ` seq=${String(lastActivitySeq)}`}`); return lines; } function renderResourceNextLines(ref: AgentRunResourceRef, value: Record): string[] { const supervisor = record(value.supervisor); const runId = firstKnownString(value.runId, value.activeRunId, value.lastRunId, supervisor.runId, supervisor.activeRunId, supervisor.lastRunId); const commandId = firstKnownString(value.commandId, value.activeCommandId, value.lastCommandId, supervisor.commandId, supervisor.activeCommandId, supervisor.lastCommandId); const runnerJobId = stringOrNull(value.runnerJobId) ?? (ref.kind === "runnerjob" ? ref.name : null); const sessionId = firstKnownString(value.sessionId, supervisor.sessionId); const lines = [ runId === null ? null : `bun scripts/cli.ts agentrun events run/${runId} --after-seq 0 --limit 100`, sessionId === null ? null : `bun scripts/cli.ts agentrun logs session/${sessionId} --tail 100`, runId === null || commandId === null ? null : `bun scripts/cli.ts agentrun result run/${runId} --command ${commandId}`, runId === null || commandId === null ? null : `bun scripts/cli.ts agentrun describe command/${commandId} --run ${runId} --full`, runId === null || runnerJobId === null ? null : `bun scripts/cli.ts agentrun describe runnerjob/${runnerJobId} --run ${runId} --full`, ref.kind !== "session" || sessionId === null ? null : `bun scripts/cli.ts agentrun send session/${sessionId} --prompt-stdin`, ]; const unique: string[] = []; for (const line of lines) { if (line !== null && !unique.includes(line)) unique.push(line); } return unique.slice(0, 5); } function parseTaskManifest(raw: string, source: string): Record { const parsed = source.endsWith(".json") ? JSON.parse(raw) as unknown : Bun.YAML.parse(raw) as unknown; const input = record(parsed); const kind = stringOrNull(input.kind); if (kind !== null && kind.toLowerCase() !== "task") throw new Error("apply currently supports kind: Task only"); const spec = record(input.spec); const payload = Object.keys(spec).length > 0 ? spec : input; if (!isRecord(payload.payload) && !isRecord(payload.executionPolicy) && stringOrNull(payload.title) === null) { throw new Error("task manifest must contain spec with an AgentRun task payload"); } return payload; } function nextPagedResourceCommand(command: string, nextAfterSeq: string, limit: number): string { const parts = command .replace(/\s+--after-seq\s+\S+/gu, "") .replace(/\s+--tail(?:\s+\S+)?/gu, "") .replace(/\s+--limit\s+\S+/gu, "") .trim(); return `${parts} --after-seq ${nextAfterSeq} --limit ${limit}`; } function agentRunExplain(kindRaw: string): string { if (kindRaw === "session-policy" || kindRaw === "provider-policy") return renderAgentRunSessionPolicyExplanation(); const kind = parseResourceKind(kindRaw); if (kind === "task") { return [ "KIND: task", "FIELDS:", " metadata.name: optional client name/idempotency hint", " spec.tenantId, spec.projectId, spec.queue, spec.title", " spec.payload.prompt or spec.payload.", " spec.executionPolicy, spec.resourceBundleRef, spec.workspaceRef", "", "Apply example:", " kind: Task", " spec:", " tenantId: unidesk", " projectId: pikasTech/unidesk", " queue: commander", " title: Example", " payload:", " prompt: Do the work", ].join("\n"); } return `KIND: ${kindRaw}\nUse get/describe with -o json to inspect the current stable UniDesk resource schema.`; } function arrayRecords(value: unknown): Record[] { if (Array.isArray(value)) return value.map(record).filter((item) => Object.keys(item).length > 0); if (typeof value === "object" && value !== null && Array.isArray((value as { items?: unknown }).items)) return arrayRecords((value as { items: unknown }).items); return []; } function renderTable(headers: string[], rows: string[][]): string { const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length))); const line = (values: string[]): string => values.map((value, index) => value.padEnd(widths[index] ?? value.length)).join(" ").trimEnd(); return [line(headers), ...rows.map(line)].join("\n"); } function resourceName(item: Record): string { const kind = String(item.kind ?? "resource"); return `${kind}/${shortId(displayValue(item.name))}`; } function shortId(value: string): string { if (value === "-" || value.length <= 18) return value; const known = ["qt_", "run_", "cmd_", "rjob_"]; const prefix = known.find((item) => value.startsWith(item)); if (prefix !== undefined) return `${prefix}${value.slice(prefix.length, prefix.length + 8)}...`; if (/^\w+-\d+/u.test(value)) return value.length > 28 ? `${value.slice(0, 25)}...` : value; return `${value.slice(0, 14)}...`; } function displayValue(value: unknown): string { if (value === undefined || value === null || value === "") return "-"; return String(value); } function stringOrDash(value: unknown): string { return displayValue(value); } function truncateOneLine(value: string, maxChars: number): string { const oneLine = value.replace(/\s+/gu, " ").trim(); return oneLine.length <= maxChars ? oneLine : `${oneLine.slice(0, Math.max(0, maxChars - 3))}...`; } function truncateMultiline(value: string, maxChars: number): string { return value.length <= maxChars ? value : `${value.slice(0, maxChars)}\n...[truncated ${value.length - maxChars} chars; rerun with --full-text or --full]`; } function relativeAge(iso: string | null): string { if (iso === null) return "-"; const ms = Date.now() - Date.parse(iso); if (!Number.isFinite(ms) || ms < 0) return "-"; const minutes = Math.floor(ms / 60000); if (minutes < 60) return `${minutes}m`; const hours = Math.floor(minutes / 60); if (hours < 48) return `${hours}h`; return `${Math.floor(hours / 24)}d`; } function pickCompact(value: Record, keys: string[]): Record { const picked: Record = {}; for (const key of keys) { if (value[key] !== undefined) picked[key] = value[key]; } return picked; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } interface TriggerOptions { confirm: boolean; dryRun: boolean; node: string | null; lane: string | null; wait: boolean; } interface ConfirmOptions { confirm: boolean; dryRun: boolean; } interface SecretSyncOptions extends ConfirmOptions { node: string | null; lane: string | null; } interface LaneConfirmOptions extends ConfirmOptions { node: string | null; lane: string | null; } interface RefreshOptions extends ConfirmOptions { node: string | null; lane: string | null; } interface GitMirrorStatusOptions extends DisclosureOptions { node: string | null; lane: string | null; } interface GitMirrorOptions extends ConfirmOptions { node: string | null; lane: string | null; timeoutSeconds: number; wait: boolean; } interface CleanupRunnersOptions extends ConfirmOptions { node: string | null; lane: string | null; timeoutSeconds: number; forceActive: boolean; } interface CleanupRunsOptions extends ConfirmOptions { node: string | null; lane: string | null; minAgeMinutes: number; limit: number; timeoutSeconds: number; } interface CleanupReleasedPvOptions extends ConfirmOptions { node: string | null; lane: string | null; limit: number; timeoutSeconds: number; } interface DisclosureOptions { full: boolean; raw: boolean; } interface StatusOptions extends DisclosureOptions { node: string | null; lane: string | null; sourceCommit: string | null; pipelineRun: string | null; targetMode: "latest-source-head" | "source-commit" | "pipeline-run"; } interface TimedValue { value: T; elapsedMs: number; } function parseDisclosureOptions(args: string[]): DisclosureOptions { for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--node" || arg === "--lane") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); index += 1; continue; } if (arg !== "--full" && arg !== "--raw") throw new Error(`unsupported status option: ${arg}`); } const raw = args.includes("--raw"); return { full: raw || args.includes("--full"), raw }; } function parseGitMirrorStatusOptions(args: string[]): GitMirrorStatusOptions { validateOptions(args, new Set(["--full", "--raw"]), new Set(["--node", "--lane"])); const raw = args.includes("--raw"); return { full: raw || args.includes("--full"), raw, node: optionValue(args, "--node") ?? null, lane: optionValue(args, "--lane") ?? null, }; } function parseStatusOptions(args: string[]): StatusOptions { let node: string | null = null; let lane: string | null = null; let sourceCommit: string | null = null; let pipelineRun: string | null = null; let full = false; let raw = false; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--full") { full = true; continue; } if (arg === "--raw") { raw = true; full = true; continue; } if (arg === "--node") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); node = value; index += 1; continue; } if (arg === "--lane") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); lane = value; index += 1; continue; } if (arg === "--source-commit") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--source-commit requires a value"); if (!isGitSha(value)) throw new Error("--source-commit must be a full 40-character git SHA"); sourceCommit = value; index += 1; continue; } if (arg === "--pipeline-run") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--pipeline-run requires a value"); if (!isAgentRunPipelineRunName(value)) throw new Error("--pipeline-run must be an agentrun-vNN-ci-<12+ hex> PipelineRun name"); pipelineRun = value; index += 1; continue; } throw new Error(`unsupported status option: ${arg}`); } if (sourceCommit !== null && pipelineRun !== null) throw new Error("control-plane status accepts only one of --source-commit or --pipeline-run"); return { full, raw, node, lane, sourceCommit, pipelineRun, targetMode: pipelineRun !== null ? "pipeline-run" : sourceCommit !== null ? "source-commit" : "latest-source-head", }; } function parseTriggerOptions(args: string[]): TriggerOptions { const base = parseConfirmOptions(args); let node: string | null = null; let lane: string | null = null; let wait = false; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--confirm" || arg === "--dry-run") continue; if (arg === "--wait") { wait = true; continue; } if (arg === "--node") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); node = value; index += 1; continue; } if (arg === "--lane") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); lane = value; index += 1; continue; } throw new Error(`unsupported trigger-current option: ${arg}`); } return { ...base, node, lane, wait }; } function parseSecretSyncOptions(args: string[]): SecretSyncOptions { const base = parseConfirmOptions(args); let node: string | null = null; let lane: string | null = null; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--confirm" || arg === "--dry-run") continue; if (arg === "--node") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); node = value; index += 1; continue; } if (arg === "--lane") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); lane = value; index += 1; continue; } throw new Error(`unsupported secret-sync option: ${arg}`); } return { ...base, node, lane }; } function parseLaneConfirmOptions(args: string[]): LaneConfirmOptions { const base = parseConfirmOptions(args); let node: string | null = null; let lane: string | null = null; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--confirm" || arg === "--dry-run") continue; if (arg === "--node") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); node = value; index += 1; continue; } if (arg === "--lane") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); lane = value; index += 1; continue; } throw new Error(`unsupported control-plane option: ${arg}`); } if (node === null && lane === null) throw new Error("control-plane apply requires --node and --lane"); return { ...base, node, lane }; } function parseRefreshOptions(args: string[]): RefreshOptions { const base = parseConfirmOptions(args); let node: string | null = null; let lane: string | null = null; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--confirm" || arg === "--dry-run") continue; if (arg === "--node") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); node = value; index += 1; continue; } if (arg === "--lane") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); lane = value; index += 1; continue; } throw new Error(`unsupported refresh option: ${arg}`); } return { ...base, node, lane }; } function parseConfirmOptions(args: string[]): ConfirmOptions { if (args.includes("--confirm") && args.includes("--dry-run")) throw new Error("accepts only one of --confirm or --dry-run"); return { confirm: args.includes("--confirm"), dryRun: args.includes("--dry-run") || !args.includes("--confirm"), }; } function parseGitMirrorOptions(args: string[]): GitMirrorOptions { validateOptions(args, new Set(["--confirm", "--dry-run", "--wait"]), new Set(["--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); const timeoutIndex = args.indexOf("--timeout-seconds"); const timeoutSeconds = timeoutIndex >= 0 ? Number(args[timeoutIndex + 1]) : 300; if (!Number.isFinite(timeoutSeconds) || timeoutSeconds < 30) throw new Error("--timeout-seconds must be a number >= 30"); return { ...base, node: optionValue(args, "--node") ?? null, lane: optionValue(args, "--lane") ?? null, timeoutSeconds, wait: args.includes("--wait"), }; } function parseCleanupRunnersOptions(args: string[]): CleanupRunnersOptions { validateOptions(args, new Set(["--confirm", "--dry-run", "--force-active"]), new Set(["--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); return { ...base, node: optionValue(args, "--node") ?? null, lane: optionValue(args, "--lane") ?? null, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600), forceActive: args.includes("--force-active"), }; } function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions { validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); return { ...base, node: optionValue(args, "--node") ?? null, lane: optionValue(args, "--lane") ?? null, minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(args, "--limit", 20, 500), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600), }; } function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPvOptions { validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); return { ...base, node: optionValue(args, "--node") ?? null, lane: optionValue(args, "--lane") ?? null, limit: positiveIntegerOption(args, "--limit", 20, 500), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), }; } function validateOptions(args: string[], booleanOptions: Set, valueOptions: Set): void { for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (booleanOptions.has(arg)) continue; if (valueOptions.has(arg)) { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); index += 1; continue; } throw new Error(`unsupported option: ${arg}`); } } function optionValue(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index === -1) return undefined; const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const raw = optionValue(args, name); if (raw === undefined) return defaultValue; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return Math.min(value, maxValue); } async function controlPlanePlan(_config: UniDeskConfig, options: StatusOptions): Promise> { const target = resolveAgentRunLaneTarget(options); const spec = target.spec; return { ok: true, command: "agentrun control-plane plan", mode: "yaml-declared-node-lane", configPath: target.configPath, target: agentRunLaneSummary(spec), plannedChecks: [ "source-branch-exists", "source-worktree-exists-and-clean", "git-mirror-services-ready", "ci-namespace-pipeline-serviceaccount", "argo-application-alignment", "runtime-namespace-manager-service", "database-secretref-present", "local-postgres-absent-when-external", ], deploymentBoundary: { mutation: false, note: "plan/status are read-only. Long writes must use controlled AgentRun/Platform DB commands and this YAML target.", }, next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, controlPlaneApply: `bun scripts/cli.ts agentrun control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --dry-run`, }, valuesPrinted: false, }; } async function controlPlaneApply(config: UniDeskConfig, options: LaneConfirmOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const objects = renderAgentRunControlPlaneManifests(spec); const manifestYaml = `${objects.map((object) => Bun.YAML.stringify(object).trim()).join("\n---\n")}\n`; const objectRefs = objects.map((object) => manifestObjectRef(object)); const plan = { node: spec.nodeId, kubeRoute: spec.nodeKubeRoute, lane: spec.lane, version: spec.version, objectCount: objects.length, objects: objectRefs, manifestBytes: Buffer.byteLength(manifestYaml, "utf8"), manifestDigest: renderedObjectsDigest(objects), fieldManager: "unidesk-agentrun-control-plane", valuesPrinted: false, }; if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane apply", mode: "dry-run", mutation: false, configPath, target: agentRunLaneSummary(spec), plan, next: { confirm: `bun scripts/cli.ts agentrun control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } const applied = await capture(config, spec.nodeKubeRoute, ["sh", "--", applyYamlScript(manifestYaml, "unidesk-agentrun-control-plane", false)]); const payload = captureJsonPayload(applied); return { ok: applied.exitCode === 0 && payload.ok !== false, command: "agentrun control-plane apply", mode: "confirmed-apply", mutation: true, configPath, target: agentRunLaneSummary(spec), plan, result: payload, capture: compactCapture(applied, { full: applied.exitCode !== 0, stdoutTailChars: 5000, stderrTailChars: 5000 }), next: { secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } async function status(config: UniDeskConfig, options: StatusOptions): Promise> { return await statusYamlLane(config, options, resolveAgentRunLaneTarget(options)); } async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { const spec = target.spec; const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)])); const sourcePayload = captureJsonPayload(sourceProbe.value); const branchTipCommit = stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead); const initialSourceCommit = options.sourceCommit ?? stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead); const pipelineRunName = options.pipelineRun ?? (initialSourceCommit ? agentRunPipelineRunName(spec, initialSourceCommit) : null); const [runtimeProbe, mirrorProbe] = await Promise.all([ timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])), timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)])), ]); const runtimePayload = captureJsonPayload(runtimeProbe.value); const mirrorPayload = captureJsonPayload(mirrorProbe.value); const pipeline = record(runtimePayload.pipeline); const pipelineRunStatus = record(runtimePayload.pipelineRun); const argo = record(runtimePayload.argo); const manager = record(runtimePayload.manager); const database = record(runtimePayload.database); const secrets = record(runtimePayload.secrets); const localPostgres = record(runtimePayload.localPostgres); const expectedGitopsRevision = stringOrNull(mirrorPayload.gitopsCommit); const mirrorSourceCommit = stringOrNull(mirrorPayload.sourceCommit); const pipelineRunSourceCommit = stringOrNull(pipelineRunStatus.sourceCommit); const sourceCommit = options.sourceCommit ?? (options.pipelineRun !== null ? pipelineRunSourceCommit ?? initialSourceCommit : initialSourceCommit); const managerSourceMatchesExpected = Boolean(sourceCommit && manager.sourceCommit === sourceCommit); const argoSyncedToGitops = Boolean(expectedGitopsRevision && argo.revision === expectedGitopsRevision); const pipelineSucceeded = pipelineRunStatus.status === "True"; const sourceWorktreeDetached = sourcePayload.branch === "HEAD"; const targetMode = options.pipelineRun !== null ? "pipeline-run" : options.sourceCommit !== null ? "source-commit" : "latest-source-head"; const sourceBranchAdvanced = targetMode !== "latest-source-head" && sourceCommit !== null && branchTipCommit !== null && sourceCommit !== branchTipCommit; const branchDrift = { targetMode, sourceBranch: spec.source.branch, currentBranchTipCommit: branchTipCommit, targetSourceCommit: sourceCommit, pipelineRunSourceCommit, sourceBranchAdvanced, targetSupersededByCurrentBranch: sourceBranchAdvanced, closeoutHint: sourceBranchAdvanced ? "target PipelineRun/source commit is not the current branch tip; verify the branch tip contains the fix, then trigger-current again for latest-tip closeout" : null, valuesPrinted: false, }; const warnings = [ ...(sourceWorktreeDetached ? ["source-worktree-detached"] : []), ...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []), ]; const blockers = [ ...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]), ...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]), ...(sourcePayload.workspaceClean === true || sourcePayload.workspaceExists !== true ? [] : ["source-worktree-dirty"]), ...(mirrorPayload.readReady === true ? [] : ["git-mirror-read-not-ready"]), ...(mirrorPayload.writeReady === true ? [] : ["git-mirror-write-not-ready"]), ...(mirrorPayload.cachePvcExists === true ? [] : ["git-mirror-cache-pvc-missing"]), ...(runtimePayload.ciNamespaceExists === true ? [] : ["ci-namespace-missing"]), ...(pipeline.exists === true ? [] : ["pipeline-missing"]), ...(pipelineRunName !== null && pipelineRunStatus.exists !== true ? ["pipelinerun-missing"] : []), ...(pipelineRunStatus.exists !== true || pipelineRunStatus.status === undefined || pipelineRunStatus.status === null || pipelineRunStatus.status === "True" ? [] : ["pipelinerun-not-succeeded"]), ...(runtimePayload.serviceAccountExists === true ? [] : ["ci-serviceaccount-missing"]), ...(argo.exists === true ? [] : ["argo-application-missing"]), ...(mirrorPayload.readReady === true && expectedGitopsRevision === null ? ["gitops-revision-unresolved"] : []), ...(argo.exists !== true || expectedGitopsRevision === null || argoSyncedToGitops ? [] : ["argo-revision-stale"]), ...(runtimePayload.runtimeNamespaceExists === true ? [] : ["runtime-namespace-missing"]), ...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]), ...(manager.deploymentExists !== true || sourceCommit === null || managerSourceMatchesExpected ? [] : ["manager-source-stale"]), ...(manager.serviceExists === true ? [] : ["manager-service-missing"]), ...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []), ...(secrets.ready === true ? [] : ["lane-secret-missing"]), ...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []), ]; const ciBlockerNames = new Set(["ci-namespace-missing", "pipeline-missing", "pipelinerun-missing", "pipelinerun-not-succeeded", "ci-serviceaccount-missing"]); const ciBlockers = blockers.filter((blocker) => ciBlockerNames.has(blocker)); const runtimeBlockers = blockers.filter((blocker) => !ciBlockerNames.has(blocker)); const aligned = blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected; const runtimeAligned = runtimeBlockers.length === 0 && argoSyncedToGitops && managerSourceMatchesExpected; const ciEvidenceMissing = pipelineRunName !== null && pipelineRunStatus.exists !== true; const mirrorAlreadySynced = mirrorPayload.readReady === true && mirrorPayload.writeReady === true && sourceCommit !== null && mirrorSourceCommit === sourceCommit && expectedGitopsRevision !== null; const runtimeAlignment = { pipelineSucceeded, argoRevision: stringOrNull(argo.revision), argoSyncedToGitops, managerSourceCommit: stringOrNull(manager.sourceCommit), managerSourceMatchesExpected, runtimeAligned, }; const statusFullCommand = agentRunControlPlaneStatusCommand(spec, options, true); const triggerCurrentCommand = `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`; const nextAction = aligned ? { code: "none", summary: "lane aligned; no action required", command: null } : ciEvidenceMissing && runtimeAligned && mirrorAlreadySynced ? { code: "ci-evidence-missing", summary: "runtime and git mirror are aligned, but the expected PipelineRun evidence is missing; rerun trigger-current to recreate CI evidence", command: triggerCurrentCommand } : sourceBranchAdvanced ? { code: "target-superseded", summary: "target commit is no longer the branch tip; trigger-current against latest source before closeout", command: triggerCurrentCommand } : ciBlockers.length > 0 ? { code: "ci-blocked", summary: `CI evidence is blocked: ${ciBlockers.join(", ")}`, command: statusFullCommand } : runtimeBlockers.length > 0 ? { code: "runtime-blocked", summary: `runtime alignment is blocked: ${runtimeBlockers.join(", ")}`, command: statusFullCommand } : { code: "inspect-full-status", summary: "alignment is inconclusive; inspect full status details", command: statusFullCommand }; const compactSourceStatus = { workspaceExists: sourcePayload.workspaceExists ?? false, workspaceClean: sourcePayload.workspaceClean ?? null, branch: sourcePayload.branch ?? null, remoteBranchExists: sourcePayload.remoteBranchExists ?? false, remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, workspaceDetached: sourceWorktreeDetached, }; const compactGitMirrorStatus = { readReady: mirrorPayload.readReady ?? false, writeReady: mirrorPayload.writeReady ?? false, cachePvcExists: mirrorPayload.cachePvcExists ?? false, sourceCommit: mirrorSourceCommit, gitopsCommit: expectedGitopsRevision, alreadySynced: mirrorAlreadySynced, }; const compactCiStatus = { namespaceExists: runtimePayload.ciNamespaceExists ?? false, serviceAccountExists: runtimePayload.serviceAccountExists ?? false, pipelineExists: pipeline.exists ?? false, pipelineRun: { name: pipelineRunName, exists: pipelineRunStatus.exists ?? false, status: pipelineRunStatus.status ?? null, reason: pipelineRunStatus.reason ?? null, sourceCommit: pipelineRunSourceCommit, }, evidenceMissing: ciEvidenceMissing, blockers: ciBlockers, }; const compactArgoStatus = { exists: argo.exists ?? false, application: argo.application ?? null, revision: stringOrNull(argo.revision), syncStatus: argo.syncStatus ?? null, healthStatus: argo.healthStatus ?? null, syncedToGitops: argoSyncedToGitops, }; const compactRuntimeStatus = { namespaceExists: runtimePayload.runtimeNamespaceExists ?? false, manager: { deploymentExists: manager.deploymentExists ?? false, serviceExists: manager.serviceExists ?? false, sourceCommit: stringOrNull(manager.sourceCommit), sourceMatchesExpected: managerSourceMatchesExpected, }, databaseSecretPresent: database.secretPresent ?? null, secretsReady: secrets.ready ?? null, localPostgresAbsent: localPostgres.absent ?? null, blockers: runtimeBlockers, }; const detailedSummary = { aligned, runtimeAligned, ciEvidenceMissing, mirrorAlreadySynced, blockers, warnings, sourceCommit, expectedPipelineRun: pipelineRunName, expectedGitopsRevision, runtimeAlignment, branchDrift, source: { workspaceExists: sourcePayload.workspaceExists ?? false, workspaceClean: sourcePayload.workspaceClean ?? null, branch: sourcePayload.branch ?? null, workspaceDetached: sourceWorktreeDetached, localHead: sourcePayload.localHead ?? null, remoteBranchExists: sourcePayload.remoteBranchExists ?? false, remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, }, gitMirror: { readReady: mirrorPayload.readReady ?? false, writeReady: mirrorPayload.writeReady ?? false, cachePvcExists: mirrorPayload.cachePvcExists ?? false, repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null, sourceCommit: mirrorSourceCommit, gitopsCommit: expectedGitopsRevision, alreadySynced: mirrorAlreadySynced, }, ci: { namespaceExists: runtimePayload.ciNamespaceExists ?? false, serviceAccountExists: runtimePayload.serviceAccountExists ?? false, pipeline, pipelineRun: pipelineRunStatus, evidenceMissing: ciEvidenceMissing, blockers: ciBlockers, }, argo, runtime: { namespaceExists: runtimePayload.runtimeNamespaceExists ?? false, manager: { deploymentExists: manager.deploymentExists ?? false, serviceExists: manager.serviceExists ?? false, deployment: manager.deployment ?? null, service: manager.service ?? null, image: manager.image ?? null, sourceCommit: stringOrNull(manager.sourceCommit), sourceMatchesExpected: managerSourceMatchesExpected, }, database, secrets: compactLaneSecretsStatus(secrets), localPostgres, }, nextAction, }; const commanderSummary = { aligned, runtimeAligned, ciEvidenceMissing, mirrorAlreadySynced, blockers, warnings, sourceCommit, expectedPipelineRun: pipelineRunName, expectedGitopsRevision, runtimeAlignment, branchDrift, source: compactSourceStatus, gitMirror: compactGitMirrorStatus, ci: compactCiStatus, argo: compactArgoStatus, runtime: compactRuntimeStatus, nextAction, }; const result: Record = { ok: sourceProbe.value.exitCode === 0 && runtimeProbe.value.exitCode === 0 && mirrorProbe.value.exitCode === 0 && blockers.length === 0, command: "agentrun control-plane status", mode: "yaml-declared-node-lane", configPath: target.configPath, target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), summary: options.full || options.raw ? detailedSummary : commanderSummary, alignment: { aligned, runtimeAligned, ciEvidenceMissing, mirrorAlreadySynced, blockers, warnings, sourceCommit, expectedPipelineRun: pipelineRunName, expectedGitopsRevision, runtimeAlignment, branchDrift, }, timings: { sourceMs: sourceProbe.elapsedMs, runtimeMs: runtimeProbe.elapsedMs, gitMirrorMs: mirrorProbe.elapsedMs, totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs), }, disclosure: { output: options.full || options.raw ? "full" : "compact-summary", full: options.full, raw: options.raw, omittedWhenCompact: options.full || options.raw ? [] : ["full target YAML summary", "detailed source/runtime/gitMirror objects", "capture stdout/stderr tails for successful probes"], fullCommand: statusFullCommand, }, next: { action: nextAction, plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`, postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`, statusFull: statusFullCommand, triggerCurrent: ciEvidenceMissing || sourceBranchAdvanced ? triggerCurrentCommand : null, triggerLatest: sourceBranchAdvanced ? `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, }, valuesPrinted: false, }; if (options.full || options.raw || sourceProbe.value.exitCode !== 0 || runtimeProbe.value.exitCode !== 0 || mirrorProbe.value.exitCode !== 0) { result.captures = { source: compactCapture(sourceProbe.value, { full: options.full || options.raw || sourceProbe.value.exitCode !== 0 }), runtime: compactCapture(runtimeProbe.value, { full: options.full || options.raw || runtimeProbe.value.exitCode !== 0 }), gitMirror: compactCapture(mirrorProbe.value, { full: options.full || options.raw || mirrorProbe.value.exitCode !== 0 }), }; } if (options.full || options.raw) { result.source = sourcePayload; result.runtime = runtimePayload; result.gitMirror = mirrorPayload; } return result; } function agentRunControlPlaneStatusCommand(spec: AgentRunLaneSpec, options: Pick, full: boolean): string { return [ "bun scripts/cli.ts agentrun control-plane status", `--node ${spec.nodeId}`, `--lane ${spec.lane}`, options.pipelineRun === null ? null : `--pipeline-run ${options.pipelineRun}`, options.sourceCommit === null ? null : `--source-commit ${options.sourceCommit}`, full ? "--full" : null, ].filter(Boolean).join(" "); } function renderAgentRunControlPlaneStatusSummary(result: Record): RenderedCliResult { const summary = record(result.summary); const target = record(result.target); const node = record(target.node); const runtimeTarget = record(target.runtime); const ciTarget = record(target.ci); const source = record(summary.source); const gitMirror = record(summary.gitMirror); const ci = record(summary.ci); const ciRun = record(ci.pipelineRun); const argo = record(summary.argo); const runtime = record(summary.runtime); const runtimeManager = record(runtime.manager); const nextAction = record(summary.nextAction); const next = record(result.next); const disclosure = record(result.disclosure); const timings = record(result.timings); const blockers = Array.isArray(summary.blockers) ? summary.blockers.map(String) : []; const warnings = Array.isArray(summary.warnings) ? summary.warnings.map(String) : []; const lines = [ "AGENTRUN CONTROL-PLANE STATUS", renderTable( ["NODE", "LANE", "NAMESPACE", "SOURCE", "PIPELINE", "ALIGNED", "RUNTIME", "BLOCKERS"], [[ displayValue(node.id ?? target.node ?? "-"), displayValue(target.lane ?? "-"), displayValue(runtimeTarget.namespace ?? "-"), shortSha(summary.sourceCommit), displayValue(summary.expectedPipelineRun ?? "-"), yesNo(summary.aligned), yesNo(summary.runtimeAligned), blockers.length === 0 ? "-" : blockers.join(","), ]], ), "", renderTable( ["COMPONENT", "STATUS", "DETAIL"], [ ["source", yesNo(source.workspaceClean), `branch=${displayValue(source.branch)} commit=${shortSha(source.remoteBranchCommit)} detached=${yesNo(source.workspaceDetached)}`], ["git-mirror", yesNo(gitMirror.alreadySynced), `read=${yesNo(gitMirror.readReady)} write=${yesNo(gitMirror.writeReady)} gitops=${shortSha(gitMirror.gitopsCommit)}`], ["ci", yesNo(ciRun.status === "True"), `run=${displayValue(ciRun.name)} status=${displayValue(ciRun.status)} reason=${displayValue(ciRun.reason)} evidenceMissing=${yesNo(ci.evidenceMissing)}`], ["argo", yesNo(argo.syncedToGitops), `sync=${displayValue(argo.syncStatus)} health=${displayValue(argo.healthStatus)} revision=${shortSha(argo.revision)}`], ["runtime", yesNo(runtimeManager.sourceMatchesExpected), `manager=${shortSha(runtimeManager.sourceCommit)} secrets=${yesNo(runtime.secretsReady)} dbSecret=${yesNo(runtime.databaseSecretPresent)} localPgAbsent=${yesNo(runtime.localPostgresAbsent)}`], ], ), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((warning) => `- ${warning}`)].join("\n"), "", "NEXT", ` action: ${displayValue(nextAction.summary ?? "-")}`, nextAction.command ? ` run: ${displayValue(nextAction.command)}` : null, ` full: ${displayValue(next.statusFull ?? disclosure.fullCommand ?? "bun scripts/cli.ts agentrun control-plane status --full")}`, "", `TIMINGS source=${displayValue(timings.sourceMs)}ms runtime=${displayValue(timings.runtimeMs)}ms gitMirror=${displayValue(timings.gitMirrorMs)}ms total=${displayValue(timings.totalMs)}ms`, ].filter((line): line is string => line !== null); return renderedCliResult(result.ok !== false, "agentrun control-plane status", `${lines.join("\n")}\n`); } function yesNo(value: unknown): string { if (value === true) return "yes"; if (value === false) return "no"; if (value === null || value === undefined) return "-"; return displayValue(value); } function shortSha(value: unknown): string { const text = displayValue(value); if (/^[0-9a-f]{40}$/u.test(text)) return text.slice(0, 12); return text; } function compactLaneSecretsStatus(secrets: Record): Record { const items = Array.isArray(secrets.items) ? secrets.items.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; const missing = items .filter((item) => item.present !== true || item.keyPresent !== true) .map((item) => ({ namespace: item.namespace ?? null, name: item.name ?? null, key: item.key ?? null, present: item.present ?? false, keyPresent: item.keyPresent ?? false, })); return { ready: secrets.ready ?? false, count: secrets.count ?? items.length, missingCount: missing.length, missing, valuesPrinted: false, }; } function compactAgentRunLaneStatusTarget(spec: AgentRunLaneSpec): Record { return { node: { id: spec.nodeId, route: spec.nodeRoute, kubeRoute: spec.nodeKubeRoute, }, lane: spec.lane, version: spec.version, source: { repository: spec.source.repository, branch: spec.source.branch, }, runtime: { namespace: spec.runtime.namespace, managerDeployment: spec.runtime.managerDeployment, managerService: spec.runtime.managerService, }, ci: { namespace: spec.ci.namespace, pipeline: spec.ci.pipeline, pipelineRunPrefix: spec.ci.pipelineRunPrefix, }, gitops: { branch: spec.gitops.branch, argoApplication: spec.gitops.argoApplication, }, gitMirror: { namespace: spec.gitMirror.namespace, readService: spec.gitMirror.readService, writeService: spec.gitMirror.writeService, repositoryCount: spec.gitMirror.repositories.length, }, database: { mode: spec.database.mode, provider: spec.database.provider, configRef: spec.database.configRef, localPostgresExpectedAbsent: spec.database.localPostgresExpectedAbsent, valuesPrinted: false, }, secretCount: spec.secrets.length, valuesPrinted: false, }; } async function restartYamlLane(config: UniDeskConfig, options: LaneConfirmOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const plan = { node: spec.nodeId, kubeRoute: spec.nodeKubeRoute, lane: spec.lane, namespace: spec.runtime.namespace, deployment: spec.runtime.managerDeployment, annotation: "kubectl.kubernetes.io/restartedAt", reason: "reload-lane-secrets-or-runtime-env", valuesPrinted: false, }; if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane restart", mode: "dry-run", mutation: false, configPath, target: agentRunLaneSummary(spec), plan, next: { confirm: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", restartYamlLaneScript(spec)]); const payload = captureJsonPayload(result); return { ok: result.exitCode === 0 && payload.ok !== false, command: "agentrun control-plane restart", mode: "confirmed-rollout-restart", mutation: true, configPath, target: agentRunLaneSummary(spec), plan, result: payload, capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }), next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const sources = collectLaneSecretSources(spec); if (sources.length === 0) { return { ok: false, command: "agentrun control-plane secret-sync", mode: "yaml-declared-node-lane", configPath, target: agentRunLaneSummary(spec), degradedReason: "lane-secret-sources-not-declared", valuesPrinted: false, }; } const values = sources.map((source) => ({ spec: source, value: readSecretSourceValue(spec, source) })); const plan = values.map(({ spec: item, value }) => ({ id: item.id, namespace: item.targetRef.namespace, secret: item.targetRef.name, key: item.targetRef.key, sourceRef: item.sourceRef, sourceMode: item.sourceMode, sourceKey: item.sourceKey, sourcePath: value.redactedPath, fingerprint: value.fingerprint, valueBytes: value.valueBytes, valuesPrinted: false, })); if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane secret-sync", mode: "dry-run", mutation: false, configPath, target: agentRunLaneSummary(spec), plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, next: { confirm: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", secretSyncScript(spec, values.map(({ spec: item, value }) => ({ targetRef: item.targetRef, value: value.value })))]); const payload = captureJsonPayload(result); return { ok: result.exitCode === 0 && payload.ok !== false, command: "agentrun control-plane secret-sync", mode: "confirmed-sync", mutation: true, configPath, target: agentRunLaneSummary(spec), plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, result: payload, capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } async function exposeAgentRun(_config: UniDeskConfig, options: ConfirmOptions): Promise> { const clientConfig = readAgentRunClientConfig(); const exposure = clientConfig.publicExposure; if (!exposure?.enabled) { return { ok: true, command: "agentrun control-plane expose", action: "disabled-by-yaml", publicExposure: exposureSummary(exposure), valuesPrinted: false, }; } if (options.dryRun) { return { ok: true, command: "agentrun control-plane expose", dryRun: true, mutation: false, publicExposure: exposureSummary(exposure), plan: { masterFrps: { action: "ensure-allow-port", configPath: exposure.masterFrps.configPath, containerName: exposure.masterFrps.containerName, remotePort: exposure.remotePort, }, masterCaddy: { action: "ensure-https-vhost", domain: exposure.masterCaddy.domain, configPath: exposure.masterCaddy.configPath, serviceName: exposure.masterCaddy.serviceName, upstreamBaseUrl: exposure.masterCaddy.upstreamBaseUrl, responseHeaderTimeoutSeconds: exposure.masterCaddy.responseHeaderTimeoutSeconds, }, }, next: { confirm: "bun scripts/cli.ts agentrun control-plane expose --confirm" }, valuesPrinted: false, }; } const masterFrps = applyAgentRunMasterFrpsAllowPort(exposure); const masterCaddy = applyAgentRunMasterCaddySite(exposure); const ok = masterFrps.ok === true && masterCaddy.ok === true; return { ok, command: "agentrun control-plane expose", mutation: true, publicExposure: exposureSummary(exposure), masterFrps, masterCaddy, validation: { publicProbe: `curl -fsS ${exposure.publicBaseUrl.replace(/\/+$/u, "")}/health/readiness`, directClient: "bun scripts/cli.ts agentrun get tasks --queue commander -o json", valuesPrinted: false, }, valuesPrinted: false, }; } function exposureSummary(exposure: AgentRunPublicExposure | null | undefined): Record { if (!exposure) return { enabled: false, valuesPrinted: false }; return { enabled: exposure.enabled, proxyName: exposure.proxyName, remotePort: exposure.remotePort, publicBaseUrl: exposure.publicBaseUrl, masterBaseUrl: exposure.masterBaseUrl, frps: { configPath: exposure.masterFrps.configPath, containerName: exposure.masterFrps.containerName, }, caddy: { enabled: exposure.masterCaddy.enabled, domain: exposure.masterCaddy.domain, configPath: exposure.masterCaddy.configPath, serviceName: exposure.masterCaddy.serviceName, upstreamBaseUrl: exposure.masterCaddy.upstreamBaseUrl, responseHeaderTimeoutSeconds: exposure.masterCaddy.responseHeaderTimeoutSeconds, }, valuesPrinted: false, }; } function applyAgentRunMasterFrpsAllowPort(exposure: AgentRunPublicExposure): Record { const pathValue = exposure.masterFrps.configPath; const port = exposure.remotePort; const container = exposure.masterFrps.containerName; if (!existsSync(pathValue)) return { ok: false, error: "master-frps-config-missing", path: pathValue, valuesPrinted: false }; const before = readFileSync(pathValue, "utf8"); const alreadyAllowed = frpsAllowPortExists(before, port); let backupPath: string | null = null; let action = "kept-existing"; if (!alreadyAllowed) { const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z"); backupPath = `${pathValue}.bak-agentrun-${stamp}`; copyFileSync(pathValue, backupPath); writeFileSync(pathValue, `${before.replace(/\s*$/u, "")}\n\n[[allowPorts]]\nstart = ${port}\nend = ${port}\n`, "utf8"); chmodSync(pathValue, statSync(backupPath).mode & 0o777); action = "added-allow-port"; } const restart = alreadyAllowed ? null : spawnSync("docker", ["restart", container], { encoding: "utf8" }); const inspect = spawnSync("docker", ["inspect", "-f", "{{.State.Running}}", container], { encoding: "utf8" }); const ok = alreadyAllowed || (restart?.status === 0 && inspect.status === 0 && String(inspect.stdout).trim() === "true"); return { ok, action, path: pathValue, backupPath, remotePort: port, containerName: container, restart: restart === null ? null : { exitCode: restart.status, stdoutTail: String(restart.stdout).slice(-1000), stderrTail: String(restart.stderr).slice(-1000) }, inspect: { exitCode: inspect.status, running: String(inspect.stdout).trim() === "true", stderrTail: String(inspect.stderr).slice(-1000) }, valuesPrinted: false, }; } function applyAgentRunMasterCaddySite(exposure: AgentRunPublicExposure): Record { const caddy = exposure.masterCaddy; if (!caddy.enabled) return { ok: true, action: "disabled-by-yaml", valuesPrinted: false }; const pathValue = caddy.configPath; if (!existsSync(pathValue)) return { ok: false, error: "master-caddy-config-missing", path: pathValue, valuesPrinted: false }; const desiredBlock = renderAgentRunCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds); const applied = applyLocalCaddyManagedSite({ serviceId: "agentrun", markerName: "agentrun", configPath: pathValue, serviceName: caddy.serviceName, siteBlock: desiredBlock, legacySiteDomain: caddy.domain, }); return { ...applied, domain: caddy.domain, upstreamBaseUrl: caddy.upstreamBaseUrl, responseHeaderTimeoutSeconds: caddy.responseHeaderTimeoutSeconds, valuesPrinted: false, }; } function renderAgentRunCaddySiteBlock(domain: string, upstreamBaseUrl: string, responseHeaderTimeoutSeconds: number): string { const upstream = new URL(upstreamBaseUrl); const upstreamHost = `${upstream.hostname}${upstream.port ? `:${upstream.port}` : ""}`; return `${domain} { encode zstd gzip reverse_proxy ${upstreamHost} { header_up Host {host} header_up X-Real-IP {remote_host} transport http { dial_timeout 5s response_header_timeout ${responseHeaderTimeoutSeconds}s } } }`; } function frpsAllowPortExists(toml: string, port: number): boolean { const sections = toml.split(/(?=\[\[allowPorts\]\])/u); return sections.some((section) => { if (!section.includes("[[allowPorts]]")) return false; const start = section.match(/^\s*start\s*=\s*(\d+)\s*$/mu); const end = section.match(/^\s*end\s*=\s*(\d+)\s*$/mu); return Number(start?.[1]) === port && Number(end?.[1]) === port; }); } async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): Promise> { return await triggerCurrentYamlLane(config, options, resolveAgentRunLaneTarget(options)); } async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { const spec = target.spec; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapProbeScript(spec)]); const source = captureJsonPayload(probe); const sourceCommit = stringOrNull(source.sourceCommit); const remoteBranchExists = source.remoteBranchExists === true; const pipelineRun = sourceCommit !== null && isGitSha(sourceCommit) ? agentRunPipelineRunName(spec, sourceCommit) : null; const placeholderImage = sourceCommit === null ? null : placeholderAgentRunImage(spec, sourceCommit); const renderedFiles = placeholderImage === null ? [] : renderAgentRunGitopsFiles(spec, { sourceCommit, image: placeholderImage }); const plan = { node: spec.nodeId, lane: spec.lane, version: spec.version, source: { workspace: spec.source.workspace, remote: spec.source.remote, branch: spec.source.branch, bootstrapFromBranch: spec.source.bootstrapFromBranch, bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds, bootstrapPollSeconds: spec.source.bootstrapPollSeconds, remoteBranchExists, sourceCommit, }, deploymentFormat: spec.deployment.format, deploymentTruth: "config/agentrun.yaml", removedServiceDeployJson: true, pipelineRun, imageBuild: { repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`, containerfile: spec.deployment.manager.imageBuild.containerfile, buildArgNames: Object.keys(spec.deployment.manager.imageBuild.buildArgs).sort(), timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds, pollSeconds: spec.deployment.manager.imageBuild.pollSeconds, proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null, }, gitops: { branch: spec.gitops.branch, path: spec.gitops.path, renderedFileCount: renderedFiles.length, renderedFilesDigest: renderedFiles.length === 0 ? null : renderedFilesDigest(renderedFiles), }, valuesPrinted: false, }; if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane trigger-current", mode: "dry-run", mutation: false, configPath: target.configPath, target: agentRunLaneSummary(spec), plan, sourceProbe: compactCapture(probe, { full: probe.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { confirm: `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } if (options.wait) { return await triggerCurrentYamlLaneConfirmed(config, spec, target.configPath, true); } return startAsyncAgentRunJob( `agentrun_${spec.lane}_trigger_current`, ["bun", "scripts/cli.ts", "agentrun", "control-plane", "trigger-current", "--node", spec.nodeId, "--lane", spec.lane, "--confirm", "--wait"], `Run AgentRun ${spec.version} YAML-only trigger-current on ${spec.nodeId}`, ); } async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { const result = await triggerCurrentYamlLaneConfirmedSteps(config, spec, configPath, waited); const restore = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceRestoreScript(spec)]); const restorePayload = captureJsonPayload(restore); const restoreOk = restore.exitCode === 0 && restorePayload.ok === true; const existingWarnings = Array.isArray(result.warnings) ? result.warnings.filter((item): item is string => typeof item === "string") : []; return { ...result, warnings: restoreOk ? existingWarnings : Array.from(new Set([...existingWarnings, "source-worktree-restore-failed"])), sourceWorkspaceRestore: restorePayload, sourceWorkspaceRestoreCapture: compactCapture(restore, { full: !restoreOk, stdoutTailChars: 3000, stderrTailChars: 3000 }), valuesPrinted: false, }; } async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { node: spec.nodeId, lane: spec.lane, status: "submitting", }); const bootstrapSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapSubmitScript(spec)]); const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit); if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "source-bootstrap-submit", degradedReason: "yaml-lane-source-bootstrap-submit-failed", result: bootstrapSubmitPayload, capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId)); const bootstrapPayload = bootstrap.payload; const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit); if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "source-bootstrap", degradedReason: "yaml-lane-source-bootstrap-failed", result: bootstrapPayload, bootstrapStatus: bootstrap, valuesPrinted: false, }; } const buildSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageSubmitScript(spec, sourceCommit)]); const buildSubmitPayload = captureJsonPayload(buildSubmit); if (buildSubmit.exitCode !== 0 || buildSubmitPayload.ok === false) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "image-build-submit", sourceCommit, degradedReason: "yaml-lane-image-build-submit-failed", sourceBootstrap: bootstrapPayload, result: buildSubmitPayload, capture: compactCapture(buildSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } const build = await waitForYamlLaneBuildImage(config, spec, sourceCommit, stringOrNull(buildSubmitPayload.jobId)); const buildPayload = build.payload; const digest = stringOrNull(buildPayload.digest); const envIdentity = stringOrNull(buildPayload.envIdentity); if (build.ok !== true || digest === null || envIdentity === null) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "image-build", sourceCommit, degradedReason: "yaml-lane-image-build-failed", sourceBootstrap: bootstrapPayload, buildSubmit: buildSubmitPayload, result: buildPayload, buildStatus: build, valuesPrinted: false, }; } const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image }); progressEvent("agentrun.yaml-lane.gitops-publish.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, status: "submitting", }); const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]); const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit); if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "gitops-publish-submit", sourceCommit, image, degradedReason: "yaml-lane-gitops-publish-submit-failed", sourceBootstrap: bootstrapPayload, imageBuild: buildPayload, result: gitopsSubmitPayload, capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId)); const gitopsPayload = gitops.payload; if (gitops.ok !== true || gitopsPayload.ok === false) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "gitops-publish", sourceCommit, image, degradedReason: "yaml-lane-gitops-publish-failed", sourceBootstrap: bootstrapPayload, imageBuild: buildPayload, gitopsPublishSubmit: gitopsSubmitPayload, result: gitopsPayload, gitopsPublishStatus: gitops, valuesPrinted: false, }; } const mirror = await runYamlLaneGitMirrorSyncJob(config, spec); if (mirror.ok !== true) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), phase: "git-mirror-sync", sourceCommit, image, gitops: gitopsPayload, degradedReason: "yaml-lane-git-mirror-sync-failed", result: mirror, valuesPrinted: false, }; } const pipelineRun = agentRunPipelineRunName(spec, sourceCommit); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLanePipelineRunCreateScript(spec, sourceCommit, pipelineRun)]); const createPayload = captureJsonPayload(created); return { ok: created.exitCode === 0 && createPayload.ok !== false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", mutation: true, configPath, target: agentRunLaneSummary(spec), sourceCommit, pipelineRun, image, sourceBootstrap: bootstrapPayload, imageBuildSubmit: buildSubmitPayload, imageBuild: buildPayload, gitopsPublishSubmit: gitopsSubmitPayload, renderedFiles: { count: renderedFiles.length, digest: renderedFilesDigest(renderedFiles), }, gitops: gitopsPayload, gitMirror: mirror, created: createPayload, capture: compactCapture(created, { full: created.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }), next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, statusByPipelineRun: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, }, valuesPrinted: false, }; } async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise> { return await refreshYamlLane(config, options); } async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const plan = { node: spec.nodeId, lane: spec.lane, version: spec.version, argoNamespace: spec.gitops.argoNamespace, argoApplication: spec.gitops.argoApplication, gitopsBranch: spec.gitops.branch, mutation: "argocd-hard-refresh", valuesPrinted: false, }; if (options.dryRun || !options.confirm) { return { ok: true, command: "agentrun control-plane refresh", mode: "dry-run", mutation: false, configPath, target: agentRunLaneSummary(spec), plan, next: { confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } const refreshed = await capture(config, spec.nodeKubeRoute, ["sh", "--", refreshYamlLaneScript(spec)]); const payload = captureJsonPayload(refreshed); return { ok: refreshed.exitCode === 0 && payload.ok !== false, command: "agentrun control-plane refresh", mode: "confirmed-yaml-lane", mutation: true, configPath, target: agentRunLaneSummary(spec), plan, result: payload, refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; } async function cleanupRunners(config: UniDeskConfig, options: CleanupRunnersOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runners", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.runtime.namespace, retention: spec.deployment.runner.retention, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, }; } async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runs", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.ci.namespace, minAgeMinutes: options.minAgeMinutes, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`, diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-released-pvs", configPath, target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", namespace: spec.ci.namespace, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; if (options.dryRun || !options.confirm) { return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`, }, }; } return { ...base, dryRun: false, mutation: true, followUp: { cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`, diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string { const retention = spec.deployment.runner.retention; const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64"); const jobNamePrefixesB64 = Buffer.from(JSON.stringify(retention.selectors.jobNamePrefixes), "utf8").toString("base64"); return [ "set -eu", `namespace=${shQuote(spec.runtime.namespace)}`, `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, `max_runners=${String(retention.maxRunners)}`, `cleanup_order=${shQuote(retention.cleanupOrder)}`, `active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`, `age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`, `age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, `force_active=${options.forceActive ? "true" : "false"}`, `match_labels_json_b64=${shQuote(matchLabelsB64)}`, `job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`, "match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)", "job_name_prefixes_json=$(printf '%s' \"$job_name_prefixes_json_b64\" | base64 -d)", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", "facts_exit=0", "set +e", "kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'", cleanupRunnersFactsNodeScript(), "NODE", "facts_exit=$?", "set -e", "if [ \"$facts_exit\" -ne 0 ]; then", " printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"", "fi", "MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunnersPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then", " xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupRunnersFinalizeNodeScript(), "NODE", ].join("\n"); } function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string { return [ "set -eu", `namespace=${shQuote(namespace)}`, `pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`, `min_age_minutes=${String(options.minAgeMinutes)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"", "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", "NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-names.txt\" ]; then", " xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupRunsFinalizeNodeScript(), "NODE", ].join("\n"); } function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string { return [ "set -eu", `namespace=${shQuote(namespace)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupReleasedPvsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", " cat \"$tmp_dir/plan.json\"", " exit 0", "fi", "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"", "delete_exit=0", "if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then", " xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", "else", " : > \"$tmp_dir/delete.out\"", " : > \"$tmp_dir/delete.err\"", "fi", "kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"", "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", cleanupReleasedPvsFinalizeNodeScript(), "NODE", ].join("\n"); } function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", `expected_workspace=${shQuote(spec.source.workspace)}`, `source_branch=${shQuote(spec.source.branch)}`, "workspace_exists=false", "workspace_clean=null", "local_head=null", "branch=null", "remote_url=null", "remote_branch_exists=false", "remote_branch_commit=null", "status_short=''", "if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then", " actual_workspace=$(pwd)", " workspace_exists=true", " branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)", " remote_url=$(git remote get-url origin 2>/dev/null || true)", " local_head=$(git rev-parse HEAD 2>/dev/null || true)", " status_short=$(git status --short 2>/dev/null || true)", " if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi", " git fetch origin \"$source_branch\" >/dev/null 2>&1 || true", " remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)", " if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi", "else", " actual_workspace=$(pwd)", "fi", "export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace", "node <<'NODE'", "function nullable(value) { return value && value !== 'null' ? value : null; }", "function booleanValue(value) { if (value === 'true') return true; if (value === 'false') return false; return null; }", "const env = process.env;", "console.log(JSON.stringify({", " ok: env.workspace_exists === 'true',", " expectedWorkspace: env.expected_workspace,", " actualWorkspace: env.actual_workspace || null,", " workspaceExists: env.workspace_exists === 'true',", " workspaceClean: booleanValue(env.workspace_clean),", " branch: nullable(env.branch),", " remoteUrl: nullable(env.remote_url),", " localHead: nullable(env.local_head),", " remoteBranch: env.source_branch,", " remoteBranchExists: env.remote_branch_exists === 'true',", " remoteBranchCommit: nullable(env.remote_branch_commit),", " statusShort: nullable(env.status_short),", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string { return [ "set +e", `runtime_namespace=${shQuote(spec.runtime.namespace)}`, `ci_namespace=${shQuote(spec.ci.namespace)}`, `pipeline_name=${shQuote(spec.ci.pipeline)}`, `pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`, `service_account=${shQuote(spec.ci.serviceAccountName)}`, `argo_namespace=${shQuote(spec.gitops.argoNamespace)}`, `argo_application=${shQuote(spec.gitops.argoApplication)}`, `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, `manager_service=${shQuote(spec.runtime.managerService)}`, `database_secret=${shQuote(spec.database.secretRef.name)}`, `database_key=${shQuote(spec.database.secretRef.key)}`, `secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`, "export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null", "runtime_ns_exit=$?", "kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null", "ci_ns_exit=$?", "kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null", "pipeline_exit=$?", "kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null", "sa_exit=$?", "if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi", "kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null", "argo_exit=$?", "kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null", "manager_deploy_exit=$?", "kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null", "manager_svc_exit=$?", "kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null", "db_secret_exit=$?", "SECRET_REFS_JSON=\"$secrets_json\" NODE_TMP=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/secrets.json\"", "const fs = require('node:fs');", "const cp = require('node:child_process');", "const refs = JSON.parse(process.env.SECRET_REFS_JSON || '[]');", "const result = [];", "for (const ref of refs) {", " const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });", " let keyPresent = false;", " if (out.status === 0) {", " try { keyPresent = Object.prototype.hasOwnProperty.call((JSON.parse(out.stdout).data || {}), ref.key); } catch {}", " }", " result.push({ namespace: ref.namespace, name: ref.name, key: ref.key, present: out.status === 0, keyPresent, valuesPrinted: false });", "}", "console.log(JSON.stringify({ ready: result.every((item) => item.present && item.keyPresent), count: result.length, items: result, valuesPrinted: false }));", "NODE", "kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null", "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const dir = process.env.NODE_TMP;", "function readJson(name) { try { return JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); } catch { return null; } }", "function exists(exitName) { return process.env[exitName] === '0'; }", "function condition(obj) { return obj?.status?.conditions?.[0] || null; }", "function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }", "function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }", "const pipelineRun = readJson('pipelinerun.json');", "const argo = readJson('argo.json');", "const managerDeploy = readJson('manager-deploy.json');", "const managerSvc = readJson('manager-svc.json');", "const dbSecret = readJson('db-secret.json');", "const secrets = readJson('secrets.json') || { ready: true, count: 0, items: [], valuesPrinted: false };", "let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}", "const c = condition(pipelineRun);", "console.log(JSON.stringify({", " ok: exists('RUNTIME_NS_EXIT') && exists('CI_NS_EXIT'),", " runtimeNamespaceExists: exists('RUNTIME_NS_EXIT'),", " ciNamespaceExists: exists('CI_NS_EXIT'),", " serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),", " pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },", " pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },", " argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },", " manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },", " database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },", " secrets,", " localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string { return [ "set +e", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, "workspace_exists=false", "remote_branch_exists=false", "source_commit=null", "if [ -d \"$workspace/.git\" ]; then", " workspace_exists=true", " cd \"$workspace\"", " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi", "else", " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi", "fi", "export workspace_exists remote_branch_exists source_commit", "node <<'NODE'", "console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));", "NODE", ].join("\n"); } function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string { const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch; const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, `bootstrap_branch=${shQuote(bootstrap)}`, `state_dir=${shQuote(stateDir)}`, "mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"", "job_id=\"source-bootstrap-$(date +%s)-$$\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", " :", " else", " rm -rf \"$workspace\"", " git clone --no-checkout \"$remote\" \"$workspace\"", " fi", " cd \"$workspace\"", " git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", " git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", " if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", " else", " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", " fi", " if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", " git add -A deploy/deploy.json 2>/dev/null || true", " if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", " fi", " git push -u origin \"$branch\"", " source_commit=$(git rev-parse HEAD)", " status_short=$(git status --short)", " SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); } async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000; const pollMs = spec.source.bootstrapPollSeconds * 1000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { node: spec.nodeId, lane: spec.lane, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", sourceCommit: stringOrNull(payload.sourceCommit), elapsedMs: Date.now() - startedAt, }); if (payload.status === "succeeded" && stringOrNull(payload.sourceCommit) !== null) return { ok: payload.ok === true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(pollMs); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); } function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string { return [ "set +e", `workspace=${shQuote(spec.source.workspace)}`, `branch=${shQuote(spec.source.branch)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "workspace_exists=false", "if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi", "if [ \"$workspace_exists\" != true ]; then", " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'", "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));", "NODE", " exit 0", "fi", "before_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", "before_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", "status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", "if [ -n \"$status_short\" ]; then", " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'", "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));", "NODE", " exit 0", "fi", "git -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"", "fetch_exit=$?", "remote_branch_commit=$(git -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)", "checkout_exit=1", "if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then", " git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"", " checkout_exit=$?", "else", " : > \"$tmp_dir/checkout.out\"", " : > \"$tmp_dir/checkout.err\"", "fi", "after_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", "after_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", "after_status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", "fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", "checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", "WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'", "const fetchExit = Number(process.env.FETCH_EXIT || '1');", "const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');", "const afterClean = !process.env.AFTER_STATUS_SHORT;", "const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;", "let failureKind = null;", "if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';", "else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';", "else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';", "else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';", "else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';", "console.log(JSON.stringify({", " ok,", " status: ok ? 'restored' : 'failed',", " failureKind,", " workspace: process.env.WORKSPACE,", " branch: process.env.BRANCH,", " before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },", " after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },", " remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,", " fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },", " checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string { const build = spec.deployment.manager.imageBuild; const noProxy = build.noProxy.join(","); const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; const buildArgs = Object.entries(build.buildArgs) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, value]) => `${key}=${value}`); const script = [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `source_commit=${shQuote(sourceCommit)}`, `state_dir=${shQuote(stateDir)}`, `containerfile=${shQuote(build.containerfile)}`, `context_dir=${shQuote(build.context)}`, `image_repository=${shQuote(imageRepository)}`, `network=${shQuote(build.network)}`, `http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`, `https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`, `no_proxy_value=${shQuote(noProxy)}`, `env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`, `build_args_json=${shQuote(JSON.stringify(buildArgs))}`, "mkdir -p \"$state_dir\"", "cd \"$workspace\"", "git checkout \"$source_commit\"", "env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", "const { createHash } = require('node:crypto');", "const { readFileSync, existsSync } = require('node:fs');", "const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');", "const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "const hash = createHash('sha256');", "for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }", "for (const file of files) { hash.update(file); hash.update('\\0'); if (existsSync(file)) hash.update(readFileSync(file)); hash.update('\\0'); }", "process.stdout.write(hash.digest('hex').slice(0, 24));", "NODE", ")", "source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)", "job_id=\"$source_short-$env_identity\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " cd \"$workspace\"", " image=\"$image_repository:$env_identity\"", " args=\"--network $network\"", " if [ -n \"$http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$http_proxy_value --build-arg http_proxy=$http_proxy_value\"; fi", " if [ -n \"$https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$https_proxy_value --build-arg https_proxy=$https_proxy_value\"; fi", " if [ -n \"$no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$no_proxy_value --build-arg no_proxy=$no_proxy_value\"; fi", " build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", "const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "for (const value of values) console.log(value);", "NODE", " )", " while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done </dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi", " manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'", " fetch_manifest_digest() {", " ref=\"$1\"", " attempt=1", " max_attempts=30", " while [ \"$attempt\" -le \"$max_attempts\" ]; do", " headers=$(mktemp)", " if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$ref\" >\"$headers\"; then", " found_digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)", " rm -f \"$headers\"", " if [ -n \"$found_digest\" ]; then printf '%s' \"$found_digest\"; return 0; fi", " else", " rm -f \"$headers\"", " fi", " printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"retrying\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$attempt\" \"$max_attempts\" >&2", " sleep 1", " attempt=$((attempt + 1))", " done", " printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"exhausted\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$max_attempts\" \"$max_attempts\" >&2", " return 1", " }", " docker push \"$image\"", " digest=$(fetch_manifest_digest \"$env_identity\" || true)", " if [ -n \"$digest\" ]; then verified_digest=$(fetch_manifest_digest \"$digest\" || true); if [ \"$verified_digest\" != \"$digest\" ]; then digest=; fi; fi", " STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", "const digest = process.env.DIGEST || null;", "console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); return script; } async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.image-build.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", elapsedMs: Date.now() - startedAt, }); if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, `stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`, `stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", "if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi", ].join("\n"); } function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string { const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`; const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({ path: file.path, contentBase64: Buffer.from(file.content, "utf8").toString("base64"), }))), "utf8").toString("base64"); return [ "set -eu", `state_dir=${shQuote(stateDir)}`, `remote=${shQuote(spec.source.remote)}`, `gitops_branch=${shQuote(spec.gitops.branch)}`, `gitops_root=${shQuote(spec.deployment.gitopsRoot)}`, `artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`, `files_b64=${shQuote(filesB64)}`, "mkdir -p \"$state_dir\"", "job_id=\"gitops-publish-$(date +%s)-$$\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", "publish_root=\"$state_dir/worktrees\"", "publish_worktree=\"$publish_root/$job_id\"", "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" PUBLISH_WORKSPACE=\"$publish_worktree\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"", "const code = Number(process.env.CODE || 1);", "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, publishWorkspace: process.env.PUBLISH_WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", " fi; exit \"$code\"; }", " trap write_failed_status EXIT", " mkdir -p \"$publish_root\"", " rm -rf \"$publish_worktree\"", " git clone --no-checkout \"$remote\" \"$publish_worktree\"", " cd \"$publish_worktree\"", " git fetch origin \"$gitops_branch\" || true", " if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then", " git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"", " else", " git checkout --orphan \"$gitops_branch\"", " git rm -rf . >/dev/null 2>&1 || true", " fi", " git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true", " rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json", " FILES_B64=\"$files_b64\" node <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", "for (const file of files) {", " const target = path.resolve(process.cwd(), file.path);", " if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);", " fs.mkdirSync(path.dirname(target), { recursive: true });", " fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));", "}", "NODE", " git add source.json \"$artifact_catalog\" \"$gitops_root\"", " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi", " git push -u origin \"$gitops_branch\"", " gitops_commit=$(git rev-parse HEAD)", " rm -rf \"$publish_worktree\"", " CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" node <<'NODE' > \"$status_file\"", "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));", "NODE", " trap - EXIT", ") >\"$stdout_file\" 2>\"$stderr_file\" &", "pid=$!", "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", "NODE", ].join("\n"); } async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } }; const startedAt = Date.now(); const timeoutMs = 300_000; let lastPayload: Record = {}; let polls = 0; while (Date.now() - startedAt < timeoutMs) { polls += 1; const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]); const payload = captureJsonPayload(probe); lastPayload = payload; progressEvent("agentrun.yaml-lane.gitops-publish.progress", { node: spec.nodeId, lane: spec.lane, sourceCommit, jobId, polls, status: stringOrNull(payload.status) ?? "unknown", gitopsCommit: stringOrNull(payload.gitopsCommit), elapsedMs: Date.now() - startedAt, }); if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; await sleep(5_000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string { const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`; return [ "set +e", `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); } const AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS = 5; const AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN = /(kex_exchange_identification|connection closed by remote host|connection closed by unknown|could not read from remote repository|ssh\.github\.com|github\.com|fetch-pack|early eof|econnreset|read econnreset|proxy-connect|tunnel socket error)/i; function agentRunGitMirrorRetryDelayMs(attempt: number): number { return Math.min(15_000, 1_000 * Math.pow(2, Math.max(0, attempt - 1))); } function agentRunGitMirrorFailureText(value: unknown): string { try { return JSON.stringify(value); } catch { return String(value); } } function agentRunGitMirrorRetryableFailure( spec: AgentRunLaneSpec, action: "sync" | "flush", attempt: number, value: unknown, ): Record | null { const text = agentRunGitMirrorFailureText(value); if (!AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN.test(text)) return null; const retryExhausted = attempt >= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; const retryBackoffMs = retryExhausted ? 0 : agentRunGitMirrorRetryDelayMs(attempt); return { retryable: true, retryAttempt: attempt, retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, retryExhausted, retryBackoffMs, backoffPolicy: "exponential", failureKind: "git-mirror-upstream-transient", upstream: "git-mirror-ssh", action, node: spec.nodeId, lane: spec.lane, message: retryExhausted ? "AgentRun git-mirror upstream transient failure exhausted retries; stopping instead of continuing silently." : "AgentRun git-mirror upstream transient failure; retrying with exponential backoff.", next: retryExhausted ? { stopped: true, reason: "retry-exhausted" } : { retry: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, delayMs: retryBackoffMs, }, valuesPrinted: false, }; } function agentRunGitMirrorRetrySummary(attempts: Record[]): Record { const last = attempts.length > 0 ? record(attempts[attempts.length - 1]) : {}; return { policy: "exponential", maxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS, attempts, exhausted: last.retryExhausted === true, stopped: last.retryExhausted === true, valuesPrinted: false, }; } async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise> { const attempts: Record[] = []; for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) { const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63); const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); if (created.exitCode !== 0) { const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false }; const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); await sleep(agentRunGitMirrorRetryDelayMs(attempt)); continue; } return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; } const startedAt = Date.now(); let polls = 0; let lastProbe: SshCaptureResult | null = null; while (Date.now() - startedAt < 300_000) { polls += 1; lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); const payload = captureJsonPayload(lastProbe); progressEvent("agentrun.yaml-lane.git-mirror.progress", { node: spec.nodeId, lane: spec.lane, jobName, polls, retryAttempt: attempt, retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, succeeded: payload.succeeded === true, failed: payload.failed === true, elapsedMs: Date.now() - startedAt, }); if (payload.succeeded === true) { attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, polls, elapsedMs: Date.now() - startedAt }); return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false }; } if (payload.failed === true) { const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); await sleep(agentRunGitMirrorRetryDelayMs(attempt)); break; } return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; } await sleep(5_000); } if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue; const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); await sleep(agentRunGitMirrorRetryDelayMs(attempt)); continue; } return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; } return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false }; } function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string { const manifest = { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: spec.ci.namespace, labels: { "app.kubernetes.io/part-of": "agentrun", "agentrun.pikastech.local/lane": spec.version, "agentrun.pikastech.local/node": spec.nodeId, "agentrun.pikastech.local/source-commit": sourceCommit, "agentrun.pikastech.local/trigger": "unidesk-yaml-only", }, }, spec: { pipelineRef: { name: spec.ci.pipeline }, taskRunTemplate: { serviceAccountName: spec.ci.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } }, }, params: [ { name: "git-url", value: spec.source.remote }, { name: "git-read-url", value: spec.gitMirror.readUrl }, { name: "git-write-url", value: spec.gitMirror.writeUrl }, { name: "source-branch", value: spec.source.branch }, { name: "gitops-branch", value: spec.gitops.branch }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: spec.ci.registryPrefix }, { name: "tools-image", value: spec.ci.toolsImage }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "4Gi" } } } } }, { name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName } }, ], }, }; const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); return [ "set -eu", `namespace=${shQuote(spec.ci.namespace)}`, `pipeline_run=${shQuote(pipelineRun)}`, `manifest_b64=${shQuote(manifestB64)}`, "tmp=$(mktemp)", "trap 'rm -f \"$tmp\"' EXIT", "printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"", "existing_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", "if [ \"$existing_status\" = False ]; then kubectl -n \"$namespace\" delete pipelinerun \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true; fi", "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then created=false; else kubectl create -f \"$tmp\"; created=true; fi", "status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", "CREATED=\"$created\" STATUS=\"$status\" PIPELINE_RUN=\"$pipeline_run\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, pipelineRun: process.env.PIPELINE_RUN, created: process.env.CREATED === 'true', status: process.env.STATUS || null, valuesPrinted: false }));", "NODE", ].join("\n"); } function createYamlLaneJobScript(namespace: string, jobName: string, manifest: Record): string { const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); return [ "set -eu", `namespace=${shQuote(namespace)}`, `job=${shQuote(jobName)}`, `manifest_b64=${shQuote(manifestB64)}`, "tmp=$(mktemp)", "trap 'rm -f \"$tmp\"' EXIT", "printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"", "kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "kubectl create -f \"$tmp\"", "JOB=\"$job\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, jobName: process.env.JOB, valuesPrinted: false }));", "NODE", ].join("\n"); } function yamlLaneJobProbeScript(namespace: string, jobName: string): string { return [ "set +e", `namespace=${shQuote(namespace)}`, `job=${shQuote(jobName)}`, "kubectl -n \"$namespace\" get job \"$job\" -o json > /tmp/agentrun-job.json 2>/dev/null", "job_exit=$?", "kubectl -n \"$namespace\" logs \"job/$job\" --tail=120 > /tmp/agentrun-job.log 2>/dev/null", "JOB_EXIT=\"$job_exit\" JOB=\"$job\" node <<'NODE'", "const fs = require('node:fs');", "let job = null; try { job = JSON.parse(fs.readFileSync('/tmp/agentrun-job.json', 'utf8')); } catch {}", "let log = ''; try { log = fs.readFileSync('/tmp/agentrun-job.log', 'utf8'); } catch {}", "const succeeded = Number(job?.status?.succeeded || 0) > 0;", "const failed = Number(job?.status?.failed || 0) > 0;", "console.log(JSON.stringify({ ok: process.env.JOB_EXIT === '0', jobName: process.env.JOB, succeeded, failed, active: job?.status?.active || 0, logsTail: log.slice(-4000), valuesPrinted: false }));", "NODE", ].join("\n"); } function yamlLaneGitMirrorCacheVolume(spec: AgentRunLaneSpec): Record { if (spec.gitMirror.cacheHostPath !== null) { return { name: "cache", hostPath: { path: spec.gitMirror.cacheHostPath, type: "DirectoryOrCreate" } }; } return { name: "cache", persistentVolumeClaim: { claimName: spec.gitMirror.cachePvc } }; } function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync" | "flush", name: string): Record { const proxyHostNeedsHostNetwork = spec.gitMirror.githubProxy.host === "127.0.0.1" || spec.gitMirror.githubProxy.host === "localhost"; return { apiVersion: "batch/v1", kind: "Job", metadata: { name, namespace: spec.gitMirror.namespace, labels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "agentrun", "agentrun.pikastech.local/lane": spec.version, "agentrun.pikastech.local/node": spec.nodeId, "agentrun.pikastech.local/component": action, }, }, spec: { backoffLimit: 0, activeDeadlineSeconds: 600, ttlSecondsAfterFinished: 3600, template: { metadata: { labels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "agentrun", "agentrun.pikastech.local/lane": spec.version, "agentrun.pikastech.local/node": spec.nodeId, "agentrun.pikastech.local/component": action, }, }, spec: { restartPolicy: "Never", ...(proxyHostNeedsHostNetwork ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}), volumes: [ yamlLaneGitMirrorCacheVolume(spec), { name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName, defaultMode: 0o400 } }, ], containers: [{ name: action, image: spec.gitMirror.toolsImage, imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-ec", action === "sync" ? yamlLaneGitMirrorSyncShell(spec) : yamlLaneGitMirrorFlushShell(spec)], volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }], }, }, }, }; } function yamlLaneGitMirrorSshSetupShellLines(spec: AgentRunLaneSpec): string[] { const proxyHost = spec.gitMirror.githubProxy.host; const proxyPort = spec.gitMirror.githubProxy.port; const proxyUrl = `http://${proxyHost}:${proxyPort}`; const proxySummary = `agentrun git-mirror-egress-proxy node=${spec.nodeId} lane=${spec.lane} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`; const proxyCommand = `ProxyCommand=node /tmp/agentrun-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; return [ "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", `printf '%s\\n' ${shQuote(proxySummary)} >&2`, "cat > /tmp/agentrun-github-proxy-connect.cjs <<'NODE_PROXY'", "#!/usr/bin/env node", "const net = require('node:net');", "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", "const targetPort = Number.parseInt(targetPortRaw || '', 10);", "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", " console.error('agentrun git-mirror proxy-connect: invalid ProxyCommand arguments');", " process.exit(64);", "}", "let settled = false;", "let tunnelEstablished = false;", "function finish(code, message) {", " if (settled) return;", " settled = true;", " if (message) console.error('agentrun git-mirror proxy-connect: ' + message);", " process.exit(code);", "}", "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", "let buffer = Buffer.alloc(0);", "socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", "function onData(chunk) {", " buffer = Buffer.concat([buffer, chunk]);", " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", " if (headerEnd === -1 && buffer.length < 8192) return;", " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", " const statusLine = head.split('\\r\\n', 1)[0] || '';", " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", " socket.destroy();", " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", " return;", " }", " socket.off('data', onData);", " socket.setTimeout(0);", " tunnelEstablished = true;", " const rest = buffer.slice(headerEnd + 4);", " if (rest.length) process.stdout.write(rest);", " process.stdin.on('error', () => {});", " process.stdout.on('error', () => {});", " process.stdin.pipe(socket);", " socket.pipe(process.stdout);", "}", "socket.on('data', onData);", "NODE_PROXY", "chmod 0700 /tmp/agentrun-github-proxy-connect.cjs", "cat > /tmp/agentrun-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`, "SH_PROXY", "chmod 0700 /tmp/agentrun-git-ssh-proxy.sh", `export HTTP_PROXY=${shQuote(proxyUrl)}`, `export HTTPS_PROXY=${shQuote(proxyUrl)}`, `export ALL_PROXY=${shQuote(proxyUrl)}`, `export http_proxy=${shQuote(proxyUrl)}`, `export https_proxy=${shQuote(proxyUrl)}`, `export all_proxy=${shQuote(proxyUrl)}`, "export GIT_SSH=/tmp/agentrun-git-ssh-proxy.sh", "unset GIT_SSH_COMMAND", ]; } function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string { return [ "set -eu", ...yamlLaneGitMirrorSshSetupShellLines(spec), `repository=${shQuote(spec.source.repository)}`, `source_branch=${shQuote(spec.source.branch)}`, `gitops_branch=${shQuote(spec.gitops.branch)}`, "repo=\"/cache/${repository}.git\"", "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", "mkdir -p \"$(dirname \"$repo\")\"", "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", "else", " rm -rf \"$repo\"", " git init --bare \"$repo\"", " git --git-dir=\"$repo\" remote add origin \"$remote\"", "fi", "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", "git --git-dir=\"$repo\" config http.uploadpack true", "git --git-dir=\"$repo\" config http.receivepack true", "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", "gitops_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\")", "git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$gitops_sha\"", "git --git-dir=\"$repo\" update-server-info", "SOURCE_SHA=\"$source_sha\" GITOPS_SHA=\"$gitops_sha\" node <<'NODE'", "console.log(JSON.stringify({ ok: true, localSource: process.env.SOURCE_SHA, localGitops: process.env.GITOPS_SHA, valuesPrinted: false }));", "NODE", ].join("\n"); } function yamlLaneGitMirrorFlushShell(spec: AgentRunLaneSpec): string { return [ "set -eu", ...yamlLaneGitMirrorSshSetupShellLines(spec), `repository=${shQuote(spec.source.repository)}`, `gitops_branch=${shQuote(spec.gitops.branch)}`, "repo=\"/cache/${repository}.git\"", "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", "test -d \"$repo\"", "git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", "if [ -n \"$local_gitops\" ]; then", " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", " git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", "fi", "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", "LOCAL_GITOPS=\"$local_gitops\" GITHUB_GITOPS=\"$github_gitops\" PENDING=\"$pending\" node <<'NODE'", "console.log(JSON.stringify({ ok: process.env.PENDING !== 'true', localGitops: process.env.LOCAL_GITOPS || null, githubGitops: process.env.GITHUB_GITOPS || null, pendingFlush: process.env.PENDING === 'true', valuesPrinted: false }));", "NODE", ].join("\n"); } function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", `namespace=${shQuote(spec.gitMirror.namespace)}`, `read_deployment=${shQuote(spec.gitMirror.readDeployment)}`, `read_service=${shQuote(spec.gitMirror.readService)}`, `write_service=${shQuote(spec.gitMirror.writeService)}`, `cache_pvc=${shQuote(spec.gitMirror.cachePvc)}`, `repository=${shQuote(spec.source.repository)}`, `source_branch=${shQuote(spec.source.branch)}`, `gitops_branch=${shQuote(spec.gitops.branch)}`, `repositories_json=${shQuote(JSON.stringify(spec.gitMirror.repositories))}`, "kubectl -n \"$namespace\" get svc \"$read_service\" -o json >/tmp/agentrun-gitmirror-read.json 2>/dev/null", "read_exit=$?", "kubectl -n \"$namespace\" get svc \"$write_service\" -o json >/tmp/agentrun-gitmirror-write.json 2>/dev/null", "write_exit=$?", "kubectl -n \"$namespace\" get pvc \"$cache_pvc\" -o json >/tmp/agentrun-gitmirror-cache.json 2>/dev/null", "cache_exit=$?", "kubectl -n \"$namespace\" get deploy,svc,pvc -o name > /tmp/agentrun-gitmirror-names.txt 2>/dev/null", "repo_path=\"/cache/${repository}.git\"", "rm -f /tmp/agentrun-gitmirror-refs.txt", "if [ \"$read_exit\" -eq 0 ]; then", " kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; gitops_branch=\"$3\"; printf \"sourceCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$source_branch\" 2>/dev/null || true; printf \"gitopsCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$gitops_branch\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" \"$gitops_branch\" > /tmp/agentrun-gitmirror-refs.txt 2>/dev/null", "fi", "NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" REPOSITORY=\"$repository\" SOURCE_BRANCH=\"$source_branch\" GITOPS_BRANCH=\"$gitops_branch\" REPOSITORIES_JSON=\"$repositories_json\" node <<'NODE'", "const fs = require('node:fs');", "const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');", "let names = ''; try { names = fs.readFileSync('/tmp/agentrun-gitmirror-names.txt', 'utf8'); } catch {}", "let refs = ''; try { refs = fs.readFileSync('/tmp/agentrun-gitmirror-refs.txt', 'utf8'); } catch {}", "const refValue = (key) => refs.split(/\\r?\\n/).find((line) => line.startsWith(`${key}=`))?.slice(key.length + 1).trim() || null;", "const readReady = process.env.READ_EXIT === '0';", "const writeReady = process.env.WRITE_EXIT === '0';", "const cachePvcExists = process.env.CACHE_EXIT === '0';", "console.log(JSON.stringify({", " ok: readReady && writeReady && cachePvcExists,", " namespace: process.env.NAMESPACE,", " readReady,", " writeReady,", " cachePvcExists,", " resources: names.split(/\\r?\\n/).filter(Boolean).slice(0, 40),", " repository: process.env.REPOSITORY,", " sourceBranch: process.env.SOURCE_BRANCH,", " gitopsBranch: process.env.GITOPS_BRANCH,", " sourceCommit: refValue('sourceCommit'),", " gitopsCommit: refValue('gitopsCommit'),", " repositories,", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } type LaneSecretSource = { id: string; sourceRef: string; sourceMode: "env" | "file"; sourceKey: string | null; targetRef: { namespace: string; name: string; key: string }; }; function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource): { redactedPath: string; value: string; valueBytes: number; fingerprint: string } { const { sourceRef } = source; if (sourceRef.includes("..")) throw new Error(`secret sourceRef must not contain ..: ${sourceRef}`); const sourcePath = sourceRef.startsWith("/") ? sourceRef : join(resolveSecretSourceRoot(spec), ...sourceRef.split("/")); if (!existsSync(sourcePath)) throw new Error(`secret source ${sourceRef} is missing`); let value: string; if (source.sourceMode === "file") { value = readFileSync(sourcePath, "utf8"); } else { if (source.sourceKey === null) throw new Error(`secret source ${sourceRef} is missing sourceKey`); const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const envValue = values.get(source.sourceKey); if (envValue === undefined || envValue.length === 0) throw new Error(`secret source ${sourceRef} is missing required key ${source.sourceKey}`); value = envValue; } if (value.length === 0) throw new Error(`secret source ${sourceRef} is empty`); return { redactedPath: sourceRef.startsWith("/") ? redactAbsoluteSecretPath(sourceRef) : `.state/secrets/${sourceRef}`, value, valueBytes: Buffer.byteLength(value, "utf8"), fingerprint: sha256Fingerprint(value), }; } function redactAbsoluteSecretPath(sourceRef: string): string { const parts = sourceRef.split("/").filter(Boolean); return parts.length === 0 ? "/" : `/${parts.slice(0, -1).join("/")}/`; } function resolveSecretSourceRoot(spec: AgentRunLaneSpec): string { if (spec.database.configRef === null) return rootPath(".state", "secrets"); const configPath = spec.database.configRef.startsWith("/") ? spec.database.configRef : rootPath(spec.database.configRef); const configRoot = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); const secrets = record(configRoot.secrets); const root = stringOrNull(secrets.root); if (root === null || !root.startsWith("/") || root.includes("..")) throw new Error(`${spec.database.configRef}.secrets.root must be an absolute path without ..`); return root; } function parseEnvFile(text: string): Map { const result = new Map(); for (const rawLine of text.split(/\r?\n/u)) { let line = rawLine.trim(); if (line.length === 0 || line.startsWith("#")) continue; if (line.startsWith("export ")) line = line.slice("export ".length).trim(); const index = line.indexOf("="); if (index <= 0) continue; const key = line.slice(0, index).trim(); let value = line.slice(index + 1).trim(); if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { value = value.slice(1, -1); } result.set(key, value); } return result; } function collectLaneSecretSources(spec: AgentRunLaneSpec): LaneSecretSource[] { const result: LaneSecretSource[] = []; if (spec.database.secretSourceRef !== null) { result.push({ id: "database", sourceRef: spec.database.secretSourceRef, sourceMode: "env", sourceKey: spec.database.secretRef.key, targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: spec.database.secretRef.key }, }); } for (const secret of spec.secrets) { result.push({ id: secret.id, sourceRef: secret.sourceRef, sourceMode: secret.sourceMode, sourceKey: secret.sourceKey, targetRef: secret.targetRef, }); } const seen = new Set(); return result.filter((item) => { const key = `${item.targetRef.namespace}/${item.targetRef.name}/${item.targetRef.key}`; if (seen.has(key)) return false; seen.add(key); return true; }); } function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ targetRef: { namespace: string; name: string; key: string }; value: string }>): string { const encoded = Buffer.from(JSON.stringify(values.map((item) => ({ targetRef: item.targetRef, valueBase64: Buffer.from(item.value, "utf8").toString("base64"), }))), "utf8").toString("base64"); return [ "set -eu", `payload_b64=${shQuote(encoded)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "payload_json=\"$tmp_dir/payload.json\"", "printf '%s' \"$payload_b64\" | base64 -d > \"$payload_json\"", "PAYLOAD_JSON=\"$payload_json\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", "const fs = require('node:fs');", "const cp = require('node:child_process');", "const crypto = require('node:crypto');", "const items = JSON.parse(fs.readFileSync(process.env.PAYLOAD_JSON, 'utf8'));", "const results = [];", "const groups = new Map();", "function run(argv, input) {", " const out = cp.spawnSync(argv[0], argv.slice(1), { input, encoding: 'utf8' });", " if (out.status !== 0) throw new Error(`${argv.join(' ')} failed: ${out.stderr || out.stdout}`);", " return out;", "}", "function tryRun(argv) {", " return cp.spawnSync(argv[0], argv.slice(1), { encoding: 'utf8' });", "}", "for (let index = 0; index < items.length; index += 1) {", " const item = items[index];", " const ref = item.targetRef;", " const groupKey = `${ref.namespace}\\u0000${ref.name}`;", " if (!groups.has(groupKey)) groups.set(groupKey, { namespace: ref.namespace, name: ref.name, items: [] });", " groups.get(groupKey).items.push({ key: ref.key, valueBase64: item.valueBase64 });", "}", "for (const group of groups.values()) {", " const ns = run(['kubectl', 'create', 'namespace', group.namespace, '--dry-run=client', '-o', 'yaml']).stdout;", " run(['kubectl', 'apply', '--server-side', '--field-manager=unidesk-agentrun-secret-sync', '-f', '-'], ns);", " const existing = tryRun(['kubectl', '-n', group.namespace, 'get', 'secret', group.name]);", " if (existing.status !== 0) run(['kubectl', '-n', group.namespace, 'create', 'secret', 'generic', group.name]);", " const patch = { data: {} };", " for (const entry of group.items) patch.data[entry.key] = entry.valueBase64;", " const patchFile = `${process.env.TMP_DIR}/${group.namespace}-${group.name}-patch.json`;", " fs.writeFileSync(patchFile, JSON.stringify(patch));", " run(['kubectl', '-n', group.namespace, 'patch', 'secret', group.name, '--type=merge', `--patch-file=${patchFile}`]);", " const fetched = JSON.parse(run(['kubectl', '-n', group.namespace, 'get', 'secret', group.name, '-o', 'json']).stdout);", " for (const entry of group.items) {", " const raw = fetched.data?.[entry.key] || '';", " const decoded = raw ? Buffer.from(raw, 'base64') : Buffer.alloc(0);", " results.push({ namespace: group.namespace, secret: group.name, key: entry.key, ok: raw.length > 0, valueBytes: decoded.length, fingerprint: raw ? 'sha256:' + crypto.createHash('sha256').update(decoded).digest('hex') : null, valuesPrinted: false });", " }", "}", "console.log(JSON.stringify({ ok: results.every((item) => item.ok), secretCount: results.length, items: results, valuesPrinted: false }));", "NODE", ].join("\n"); } function restartYamlLaneScript(spec: AgentRunLaneSpec): string { return [ "set +e", `namespace=${shQuote(spec.runtime.namespace)}`, `deployment=${shQuote(spec.runtime.managerDeployment)}`, "ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)", "kubectl -n \"$namespace\" patch deployment \"$deployment\" --type='merge' -p \"{\\\"spec\\\":{\\\"template\\\":{\\\"metadata\\\":{\\\"annotations\\\":{\\\"kubectl.kubernetes.io/restartedAt\\\":\\\"$ts\\\"}}}}}\" >/tmp/agentrun-restart-patch.out 2>/tmp/agentrun-restart-patch.err", "patch_rc=$?", "if [ \"$patch_rc\" -eq 0 ]; then", " kubectl -n \"$namespace\" rollout status deployment/\"$deployment\" --timeout=180s >/tmp/agentrun-rollout-status.out 2>/tmp/agentrun-rollout-status.err", " rollout_rc=$?", "else", " rollout_rc=$patch_rc", "fi", "pods_json=$(kubectl -n \"$namespace\" get pod -l app.kubernetes.io/name=agentrun-mgr -o json 2>/dev/null || printf '{}')", "deployment_json=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o json 2>/dev/null || printf '{}')", "patch_err=$(cat /tmp/agentrun-restart-patch.err 2>/dev/null || true)", "rollout_err=$(cat /tmp/agentrun-rollout-status.err 2>/dev/null || true)", "PODS_JSON=\"$pods_json\" DEPLOYMENT_JSON=\"$deployment_json\" PATCH_RC=\"$patch_rc\" ROLLOUT_RC=\"$rollout_rc\" TS=\"$ts\" PATCH_ERR=\"$patch_err\" ROLLOUT_ERR=\"$rollout_err\" node <<'NODE'", "const pods = JSON.parse(process.env.PODS_JSON || '{}');", "const deployment = JSON.parse(process.env.DEPLOYMENT_JSON || '{}');", "const conditions = Array.isArray(deployment.status?.conditions) ? deployment.status.conditions : [];", "const items = Array.isArray(pods.items) ? pods.items : [];", "const compactError = (value) => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, 500) || null;", "console.log(JSON.stringify({", " ok: process.env.PATCH_RC === '0' && process.env.ROLLOUT_RC === '0',", " restartedAt: process.env.TS,", " patch: { exitCode: Number(process.env.PATCH_RC || '1'), error: compactError(process.env.PATCH_ERR) },", " rollout: { exitCode: Number(process.env.ROLLOUT_RC || '1'), error: compactError(process.env.ROLLOUT_ERR) },", " readyReplicas: deployment.status?.readyReplicas ?? null,", " replicas: deployment.status?.replicas ?? null,", " conditions: conditions.map((item) => ({ type: item.type, status: item.status, reason: item.reason ?? null })),", " pods: items.map((pod) => ({", " name: pod.metadata?.name ?? null,", " phase: pod.status?.phase ?? null,", " ready: (pod.status?.containerStatuses ?? []).map((item) => item.ready),", " restarts: (pod.status?.containerStatuses ?? []).map((item) => item.restartCount),", " waiting: (pod.status?.containerStatuses ?? []).map((item) => item.state?.waiting?.reason ?? null),", " })),", " valuesPrinted: false", "}));", "NODE", "exit \"$rollout_rc\"", ].join("\n"); } function applyYamlScript(yaml: string, fieldManager: string, dryRun: boolean): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); return [ "set -eu", `manifest_b64=${shQuote(encoded)}`, `field_manager=${shQuote(fieldManager)}`, `dry_run=${dryRun ? "true" : "false"}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "manifest=\"$tmp_dir/manifest.yaml\"", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest\"", "args=\"--server-side --force-conflicts --field-manager=$field_manager\"", "if [ \"$dry_run\" = true ]; then args=\"$args --dry-run=server\"; fi", "set +e", "kubectl apply $args -f \"$manifest\" > \"$tmp_dir/apply.out\" 2> \"$tmp_dir/apply.err\"", "apply_exit=$?", "set -e", "APPLY_EXIT=\"$apply_exit\" APPLY_OUT=\"$tmp_dir/apply.out\" APPLY_ERR=\"$tmp_dir/apply.err\" MANIFEST=\"$manifest\" node <<'NODE'", "const fs = require('node:fs');", "const crypto = require('node:crypto');", "const out = fs.readFileSync(process.env.APPLY_OUT, 'utf8');", "const err = fs.readFileSync(process.env.APPLY_ERR, 'utf8');", "const manifest = fs.readFileSync(process.env.MANIFEST, 'utf8');", "const resources = out.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean).slice(0, 80);", "console.log(JSON.stringify({ ok: process.env.APPLY_EXIT === '0', exitCode: Number(process.env.APPLY_EXIT), resourceCount: resources.length, resources, manifestBytes: Buffer.byteLength(manifest, 'utf8'), manifestDigest: 'sha256:' + crypto.createHash('sha256').update(manifest).digest('hex'), stderrTail: err.slice(-3000), valuesPrinted: false }));", "NODE", "exit \"$apply_exit\"", ].join("\n"); } function manifestObjectRef(object: Record): Record { const metadata = record(object.metadata); return { apiVersion: stringOrNull(object.apiVersion), kind: stringOrNull(object.kind), namespace: stringOrNull(metadata.namespace), name: stringOrNull(metadata.name), }; } function cleanupRunnersFactsNodeScript(): string { return String.raw` function iso(value) { if (value instanceof Date) return value.toISOString(); if (typeof value === "string" && value.length > 0) { const ms = Date.parse(value); return Number.isFinite(ms) ? new Date(ms).toISOString() : null; } return null; } try { const { Pool } = await import("pg"); const namespace = process.env.RETENTION_NAMESPACE || process.env.AGENTRUN_RUNTIME_NAMESPACE || ""; const pool = new Pool({ connectionString: process.env.DATABASE_URL }); const sql = [ "SELECT", " j.id, j.run_id, j.command_id, j.attempt_id, j.runner_id, j.namespace, j.job_name, j.created_at, j.updated_at,", " r.registered_at AS runner_registered_at, r.heartbeat_at AS runner_heartbeat_at,", " rr.status AS run_status, rr.terminal_status AS run_terminal_status, rr.failure_kind AS run_failure_kind, rr.updated_at AS run_updated_at", "FROM agentrun_runner_jobs j", "LEFT JOIN agentrun_runners r ON r.id = j.runner_id", "LEFT JOIN agentrun_runs rr ON rr.id = j.run_id", "WHERE j.namespace = $1", "ORDER BY COALESCE(r.heartbeat_at, j.updated_at, j.created_at) ASC", ].join("\n"); const result = await pool.query(sql, [namespace]); await pool.end(); console.log(JSON.stringify({ ok: true, itemCount: result.rows.length, items: result.rows.map((row) => ({ id: row.id, runId: row.run_id, commandId: row.command_id, attemptId: row.attempt_id, runnerId: row.runner_id, namespace: row.namespace, jobName: row.job_name, runnerJobCreatedAt: iso(row.created_at), runnerJobUpdatedAt: iso(row.updated_at), runnerRegisteredAt: iso(row.runner_registered_at), runnerHeartbeatAt: iso(row.runner_heartbeat_at), runStatus: row.run_status || null, runTerminalStatus: row.run_terminal_status || null, runFailureKind: row.run_failure_kind || null, runUpdatedAt: iso(row.run_updated_at), valuesPrinted: false, })), valuesPrinted: false, })); } catch (error) { console.log(JSON.stringify({ ok: false, itemCount: 0, items: [], failureKind: "manager-runner-facts-query-failed", message: error instanceof Error ? error.message : String(error), valuesPrinted: false, })); } `; } function cleanupRunnersPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const tmp = process.env.TMP_DIR; const namespace = process.env.NAMESPACE; const maxRunners = Number(process.env.MAX_RUNNERS || 0); const cleanupOrder = process.env.CLEANUP_ORDER || ""; const activeHeartbeatMaxAgeMs = Number(process.env.ACTIVE_HEARTBEAT_MAX_AGE_MS || 0); const ageBasedCleanupEnabled = process.env.AGE_BASED_CLEANUP_ENABLED === "true"; const ageBasedMaxAgeHours = process.env.AGE_BASED_MAX_AGE_HOURS ? Number(process.env.AGE_BASED_MAX_AGE_HOURS) : null; const forceActive = process.env.FORCE_ACTIVE === "true"; const matchLabels = JSON.parse(process.env.MATCH_LABELS_JSON || "{}"); const jobNamePrefixes = JSON.parse(process.env.JOB_NAME_PREFIXES_JSON || "[]"); const now = Date.now(); function readJson(name) { return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8")); } function dateMs(value) { const ms = Date.parse(value || ""); return Number.isFinite(ms) ? ms : null; } function ageMs(value) { const ms = dateMs(value); return ms === null ? null : Math.max(0, now - ms); } function labelsOf(item) { return item?.metadata?.labels && typeof item.metadata.labels === "object" ? item.metadata.labels : {}; } function annotationsOf(item) { return item?.metadata?.annotations && typeof item.metadata.annotations === "object" ? item.metadata.annotations : {}; } function matchesLabels(labels) { return Object.entries(matchLabels).every(([key, value]) => labels?.[key] === value); } function matchesPrefix(name) { return jobNamePrefixes.length === 0 || jobNamePrefixes.some((prefix) => name === prefix || name.startsWith(prefix + "-")); } function terminalRunStatus(value) { return value === "completed" || value === "failed" || value === "blocked" || value === "cancelled"; } function jobCondition(job, type) { const conditions = Array.isArray(job?.status?.conditions) ? job.status.conditions : []; return conditions.find((entry) => entry?.type === type && entry?.status === "True") || null; } function isTerminalJob(job) { return jobCondition(job, "Complete") !== null || jobCondition(job, "Failed") !== null || Number(job?.status?.succeeded || 0) > 0 || Number(job?.status?.failed || 0) > 0; } function isTerminalPod(pod) { const phase = pod?.status?.phase || ""; return phase === "Succeeded" || phase === "Failed"; } function jobNameForPod(pod) { const labels = labelsOf(pod); if (typeof labels["job-name"] === "string") return labels["job-name"]; const owner = (Array.isArray(pod?.metadata?.ownerReferences) ? pod.metadata.ownerReferences : []).find((entry) => entry?.kind === "Job" && typeof entry?.name === "string"); return owner?.name || null; } function preferredLastActiveAt(fact, job) { return fact?.runnerHeartbeatAt || fact?.runUpdatedAt || fact?.runnerJobUpdatedAt || job?.status?.completionTime || job?.status?.startTime || job?.metadata?.creationTimestamp || null; } function sortOldestLastActive(left, right) { const leftMs = dateMs(left.lastActiveAt) ?? dateMs(left.createdAt) ?? Number.MAX_SAFE_INTEGER; const rightMs = dateMs(right.lastActiveAt) ?? dateMs(right.createdAt) ?? Number.MAX_SAFE_INTEGER; if (leftMs !== rightMs) return leftMs - rightMs; return String(left.name).localeCompare(String(right.name)); } function countNonTerminalPods(pods) { return pods.filter((pod) => !isTerminalPod(pod)).length; } const jobs = readJson("jobs.json"); const pods = readJson("pods.json"); const facts = readJson("runner-facts.json"); const factItems = Array.isArray(facts.items) ? facts.items : []; const factByJob = new Map(factItems.map((item) => [item.jobName, item])); const allPods = Array.isArray(pods.items) ? pods.items : []; const podsByJob = new Map(); for (const pod of allPods) { const jobName = jobNameForPod(pod); if (!jobName) continue; const entry = podsByJob.get(jobName) || []; entry.push(pod); podsByJob.set(jobName, entry); } const matchedJobs = (Array.isArray(jobs.items) ? jobs.items : []) .map((job) => { const name = job?.metadata?.name || ""; return { job, name, labels: labelsOf(job), annotations: annotationsOf(job) }; }) .filter((item) => item.name && matchesLabels(item.labels) && matchesPrefix(item.name)); const runnerJobs = matchedJobs.map(({ job, name, labels, annotations }) => { const fact = factByJob.get(name) || null; const jobPods = podsByJob.get(name) || []; const nonTerminalPods = jobPods.filter((pod) => !isTerminalPod(pod)); const terminatingPods = nonTerminalPods.filter((pod) => typeof pod?.metadata?.deletionTimestamp === "string"); const terminal = isTerminalJob(job) || terminalRunStatus(fact?.runStatus) || terminalRunStatus(fact?.runTerminalStatus); const heartbeatAgeMs = ageMs(fact?.runnerHeartbeatAt); const heartbeatFresh = heartbeatAgeMs !== null && heartbeatAgeMs <= activeHeartbeatMaxAgeMs; const hasActivePod = Number(job?.status?.active || 0) > 0 || nonTerminalPods.length > 0; const lastActiveAt = preferredLastActiveAt(fact, job); let inactive = false; let protectedActive = false; let classification = "unknown"; if (terminal) { inactive = true; classification = "terminal"; } else if (heartbeatFresh) { protectedActive = true; classification = "active-fresh-heartbeat"; } else if (heartbeatAgeMs !== null) { inactive = true; classification = "inactive-stale-heartbeat"; } else if (terminatingPods.length > 0 && nonTerminalPods.length === terminatingPods.length) { inactive = true; classification = "inactive-terminating"; } else if (hasActivePod) { protectedActive = true; classification = facts.ok === true ? "active-no-heartbeat-row" : "active-unverified-manager-facts"; } else { inactive = true; classification = "inactive-no-active-pod"; } return { name, namespace, createdAt: job?.metadata?.creationTimestamp || null, lastActiveAt, lastActiveAgeMs: ageMs(lastActiveAt), lastActiveSource: fact?.runnerHeartbeatAt ? "runner-heartbeat" : fact?.runUpdatedAt ? "run-updated" : fact?.runnerJobUpdatedAt ? "runner-job-updated" : job?.status?.completionTime ? "job-completion" : job?.status?.startTime ? "job-start" : "job-created", runId: annotations["agentrun.pikastech.local/run-id"] || fact?.runId || null, commandId: annotations["agentrun.pikastech.local/command-id"] || fact?.commandId || null, runnerJobId: fact?.id || null, runnerId: fact?.runnerId || null, runStatus: fact?.runStatus || null, runTerminalStatus: fact?.runTerminalStatus || null, jobStatus: { active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), ready: Number(job?.status?.ready || 0), terminating: Number(job?.status?.terminating || 0), }, podCount: jobPods.length, nonTerminalPodCount: nonTerminalPods.length, terminatingPodCount: terminatingPods.length, inactive, protectedActive, classification, labels, valuesPrinted: false, }; }); const runnerJobCount = runnerJobs.length; const inactiveCandidates = runnerJobs.filter((item) => item.inactive).sort(sortOldestLastActive); const overLimitCount = Math.max(0, runnerJobCount - maxRunners); const selectedByName = new Map(); const selectionReasons = new Map(); for (const item of inactiveCandidates.slice(0, overLimitCount)) { selectedByName.set(item.name, item); selectionReasons.set(item.name, "over-max-runners"); } if (ageBasedCleanupEnabled && ageBasedMaxAgeHours !== null) { const maxAgeMs = ageBasedMaxAgeHours * 3600 * 1000; for (const item of inactiveCandidates) { const itemAgeMs = item.lastActiveAgeMs; if (itemAgeMs !== null && itemAgeMs >= maxAgeMs) { selectedByName.set(item.name, item); if (!selectionReasons.has(item.name)) selectionReasons.set(item.name, "age-based-cleanup"); } } } if (forceActive) { for (const item of runnerJobs) { selectedByName.set(item.name, item); selectionReasons.set(item.name, item.protectedActive ? "force-active-runner-cleanup" : "force-runner-cleanup"); } } const selectionPool = forceActive ? runnerJobs : inactiveCandidates; const selected = selectionPool .filter((item) => selectedByName.has(item.name)) .map((item) => ({ ...item, selectionReason: selectionReasons.get(item.name) || "selected" })); const matchedPodNames = new Set(runnerJobs.map((item) => item.name)); const matchedPods = allPods.filter((pod) => { const jobName = jobNameForPod(pod); return jobName !== null && matchedPodNames.has(jobName); }); const remainingAfterSelection = runnerJobCount - selected.length; console.log(JSON.stringify({ ok: true, planKind: "agentrun-runtime-runner-retention", generatedAt: new Date(now).toISOString(), namespace, criteria: { maxRunners, cleanupOrder, activeHeartbeatMaxAgeMs, forceActive, selectors: { matchLabels, jobNamePrefixes }, ageBasedCleanup: { enabled: ageBasedCleanupEnabled, maxAgeHours: ageBasedMaxAgeHours }, }, managerFacts: { ok: facts.ok === true, factCount: factItems.length, factsExit: Number(process.env.FACTS_EXIT || 0), failureKind: facts.failureKind || null, message: facts.ok === true ? null : facts.message || null, valuesPrinted: false, }, runnerJobCount, nonTerminalRunnerPodCount: countNonTerminalPods(matchedPods), overLimitCount, inactiveCandidateCount: inactiveCandidates.length, protectedActiveRunnerCount: runnerJobs.filter((item) => item.protectedActive).length, activeRunRisk: remainingAfterSelection > maxRunners, remainingRunnerJobCountAfterSelection: remainingAfterSelection, runnerJobs: runnerJobs.sort(sortOldestLastActive), candidates: inactiveCandidates, selected, selectedRunnerJobs: selected.map((item) => item.name), selectedRunnerJobCount: selected.length, valuesPrinted: false, })); `; } function cleanupRunnersFinalizeNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const tmp = process.env.TMP_DIR; const deleteExit = Number(process.env.DELETE_EXIT || 0); const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8")); const jobsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "jobs-after.json"), "utf8")); const podsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pods-after.json"), "utf8")); const matchLabels = plan.criteria?.selectors?.matchLabels || {}; const jobNamePrefixes = plan.criteria?.selectors?.jobNamePrefixes || []; function labelsOf(item) { return item?.metadata?.labels && typeof item.metadata.labels === "object" ? item.metadata.labels : {}; } function matchesLabels(labels) { return Object.entries(matchLabels).every(([key, value]) => labels?.[key] === value); } function matchesPrefix(name) { return jobNamePrefixes.length === 0 || jobNamePrefixes.some((prefix) => name === prefix || name.startsWith(prefix + "-")); } function isTerminalPod(pod) { const phase = pod?.status?.phase || ""; return phase === "Succeeded" || phase === "Failed"; } function jobNameForPod(pod) { const labels = labelsOf(pod); if (typeof labels["job-name"] === "string") return labels["job-name"]; const owner = (Array.isArray(pod?.metadata?.ownerReferences) ? pod.metadata.ownerReferences : []).find((entry) => entry?.kind === "Job" && typeof entry?.name === "string"); return owner?.name || null; } function tail(name) { try { const text = fs.readFileSync(path.join(tmp, name), "utf8"); return text.length > 3000 ? text.slice(-3000) : text; } catch { return ""; } } const selected = new Set(Array.isArray(plan.selectedRunnerJobs) ? plan.selectedRunnerJobs : []); const remainingJobs = (Array.isArray(jobsAfter.items) ? jobsAfter.items : []) .map((job) => ({ name: job?.metadata?.name || "", labels: labelsOf(job) })) .filter((item) => item.name && matchesLabels(item.labels) && matchesPrefix(item.name)); const remainingJobNames = new Set(remainingJobs.map((item) => item.name)); const matchedPods = (Array.isArray(podsAfter.items) ? podsAfter.items : []).filter((pod) => { const jobName = jobNameForPod(pod); return jobName !== null && remainingJobNames.has(jobName); }); const remainingSelectedRunnerJobs = Array.from(selected).filter((name) => remainingJobNames.has(name)); const deletedRunnerJobs = Array.from(selected).filter((name) => !remainingJobNames.has(name)); console.log(JSON.stringify({ ...plan, ok: deleteExit === 0, deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") }, deletedRunnerJobs, deletedRunnerJobCount: deletedRunnerJobs.length, remainingSelectedRunnerJobs, remainingSelectedRunnerJobCount: remainingSelectedRunnerJobs.length, after: { runnerJobCount: remainingJobs.length, nonTerminalRunnerPodCount: matchedPods.filter((pod) => !isTerminalPod(pod)).length, overLimitCount: Math.max(0, remainingJobs.length - Number(plan.criteria?.maxRunners || 0)), }, valuesPrinted: false, })); `; } function cleanupRunsPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const cp = require("node:child_process"); const tmp = process.env.TMP_DIR; const namespace = process.env.NAMESPACE; const pipelineRunPrefix = process.env.PIPELINE_RUN_PREFIX || ""; const minAgeMinutes = Number(process.env.MIN_AGE_MINUTES || 60); const limit = Number(process.env.LIMIT || 20); const now = Date.now(); function readJson(name) { return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8")); } function conditionOf(item) { const conditions = Array.isArray(item?.status?.conditions) ? item.status.conditions : []; return conditions.find((entry) => entry.type === "Succeeded") || conditions[0] || {}; } function ageMinutes(createdAt) { const createdMs = Date.parse(createdAt || ""); return Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null; } function localPathOf(pv) { return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null; } function duBytes(hostPath) { if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null; try { const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); const bytes = Number(String(out).trim().split(/\s+/)[0]); return Number.isFinite(bytes) ? bytes : null; } catch { return null; } } function formatBytes(bytes) { if (!Number.isFinite(bytes)) return null; const units = ["B", "KiB", "MiB", "GiB", "TiB"]; let value = bytes; let unit = 0; while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; } return value.toFixed(unit === 0 ? 0 : 1) + units[unit]; } const pipelineRuns = readJson("pipelineruns.json"); const pvcs = readJson("pvcs.json"); const pvs = readJson("pvs.json"); const pods = readJson("pods.json"); const pvByName = new Map((Array.isArray(pvs.items) ? pvs.items : []).map((item) => [item?.metadata?.name, item])); const activeClaimPods = new Map(); for (const pod of Array.isArray(pods.items) ? pods.items : []) { const phase = pod?.status?.phase || ""; if (phase === "Succeeded" || phase === "Failed") continue; for (const volume of Array.isArray(pod?.spec?.volumes) ? pod.spec.volumes : []) { const claimName = volume?.persistentVolumeClaim?.claimName; if (!claimName) continue; const entry = activeClaimPods.get(claimName) || []; entry.push(pod?.metadata?.name || null); activeClaimPods.set(claimName, entry.filter(Boolean)); } } const pvcsByOwner = new Map(); for (const pvc of Array.isArray(pvcs.items) ? pvcs.items : []) { const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun"); if (!owner?.name) continue; const entry = pvcsByOwner.get(owner.name) || []; entry.push(pvc); pvcsByOwner.set(owner.name, entry); } const allPipelineRuns = (Array.isArray(pipelineRuns.items) ? pipelineRuns.items : []) .map((item) => { const condition = conditionOf(item); return { name: item?.metadata?.name || "", createdAt: item?.metadata?.creationTimestamp || null, ageMinutes: ageMinutes(item?.metadata?.creationTimestamp), status: condition.status || null, reason: condition.reason || null, }; }) .filter((item) => pipelineRunPrefix.length === 0 || item.name.startsWith(pipelineRunPrefix + "-")); const protectedActivePipelineRuns = allPipelineRuns .filter((item) => item.status !== "True" && item.status !== "False") .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))); const protectedLatestPipelineRun = allPipelineRuns .filter((item) => item.status === "True" || item.status === "False") .sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0]?.name || null; const candidates = allPipelineRuns .filter((item) => item.status === "True" || item.status === "False") .filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= minAgeMinutes) .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) .slice(0, limit) .map((item) => { const owned = pvcsByOwner.get(item.name) || []; const activeMountPods = owned.flatMap((pvc) => activeClaimPods.get(pvc?.metadata?.name) || []); const protectedLatest = item.name === protectedLatestPipelineRun; return { ...item, selected: activeMountPods.length === 0 && !protectedLatest, selectedReason: protectedLatest ? "protected-latest-pipelinerun" : activeMountPods.length === 0 ? "terminal-and-unmounted" : "owned-pvc-active-mounted", ownedPvcCount: owned.length, activeMountPods, }; }); const selectedPipelineRuns = candidates.filter((item) => item.selected).map((item) => item.name); const selectedSet = new Set(selectedPipelineRuns); const ownedPvcs = []; const protectedOwnedPvcs = []; for (const [owner, items] of pvcsByOwner.entries()) { if (!selectedSet.has(owner) && !candidates.some((item) => item.name === owner)) continue; for (const pvc of items) { const volume = pvc?.spec?.volumeName || null; const pv = volume ? pvByName.get(volume) : null; const hostPath = localPathOf(pv); const activeMountPods = activeClaimPods.get(pvc?.metadata?.name) || []; const entry = { name: pvc?.metadata?.name || null, volume, phase: pvc?.status?.phase || null, ownerKind: "PipelineRun", owner, storageClass: pv?.spec?.storageClassName || null, reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null, hostPath, estimatedBytes: duBytes(hostPath), activeMountPods, }; if (selectedSet.has(owner)) ownedPvcs.push(entry); else protectedOwnedPvcs.push(entry); } } const estimatedReclaimBytes = ownedPvcs.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0); console.log(JSON.stringify({ ok: true, planKind: "agentrun-ci-completed-pipelinerun-workspace-retention", generatedAt: new Date().toISOString(), namespace, criteria: { prefix: pipelineRunPrefix + "-", terminalStatuses: ["True", "False"], minAgeMinutes, limit }, candidates, candidateCount: candidates.length, protectedActivePipelineRuns, protectedActivePipelineRunCount: protectedActivePipelineRuns.length, protectedLatestPipelineRun, selectedPipelineRuns, selectedPipelineRunCount: selectedPipelineRuns.length, ownedPvcs, ownedPvcCount: ownedPvcs.length, protectedOwnedPvcs, protectedOwnedPvcCount: protectedOwnedPvcs.length, estimatedReclaimBytes, estimatedReclaimHuman: formatBytes(estimatedReclaimBytes), })); `; } function cleanupRunsFinalizeNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const tmp = process.env.TMP_DIR; const deleteExit = Number(process.env.DELETE_EXIT || 0); const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8")); const pvcsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvcs-after.json"), "utf8")); const selected = new Set(Array.isArray(plan.selectedPipelineRuns) ? plan.selectedPipelineRuns : []); const remainingOwnedPvcs = (Array.isArray(pvcsAfter.items) ? pvcsAfter.items : []) .map((pvc) => { const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun"); return { name: pvc?.metadata?.name || null, volume: pvc?.spec?.volumeName || null, phase: pvc?.status?.phase || null, ownerKind: owner?.kind || null, owner: owner?.name || null, }; }) .filter((item) => item.owner && selected.has(item.owner)); function tail(name) { try { const text = fs.readFileSync(path.join(tmp, name), "utf8"); return text.length > 3000 ? text.slice(-3000) : text; } catch { return ""; } } console.log(JSON.stringify({ ...plan, ok: deleteExit === 0, deletedPipelineRuns: Array.from(selected), deletedPipelineRunCount: selected.size, deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") }, remainingOwnedPvcs, remainingOwnedPvcCount: remainingOwnedPvcs.length, })); `; } function cleanupReleasedPvsPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const cp = require("node:child_process"); const tmp = process.env.TMP_DIR; const namespace = process.env.NAMESPACE; const limit = Number(process.env.LIMIT || 20); function readJson(name) { return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8")); } function localPathOf(pv) { return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null; } function duBytes(hostPath) { if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null; try { const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); const bytes = Number(String(out).trim().split(/\s+/)[0]); return Number.isFinite(bytes) ? bytes : null; } catch { return null; } } function formatBytes(bytes) { if (!Number.isFinite(bytes)) return null; const units = ["B", "KiB", "MiB", "GiB", "TiB"]; let value = bytes; let unit = 0; while (value >= 1024 && unit < units.length - 1) { value /= 1024; unit += 1; } return value.toFixed(unit === 0 ? 0 : 1) + units[unit]; } const pvs = readJson("pvs.json"); const candidates = (Array.isArray(pvs.items) ? pvs.items : []) .map((pv) => { const hostPath = localPathOf(pv); return { name: pv?.metadata?.name || "", createdAt: pv?.metadata?.creationTimestamp || null, phase: pv?.status?.phase || null, storageClass: pv?.spec?.storageClassName || null, reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null, claimNamespace: pv?.spec?.claimRef?.namespace || null, claimName: pv?.spec?.claimRef?.name || null, capacity: pv?.spec?.capacity?.storage || null, hostPath, estimatedBytes: duBytes(hostPath), }; }) .filter((item) => item.phase === "Released") .filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete") .filter((item) => item.claimNamespace === namespace) .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt))) .slice(0, limit); const selectedPersistentVolumes = candidates.map((item) => item.name); const estimatedReclaimBytes = candidates.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0); console.log(JSON.stringify({ ok: true, planKind: "agentrun-ci-released-local-path-pv-retention", generatedAt: new Date().toISOString(), namespace, criteria: { phase: "Released", storageClass: "local-path", reclaimPolicy: "Delete", claimNamespace: namespace, limit }, candidates, candidateCount: candidates.length, selectedPersistentVolumes, selectedPersistentVolumeCount: selectedPersistentVolumes.length, estimatedReclaimBytes, estimatedReclaimHuman: formatBytes(estimatedReclaimBytes), })); `; } function cleanupReleasedPvsFinalizeNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const tmp = process.env.TMP_DIR; const deleteExit = Number(process.env.DELETE_EXIT || 0); const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8")); const pvsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvs-after.json"), "utf8")); const selected = new Set(Array.isArray(plan.selectedPersistentVolumes) ? plan.selectedPersistentVolumes : []); const remainingPersistentVolumes = (Array.isArray(pvsAfter.items) ? pvsAfter.items : []) .filter((pv) => selected.has(pv?.metadata?.name)) .map((pv) => ({ name: pv?.metadata?.name || null, phase: pv?.status?.phase || null, claimNamespace: pv?.spec?.claimRef?.namespace || null, claimName: pv?.spec?.claimRef?.name || null })); function tail(name) { try { const text = fs.readFileSync(path.join(tmp, name), "utf8"); return text.length > 3000 ? text.slice(-3000) : text; } catch { return ""; } } console.log(JSON.stringify({ ...plan, ok: deleteExit === 0, deletedPersistentVolumes: Array.from(selected), deletedPersistentVolumeCount: selected.size, deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") }, remainingPersistentVolumes, remainingPersistentVolumeCount: remainingPersistentVolumes.length, })); `; } function refreshYamlLaneScript(spec: AgentRunLaneSpec): string { return [ "set -eu", `namespace=${shQuote(spec.gitops.argoNamespace)}`, `application=${shQuote(spec.gitops.argoApplication)}`, "kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/agentrun-refresh-output.txt", "kubectl -n \"$namespace\" get application \"$application\" -o json >/tmp/agentrun-refresh-app.json", "NAMESPACE=\"$namespace\" APPLICATION=\"$application\" node <<'NODE'", "const fs = require('node:fs');", "let app = {};", "try { app = JSON.parse(fs.readFileSync('/tmp/agentrun-refresh-app.json', 'utf8')); } catch {}", "console.log(JSON.stringify({", " ok: true,", " namespace: process.env.NAMESPACE,", " application: process.env.APPLICATION,", " revision: app.status?.sync?.revision ?? null,", " syncStatus: app.status?.sync?.status ?? null,", " healthStatus: app.status?.health?.status ?? null,", " observedAt: new Date().toISOString(),", " valuesPrinted: false", "}));", "NODE", ].join("\n"); } async function gitMirrorStatus(config: UniDeskConfig, options: GitMirrorStatusOptions = { full: false, raw: false, node: null, lane: null }): Promise> { const target = resolveAgentRunLaneTarget(options); const spec = target.spec; const observation = await readGitMirrorStatus(config, target); const summary = observation.summary; return { ok: observation.ok, command: "agentrun git-mirror status", mode: "yaml-declared-node-lane", configPath: target.configPath, target: agentRunLaneSummary(spec), namespace: spec.gitMirror.namespace, readUrl: spec.gitMirror.readUrl, writeUrl: spec.gitMirror.writeUrl, summary, ...(options.raw ? { raw: observation.raw } : {}), probe: compactCapture(observation.result, { full: options.full || options.raw, stdoutTailChars: 6000, stderrTailChars: 3000 }), disclosure: { defaultView: "compact-low-noise", full: options.full, raw: options.raw, rawOmitted: !options.raw, probeTailOmitted: !(options.full || options.raw), expandWith: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane} --full`, rawWith: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane} --raw`, }, next: { sync: `bun scripts/cli.ts agentrun git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, flush: summary.pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, }, }; } async function readGitMirrorStatus(config: UniDeskConfig, target: { configPath: string; spec: AgentRunLaneSpec }): Promise & { result: SshCaptureResult; raw: string; summary: Record }> { const spec = target.spec; const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)]); const raw = result.stdout; const summary = captureJsonPayload(result); return { ok: result.exitCode === 0 && summary.ok !== false, configPath: target.configPath, target: agentRunLaneSummary(spec), namespace: spec.gitMirror.namespace, readUrl: spec.gitMirror.readUrl, writeUrl: spec.gitMirror.writeUrl, raw, summary, result, }; } async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", options: GitMirrorOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const previewJobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); const previewManifest = yamlLaneGitMirrorJobManifest(spec, action, previewJobName); const command = `agentrun git-mirror ${action}`; if (options.dryRun || !options.confirm) { return { ok: true, command, mode: "yaml-declared-node-lane", configPath, target: agentRunLaneSummary(spec), dryRun: true, namespace: spec.gitMirror.namespace, jobName: previewJobName, manifest: previewManifest, next: { confirm: `bun scripts/cli.ts ${command} --node ${spec.nodeId} --lane ${spec.lane} --confirm` }, }; } if (!options.wait) { const jobName = previewJobName; const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: created.exitCode === 0, command, mode: created.exitCode === 0 ? "submitted" : "create-failed", configPath, target: agentRunLaneSummary(spec), dryRun: false, namespace: spec.gitMirror.namespace, jobName, result: compactCapture(created), status, next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, }, }; } const attempts: Record[] = []; let lastCreated: SshCaptureResult | null = null; let lastWait: Record | null = null; for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) { const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63); const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); lastCreated = created; if (created.exitCode !== 0) { const result = { phase: "create-job", capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }) }; const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure); await sleep(agentRunGitMirrorRetryDelayMs(attempt)); continue; } const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: false, command, mode: "create-failed", configPath, target: agentRunLaneSummary(spec), dryRun: false, namespace: spec.gitMirror.namespace, jobName, result: compactCapture(created), retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts), status, next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, }, }; } const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds); lastWait = wait; if (wait.ok === true) { attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, wait }); const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: true, command, mode: "waited", configPath, target: agentRunLaneSummary(spec), dryRun: false, namespace: spec.gitMirror.namespace, jobName, result: compactCapture(created), wait, retry: agentRunGitMirrorRetrySummary(attempts), status, next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, }, }; } const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, wait); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, wait, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure); await sleep(agentRunGitMirrorRetryDelayMs(attempt)); continue; } const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: false, command, mode: "waited", configPath, target: agentRunLaneSummary(spec), dryRun: false, namespace: spec.gitMirror.namespace, jobName, result: compactCapture(created), wait, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts), status, next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, }, }; } const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: false, command, mode: "retry-exhausted", configPath, target: agentRunLaneSummary(spec), dryRun: false, namespace: spec.gitMirror.namespace, result: lastCreated === null ? null : compactCapture(lastCreated), wait: lastWait, retry: agentRunGitMirrorRetrySummary(attempts), status, next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` }, }; } async function waitForGitMirrorJob(config: UniDeskConfig, spec: AgentRunLaneSpec, action: "sync" | "flush", jobName: string, timeoutSeconds: number): Promise> { const startedAtMs = Date.now(); let lastProbe: SshCaptureResult | null = null; let polls = 0; while (Date.now() - startedAtMs <= timeoutSeconds * 1000) { polls += 1; lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); const summary = captureJsonPayload(lastProbe); process.stderr.write(`${JSON.stringify({ event: `agentrun.git-mirror.${action}.progress`, at: new Date().toISOString(), stage: "k8s-job", node: spec.nodeId, lane: spec.lane, status: summary.succeeded === true ? "succeeded" : summary.failed === true ? "failed" : "running", jobName, polls, elapsedMs: Date.now() - startedAtMs, })}\n`); if (summary.succeeded === true) { return { ok: true, action, jobName, polls, elapsedMs: Date.now() - startedAtMs, summary, probe: compactCapture(lastProbe), }; } if (summary.failed === true) { return { ok: false, action, jobName, polls, elapsedMs: Date.now() - startedAtMs, degradedReason: "git-mirror-job-failed", summary, probe: compactCapture(lastProbe), }; } await sleep(5_000); } return { ok: false, action, jobName, polls, elapsedMs: Date.now() - startedAtMs, degradedReason: "git-mirror-job-timeout", probe: lastProbe ? compactCapture(lastProbe) : null, }; } function startAsyncAgentRunJob(name: string, command: string[], note: string): Record { const job = startJob(name, command, note); const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`; return { ok: true, mode: "async-job", job, statusCommand, tailCommand: `tail -f ${job.stdoutFile}`, next: { status: statusCommand, tail: `tail -f ${job.stdoutFile}`, }, }; } async function runAgentRunRestCompatCommand(config: UniDeskConfig | null, group: AgentRunRestCompatGroup, args: string[], canonicalArgs: string[]): Promise> { try { return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, parseAgentRunRestTargetOptions(args)), async () => { return await runAgentRunRestCommand(config, group, stripAgentRunLaneTargetArgs(args)); }); } catch (error) { if (error instanceof AgentRunRestError) return error.toPayload(`agentrun ${canonicalArgs.join(" ")}`.trim()); throw error; } } async function runAgentRunRestCommand(config: UniDeskConfig | null, group: AgentRunRestCompatGroup, args: string[]): Promise> { void config; const compatGroup = group === "aipods" ? "aipod-specs" : group; const [action, id] = args; if (compatGroup === "queue") return await runAgentRunQueueRest(action, id, args); if (compatGroup === "sessions") return await runAgentRunSessionsRest(action, id, args); if (compatGroup === "runs") return await runAgentRunRunsRest(action, id, args); if (compatGroup === "commands") return await runAgentRunCommandsRest(action, id, args); if (compatGroup === "runner") return await runAgentRunRunnerRest(action, id, args); if (compatGroup === "aipod-specs") return await runAgentRunAipodSpecsRest(action, id, args); throw new AgentRunRestError("unsupported-version", `unsupported AgentRun REST compatibility group: ${group}`); } function parseAgentRunRestTargetOptions(args: string[]): AgentRunRestTargetOptions { return { node: agentRunOption(args, "node"), lane: agentRunOption(args, "lane"), }; } function stripAgentRunLaneTargetArgs(args: string[]): string[] { const result: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (arg === "--node" || arg === "--lane") { index += 1; continue; } if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue; result.push(arg); } return result; } async function runAgentRunQueueRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "list") return await agentRunRestRequest("agentrun queue list", "GET", `/api/v1/queue/tasks${agentRunQuery(args, ["queue", "state", "cursor", "limit", "updated-after"])}`); if (action === "commander") return await agentRunRestRequest("agentrun queue commander", "GET", `/api/v1/queue/commander${agentRunQuery(args, ["queue", "reader-id"])}`); if (action === "stats") return await agentRunRestRequest("agentrun queue stats", "GET", `/api/v1/queue/stats${agentRunQuery(args, ["queue"])}`); if (action === "show" && id) return await agentRunRestRequest("agentrun queue show", "GET", `/api/v1/queue/tasks/${encodeURIComponent(id)}`); if (action === "submit") return await submitQueueTaskRest(args); if (action === "dispatch" && id) return await mutateQueueTaskRest("queue-dispatch", id, "dispatch", queueDispatchBodyFromArgs(args), args); if (action === "read" && id) return await mutateQueueTaskRest("queue-read", id, "read", { readerId: agentRunOption(args, "reader-id") ?? "cli" }, args); if (action === "cancel" && id) return await mutateQueueTaskRest("queue-cancel", id, "cancel", cancelBodyFromArgs(args), args); if (action === "refresh" && id) return await mutateQueueTaskRest("queue-refresh", id, "refresh", {}, args); throw new AgentRunRestError("validation-failed", `unsupported queue command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`); } async function runAgentRunSessionsRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "ps") return await agentRunRestRequest("agentrun sessions ps", "GET", `/api/v1/sessions${agentRunQuery(args, ["state", "profile", "backend-profile", "reader-id", "cursor", "limit"])}`); if (action === "show" && id) return await agentRunRestRequest("agentrun sessions show", "GET", `/api/v1/sessions/${encodeURIComponent(id)}${agentRunQuery(args, ["reader-id"])}`); if ((action === "trace" || action === "output") && id) return await agentRunRestRequest(`agentrun sessions ${action}`, "GET", `/api/v1/sessions/${encodeURIComponent(id)}/${action}${agentRunQuery(args, ["after-seq", "limit", "run-id"])}`); if (action === "read" && id) { const body = { readerId: agentRunOption(args, "reader-id") ?? "cli" }; if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("session-read", `/api/v1/sessions/${encodeURIComponent(id)}/read`, body, `bun scripts/cli.ts agentrun sessions read ${id} --reader-id ${body.readerId}`); return await agentRunRestRequest("agentrun sessions read", "POST", `/api/v1/sessions/${encodeURIComponent(id)}/read`, body); } if (action === "cancel" && id) { const body = { action: "cancel", ...cancelBodyFromArgs(args) }; if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("session-cancel", `/api/v1/sessions/${encodeURIComponent(id)}/control`, body, `bun scripts/cli.ts agentrun sessions cancel ${id}`); return await agentRunRestRequest("agentrun sessions cancel", "POST", `/api/v1/sessions/${encodeURIComponent(id)}/control`, body); } if (action === "send" && id) return await sessionSendRest(id, args); throw new AgentRunRestError("validation-failed", `unsupported sessions command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`); } async function runAgentRunRunsRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "create") return await agentRunRestRequest("agentrun runs create", "POST", "/api/v1/runs", await requiredJsonBody(args, "runs create")); if (action === "show" && id) return await agentRunRestRequest("agentrun runs show", "GET", `/api/v1/runs/${encodeURIComponent(id)}`); if (action === "events" && id) return await agentRunRestRequest("agentrun runs events", "GET", `/api/v1/runs/${encodeURIComponent(id)}/events${agentRunQuery(args, ["after-seq", "limit"])}`); if (action === "result" && id) return await agentRunRestRequest("agentrun runs result", "GET", `/api/v1/runs/${encodeURIComponent(id)}/result${agentRunQuery(args, ["command-id", "command"])}`); if (action === "cancel" && id) return await agentRunRestRequest("agentrun runs cancel", "POST", `/api/v1/runs/${encodeURIComponent(id)}/cancel`, cancelBodyFromArgs(args)); throw new AgentRunRestError("validation-failed", `unsupported runs command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`); } async function runAgentRunCommandsRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "create" && id) { const body = await optionalJsonBody(args); if (!body.type) body.type = agentRunOption(args, "type") ?? "turn"; const idempotencyKey = agentRunOption(args, "idempotency-key"); if (idempotencyKey) body.idempotencyKey = idempotencyKey; return await agentRunRestRequest("agentrun commands create", "POST", `/api/v1/runs/${encodeURIComponent(id)}/commands`, body); } if (action === "show" && id) { const runId = requiredAgentRunOption(args, ["run-id", "run"], "commands show requires --run-id"); return await agentRunRestRequest("agentrun commands show", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/commands/${encodeURIComponent(id)}`); } if (action === "result" && id) { const runId = requiredAgentRunOption(args, ["run-id", "run"], "commands result requires --run-id"); return await agentRunRestRequest("agentrun commands result", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/commands/${encodeURIComponent(id)}/result`); } if (action === "cancel" && id) return await agentRunRestRequest("agentrun commands cancel", "POST", `/api/v1/commands/${encodeURIComponent(id)}/cancel`, cancelBodyFromArgs(args)); throw new AgentRunRestError("validation-failed", `unsupported commands command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`); } async function runAgentRunRunnerRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "jobs") { const runId = requiredAgentRunOption(args, ["run-id", "run"], "runner jobs requires --run-id"); return await agentRunRestRequest("agentrun runner jobs", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs${agentRunQuery(args, ["command-id", "command"])}`); } if (action === "job-status") { const runId = requiredAgentRunOption(args, ["run-id", "run"], "runner job-status requires --run-id"); const runnerJobId = id ?? agentRunOption(args, "runner-job-id"); if (!runnerJobId) return await agentRunRestRequest("agentrun runner job-status", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs${agentRunQuery(args, ["command-id", "command"])}`); return await agentRunRestRequest("agentrun runner job-status", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs/${encodeURIComponent(runnerJobId)}`); } if (action === "job") { const runId = requiredAgentRunOption(args, ["run-id", "run"], "runner job requires --run-id"); const body = queueDispatchBodyFromArgs(args); const commandId = requiredAgentRunOption(args, ["command-id", "command"], "runner job requires --command-id"); body.commandId = commandId; if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("runner-job", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, body, `bun scripts/cli.ts agentrun runner job --run-id ${runId} --command-id ${commandId}`); return await agentRunRestRequest("agentrun runner job", "POST", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, body); } throw new AgentRunRestError("validation-failed", `unsupported runner command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`); } async function runAgentRunAipodSpecsRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "list") return await agentRunRestRequest("agentrun aipod-specs list", "GET", "/api/v1/aipod-specs"); if (action === "show" && id) return await agentRunRestRequest("agentrun aipod-specs show", "GET", `/api/v1/aipod-specs/${encodeURIComponent(id)}`); if (action === "render" && id) return await agentRunRestRequest("agentrun aipod-specs render", "POST", `/api/v1/aipod-specs/${encodeURIComponent(id)}/render`, await aipodRenderInputFromArgs(args, 2)); if (action === "apply" || action === "set") { const yaml = readYamlInputFromArgs(args); const pathValue = id ? `/api/v1/aipod-specs/${encodeURIComponent(id)}` : "/api/v1/aipod-specs"; const method: AgentRunHttpMethod = id ? "PUT" : "POST"; const body = { yaml }; if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("aipod-spec-apply", pathValue, { yamlBytes: Buffer.byteLength(yaml, "utf8") }, `bun scripts/cli.ts agentrun aipod-specs apply${id ? ` ${id}` : ""} --yaml-stdin`, method); return await agentRunRestRequest("agentrun aipod-specs apply", method, pathValue, body); } if ((action === "delete" || action === "rm") && id) return await agentRunRestRequest("agentrun aipod-specs delete", "DELETE", `/api/v1/aipod-specs/${encodeURIComponent(id)}`); throw new AgentRunRestError("validation-failed", `unsupported aipod-specs command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`); } async function submitQueueTaskRest(args: string[]): Promise> { const aipod = agentRunOption(args, "aipod") ?? agentRunOption(args, "aipod-spec"); if (aipod) { const rendered = await agentRunRestRequest("agentrun aipod-specs render", "POST", `/api/v1/aipod-specs/${encodeURIComponent(aipod)}/render`, await aipodRenderInputFromArgs(args, 2)); const body = record(record(innerData(rendered)).queueTask); if (Object.keys(body).length === 0) throw new AgentRunRestError("schema-mismatch", `aipod-spec ${aipod} render did not return queueTask`); if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("queue-submit", "/api/v1/queue/tasks", body, queueSubmitConfirmCommand(args, aipod), "POST", { jsonInput: { source: "aipod-spec", aipod, valuesPrinted: false } }); return await agentRunRestRequest("agentrun queue submit", "POST", "/api/v1/queue/tasks", body); } const body = await requiredJsonBody(args, "queue submit"); const idempotencyKey = agentRunOption(args, "idempotency-key"); if (idempotencyKey) body.idempotencyKey = idempotencyKey; if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("queue-submit", "/api/v1/queue/tasks", body, queueSubmitConfirmCommand(args), "POST", { jsonInput: jsonInputDisclosureFromArgs(args) }); return await agentRunRestRequest("agentrun queue submit", "POST", "/api/v1/queue/tasks", body); } async function mutateQueueTaskRest(action: string, taskId: string, suffix: string, body: Record, args: string[]): Promise> { const pathValue = `/api/v1/queue/tasks/${encodeURIComponent(taskId)}/${suffix}`; if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan(action, pathValue, body, `bun scripts/cli.ts agentrun queue ${suffix} ${taskId}`); return await agentRunRestRequest(`agentrun queue ${suffix}`, "POST", pathValue, body); } async function sessionSendRest(sessionId: string, args: string[]): Promise> { const aipod = agentRunOption(args, "aipod") ?? agentRunOption(args, "aipod-spec"); if (aipod) return await sessionSendWithAipodRest(sessionId, aipod, args); const input = await optionalJsonBody(args); const prompt = optionalPromptFromArgs(args, 2); const sendBody = await sessionSendBodyFromRunInput(sessionId, args, input, sessionSendPayloadFromInput(input, prompt)); const response = await agentRunRestRequest("agentrun sessions send", "POST", `/api/v1/sessions/${encodeURIComponent(sessionId)}/send`, sendBody); return { ok: response.ok !== false, command: "agentrun sessions send", data: innerData(response), bridge: response.bridge }; } function sessionSendPayloadFromInput(input: Record, prompt: string | null): Record { const payload = record(input.payload); if (Object.keys(payload).length > 0) return payload; const text = prompt ?? stringOrNull(input.prompt) ?? stringOrNull(input.message) ?? stringOrNull(input.text); if (text === null) throw new AgentRunRestError("validation-failed", "send requires payload or non-empty prompt/message/text; use --prompt-stdin, --prompt-file, --prompt, a trailing prompt, or JSON payload"); return { prompt: text }; } async function sessionSendBodyFromRunInput(sessionId: string, args: string[], input: Record, payload: Record): Promise> { const fullSendBody = isExplicitSessionSendBody(input); const runInput = fullSendBody ? record(input.run ?? input.runBase) : input; const runBody = await sessionRunBodyFromArgs(sessionId, args, runInput); const runnerDefaults = fullSendBody ? record(input.runnerJob) : {}; const runnerJob = await sessionRunnerJobBody(args, runnerDefaults); const body: Record = fullSendBody ? { ...input } : {}; body.run = runBody; body.payload = record(input.payload); if (Object.keys(record(body.payload)).length === 0) body.payload = payload; body.createRunnerJob = agentRunHasFlag(args, "no-runner-job") ? false : input.createRunnerJob !== false; body.runnerJob = runnerJob; body.dryRun = agentRunHasFlag(args, "dry-run"); const commandIdempotencyKey = agentRunOption(args, "command-idempotency-key") ?? agentRunOption(args, "idempotency-key"); if (commandIdempotencyKey) body.commandIdempotencyKey = commandIdempotencyKey; return body; } function isExplicitSessionSendBody(input: Record): boolean { return isRecord(input.run) || isRecord(input.runBase) || isRecord(input.payload) || isRecord(input.runnerJob) || input.createRunnerJob !== undefined || input.commandIdempotencyKey !== undefined; } async function sessionRunBodyFromArgs(sessionId: string, args: string[], input: Record): Promise> { const existing = await fetchAgentRunSessionOrNull(sessionId, args); const sessionPolicy = readAgentRunClientConfig().client.sessionPolicy; const profile = agentRunOption(args, "profile") ?? agentRunOption(args, "backend-profile") ?? stringOrNull(input.backendProfile) ?? stringOrNull(existing?.backendProfile) ?? sessionPolicy.backendProfile; const body: Record = { ...input }; body.tenantId = agentRunOption(args, "tenant-id") ?? stringOrNull(body.tenantId) ?? stringOrNull(existing?.tenantId) ?? sessionPolicy.tenantId; body.projectId = agentRunOption(args, "project-id") ?? stringOrNull(body.projectId) ?? stringOrNull(existing?.projectId) ?? sessionPolicy.projectId; body.providerId = agentRunOption(args, "provider-id") ?? stringOrNull(body.providerId) ?? stringOrNull(existing?.providerId) ?? sessionPolicy.providerId; body.backendProfile = profile; body.workspaceRef = jsonObjectOption(args, "workspace-json") ?? record(body.workspaceRef); if (Object.keys(record(body.workspaceRef)).length === 0) body.workspaceRef = cloneJsonRecord(sessionPolicy.workspaceRef); body.executionPolicy = jsonObjectOption(args, "execution-policy-json") ?? record(body.executionPolicy); if (Object.keys(record(body.executionPolicy)).length === 0) body.executionPolicy = defaultAgentRunExecutionPolicy(profile, sessionPolicy); const inheritedSessionRef = existing === null ? {} : { conversationId: existing.conversationId, threadId: existing.threadId, metadata: existing.metadata, }; const sessionRef = { ...inheritedSessionRef, ...record(body.sessionRef) }; const metadata = { ...record(inheritedSessionRef.metadata), ...record(sessionRef.metadata) }; const title = agentRunOption(args, "title"); if (title) metadata.title = title; body.sessionRef = { ...sessionRef, sessionId, metadata }; return body; } async function sessionSendWithAipodRest(sessionId: string, aipod: string, args: string[]): Promise> { const rendered = await agentRunRestRequest("agentrun aipod-specs render", "POST", `/api/v1/aipod-specs/${encodeURIComponent(aipod)}/render`, await aipodRenderInputFromArgs(args, 2, { sessionId })); const renderedData = record(innerData(rendered)); const task = record(renderedData.queueTask); if (Object.keys(task).length === 0) throw new AgentRunRestError("schema-mismatch", `aipod-spec ${aipod} render did not return queueTask`); const sessionRef = record(task.sessionRef); const metadata = record(sessionRef.metadata); const title = agentRunOption(args, "title") ?? stringOrNull(task.title); if (title) metadata.title = title; const sessionPolicy = readAgentRunClientConfig().client.sessionPolicy; const backendProfile = stringOrNull(task.backendProfile) ?? sessionPolicy.backendProfile; const executionPolicy = record(task.executionPolicy); const runBody: Record = { tenantId: stringOrNull(task.tenantId) ?? sessionPolicy.tenantId, projectId: stringOrNull(task.projectId) ?? sessionPolicy.projectId, providerId: stringOrNull(task.providerId) ?? sessionPolicy.providerId, backendProfile, workspaceRef: task.workspaceRef ?? cloneJsonRecord(sessionPolicy.workspaceRef), sessionRef: { ...sessionRef, sessionId, metadata }, executionPolicy: Object.keys(executionPolicy).length === 0 ? defaultAgentRunExecutionPolicy(backendProfile, sessionPolicy) : executionPolicy, resourceBundleRef: task.resourceBundleRef, traceSink: { kind: "aipod-session", aipod, sessionId, valuesPrinted: false }, }; const runnerDefaults = record(record(renderedData.dispatchDefaults).runnerJob); const sendBody: Record = { run: runBody, payload: task.payload, createRunnerJob: !agentRunHasFlag(args, "no-runner-job"), runnerJob: await sessionRunnerJobBody(args, runnerDefaults), dryRun: agentRunHasFlag(args, "dry-run"), }; const commandIdempotencyKey = agentRunOption(args, "command-idempotency-key") ?? agentRunOption(args, "idempotency-key"); if (commandIdempotencyKey) sendBody.commandIdempotencyKey = commandIdempotencyKey; const response = await agentRunRestRequest("agentrun sessions send", "POST", `/api/v1/sessions/${encodeURIComponent(sessionId)}/send`, sendBody); const data = record(innerData(response)); return { ok: response.ok !== false, command: "agentrun sessions send", data: { ...data, aipod, profile: String(task.backendProfile ?? ""), valuesPrinted: false }, bridge: response.bridge }; } async function fetchAgentRunSessionOrNull(sessionId: string, args: string[]): Promise | null> { try { return record(innerData(await agentRunRestRequest("agentrun sessions show", "GET", `/api/v1/sessions/${encodeURIComponent(sessionId)}${agentRunQuery(args, ["reader-id"])}`))); } catch (error) { if (error instanceof AgentRunRestError && error.httpStatus === 404) return null; throw error; } } async function sessionRunnerJobBody(args: string[], defaults: Record = {}): Promise> { const runnerBody = { ...defaults, ...(await optionalRunnerJsonBody(args)) }; copyAgentRunOptions(args, runnerBody, ["image", "namespace", "attempt-id", "runner-id", "source-commit", "service-account-name"]); const managerUrl = agentRunOption(args, "runner-manager-url"); if (managerUrl !== null) runnerBody.managerUrl = managerUrl; const runnerIdempotencyKey = agentRunOption(args, "runner-idempotency-key"); if (runnerIdempotencyKey !== null) runnerBody.idempotencyKey = runnerIdempotencyKey; return runnerBody; } async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body?: unknown): Promise> { const clientConfig = readAgentRunClientConfig(); if (activeAgentRunRestTarget !== null) return await agentRunLaneRestRequest(command, method, pathValue, body, clientConfig, activeAgentRunRestTarget); const auth = resolveAgentRunAuth(clientConfig); const bridgeBase = agentRunRestBridgeMetadata(clientConfig, auth, method, pathValue); const startedAt = Date.now(); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), clientConfig.manager.timeoutMs); let response: Response; try { const headers: Record = { [clientConfig.auth.header]: `${clientConfig.auth.scheme} ${auth.value}`, }; const init: RequestInit = { method, headers, signal: controller.signal }; if (body !== undefined) { headers["content-type"] = "application/json"; init.body = JSON.stringify(body); } response = await fetch(new URL(pathValue, clientConfig.manager.baseUrl), init); } catch (error) { throw new AgentRunRestError("agentrun-unreachable", `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } }); } finally { clearTimeout(timeout); } const text = await response.text(); const bridge = { ...bridgeBase, httpStatus: response.status, elapsedMs: Date.now() - startedAt }; if (text.trim().length === 0) throw new AgentRunRestError("schema-mismatch", `AgentRun server returned an empty response for ${method} ${pathValue}`, { bridge, httpStatus: response.status }); let envelope: Record; try { envelope = record(JSON.parse(text) as unknown); } catch { throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge, httpStatus: response.status }); } if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) }); if (!response.ok) { const details = response.status === 404 ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, null) : safeAgentRunEnvelope(envelope); throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details }); } if (envelope.ok !== true) { const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), response.status); throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) }); } return { ok: true, command, data: envelope.data ?? null, bridge, }; } async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise> { const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue); const startedAt = Date.now(); const request = { method, path: pathValue, body: body ?? null, baseUrl: target.spec.runtime.internalBaseUrl, timeoutMs: clientConfig.manager.timeoutMs, authEnv: clientConfig.auth.env, header: clientConfig.auth.header, scheme: clientConfig.auth.scheme, valuesPrinted: false, }; const captureResult = await capture(target.config, target.spec.nodeKubeRoute, ["sh", "--", agentRunLaneRestProxyScript(target.spec, request)]); const captureSummary = compactCapture(captureResult, { full: false, stdoutTailChars: 1200, stderrTailChars: 2000 }); const bridge = { ...bridgeBase, elapsedMs: Date.now() - startedAt, capture: captureSummary }; if (captureResult.exitCode !== 0) { throw new AgentRunRestError("agentrun-unreachable", `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed for ${method} ${pathValue}`, { bridge }); } const proxy = captureJsonPayload(captureResult); if (proxy.ok !== true) { throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) }); } const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0; const text = stringOrNull(proxy.text) ?? ""; const responseBridge = { ...bridge, httpStatus, proxyElapsedMs: nonNegativeIntegerOrNull(proxy.elapsedMs) }; if (text.trim().length === 0) throw new AgentRunRestError("schema-mismatch", `AgentRun server returned an empty response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus }); let envelope: Record; try { envelope = record(JSON.parse(text) as unknown); } catch { throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus }); } if (httpStatus === 401 || httpStatus === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) }); if (httpStatus < 200 || httpStatus >= 300) { const details = httpStatus === 404 ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, target) : safeAgentRunEnvelope(envelope); throw new AgentRunRestError(httpStatus === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${httpStatus}`, { bridge: responseBridge, httpStatus, details }); } if (envelope.ok !== true) { const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), httpStatus); throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) }); } return { ok: true, command, data: envelope.data ?? null, bridge: responseBridge, }; } function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record): string { const requestB64 = Buffer.from(JSON.stringify(request), "utf8").toString("base64"); const evalScript = String.raw` const startedAt = Date.now(); try { const input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8")); const headers = {}; const authEnv = String(input.authEnv || "HWLAB_API_KEY"); const apiKey = process.env[authEnv] || process.env.AGENTRUN_API_KEY || process.env.HWLAB_API_KEY || ""; if (apiKey && input.header && input.scheme) headers[String(input.header)] = String(input.scheme) + " " + apiKey; let body = undefined; if (input.body !== null && input.body !== undefined) { headers["content-type"] = "application/json"; body = JSON.stringify(input.body); } const controller = new AbortController(); const timeoutMs = Number(input.timeoutMs || 15000); const timeout = setTimeout(() => controller.abort(), timeoutMs); let response; try { response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), { method: String(input.method || "GET"), headers, body, signal: controller.signal, }); } finally { clearTimeout(timeout); } const text = await response.text(); console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false })); } catch (error) { console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false })); } `; return [ "set -eu", `namespace=${shQuote(spec.runtime.namespace)}`, `deployment=${shQuote(spec.runtime.managerDeployment)}`, `request_b64=${shQuote(requestB64)}`, `kubectl -n "$namespace" exec deploy/"$deployment" -- env AGENTRUN_LANE_REST_REQUEST_B64="$request_b64" bun --eval ${shQuote(evalScript)}`, ].join("\n"); } function resolveAgentRunRestTarget(config: UniDeskConfig | null, options: AgentRunRestTargetOptions): AgentRunRestTarget | null { if (options.node === null && options.lane === null) return null; if (config === null) throw new AgentRunRestError("validation-failed", "--node/--lane resource queries require UniDesk config"); const { configPath, spec } = resolveAgentRunLaneTarget(options); return { config, configPath, spec }; } async function withAgentRunRestTarget(target: AgentRunRestTarget | null, action: () => Promise): Promise { const previous = activeAgentRunRestTarget; activeAgentRunRestTarget = target; try { return await action(); } finally { activeAgentRunRestTarget = previous; } } function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult { const payload = error.toPayload(command); if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false); const laneHint = record(record(payload.agentrun).laneAwareLookup); const currentEndpoint = record(laneHint.currentEndpoint); const nextCommands = Array.isArray(laneHint.nextCommands) ? laneHint.nextCommands.filter((value): value is string => typeof value === "string" && value.length > 0).slice(0, 4) : []; const hintLines = Object.keys(laneHint).length === 0 ? [] : [ "", "Lane lookup hint:", ` Current baseUrl: ${String(currentEndpoint.baseUrl ?? "(unknown)")}`, ` Config: ${String(currentEndpoint.configPath ?? "(unknown)")}`, ` ${String(laneHint.summary ?? "The resource was not found on the currently configured AgentRun manager endpoint.")}`, ]; const nextLines = nextCommands.length > 0 ? nextCommands : [ "Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.", "Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.", ]; return renderedCliResult(false, command, [ `Error: ${payload.failureKind}`, String(payload.message ?? ""), ...hintLines, "", "Next:", ...nextLines.map((line) => ` ${line}`), ].join("\n")); } class AgentRunRestError extends Error { readonly failureKind: AgentRunFailureKind; readonly bridge: Record | null; readonly httpStatus: number | null; readonly details: Record | null; constructor(failureKind: AgentRunFailureKind, message: string, options: { bridge?: Record; httpStatus?: number; details?: Record } = {}) { super(message); this.name = "AgentRunRestError"; this.failureKind = failureKind; this.bridge = options.bridge ?? null; this.httpStatus = options.httpStatus ?? null; this.details = options.details ?? null; } toPayload(command: string): Record { const transport = stringOrNull(this.bridge?.mode) ?? "direct-http"; return { ok: false, command, failureKind: this.failureKind, message: this.message, transport, clientRole: "render-only", ...(this.httpStatus === null ? {} : { httpStatus: this.httpStatus }), ...(this.bridge === null ? {} : { bridge: this.bridge }), ...(this.details === null ? {} : { agentrun: this.details }), valuesPrinted: false, }; } } function readAgentRunClientConfig(env: NodeJS.ProcessEnv = process.env): AgentRunClientConfig { const configPath = env.AGENTRUN_CLIENT_CONFIG ?? rootPath("config", "agentrun.yaml"); if (!existsSync(configPath)) throw new AgentRunRestError("validation-failed", `AgentRun client config not found: ${configPath}`); try { const parsed = parseAgentRunClientConfigYaml(readFileSync(configPath, "utf8"), configPath); return { ...parsed, sourcePath: configPath }; } catch (error) { if (error instanceof AgentRunRestError) throw error; throw new AgentRunRestError("validation-failed", `AgentRun client config invalid: ${error instanceof Error ? error.message : String(error)}`, { bridge: { mode: "direct-http", clientRole: "render-only", configPath }, }); } } export function parseAgentRunClientConfigYaml(raw: string, sourcePath = "config/agentrun.yaml"): AgentRunClientConfig { const input = record(Bun.YAML.parse(raw) as unknown); validateAgentRunConfigEnvelope(input, sourcePath); const manager = record(input.manager); const auth = record(input.auth); const client = record(input.client); const baseUrl = stringFieldFromRecord(manager, "baseUrl", "manager"); try { const parsed = new URL(baseUrl); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("invalid protocol"); } catch { throw new Error(`${sourcePath}: manager.baseUrl must be an http(s) URL`); } const timeoutMs = numberFieldFromRecord(manager, "timeoutMs", "manager", { min: 1, max: 60_000 }); const header = stringFieldFromRecord(auth, "header", "auth"); const scheme = stringFieldFromRecord(auth, "scheme", "auth"); if (header.toLowerCase() !== "authorization") throw new Error(`${sourcePath}: auth.header must be Authorization`); if (scheme !== "Bearer") throw new Error(`${sourcePath}: auth.scheme must be Bearer`); const role = stringFieldFromRecord(client, "role", "client"); const transport = stringFieldFromRecord(client, "transport", "client"); if (role !== "render-only") throw new Error(`${sourcePath}: client.role must be render-only`); if (transport !== "direct-http") throw new Error(`${sourcePath}: client.transport must be direct-http`); const sessionPolicy = readAgentRunSessionPolicyConfig(client, sourcePath); const publicExposure = readAgentRunPublicExposureConfig(input.publicExposure, baseUrl.replace(/\/+$/u, "/"), sourcePath); return { sourcePath, manager: { baseUrl: baseUrl.replace(/\/+$/u, "/"), timeoutMs, }, auth: { env: stringFieldFromRecord(auth, "env", "auth"), file: stringFieldFromRecord(auth, "file", "auth"), header, scheme, }, client: { role, transport, sessionPolicy, }, publicExposure, }; } function readAgentRunSessionPolicyConfig(client: Record, sourcePath: string): AgentRunSessionPolicyConfig { const pathValue = "client.sessionPolicy"; const sessionPolicy = record(client.sessionPolicy); const workspaceRef = record(sessionPolicy.workspaceRef); const executionPolicy = record(sessionPolicy.executionPolicy); const secretScope = record(executionPolicy.secretScope); stringFieldFromRecord(workspaceRef, "kind", `${pathValue}.workspaceRef`); stringFieldFromRecord(executionPolicy, "sandbox", `${pathValue}.executionPolicy`); stringFieldFromRecord(executionPolicy, "approval", `${pathValue}.executionPolicy`); positiveIntegerFieldFromRecord(executionPolicy, "timeoutMs", `${pathValue}.executionPolicy`); stringFieldFromRecord(executionPolicy, "network", `${pathValue}.executionPolicy`); booleanFieldFromRecord(secretScope, "allowCredentialEcho", `${pathValue}.executionPolicy.secretScope`); return { sourcePath, tenantId: stringFieldFromRecord(sessionPolicy, "tenantId", pathValue), projectId: stringFieldFromRecord(sessionPolicy, "projectId", pathValue), providerId: stringFieldFromRecord(sessionPolicy, "providerId", pathValue), backendProfile: stringFieldFromRecord(sessionPolicy, "backendProfile", pathValue), workspaceRef: cloneJsonRecord(workspaceRef), executionPolicy: cloneJsonRecord(executionPolicy), }; } function validateAgentRunConfigEnvelope(input: Record, sourcePath: string): void { if (input.version !== 1) throw new Error(`${sourcePath}: version must be 1`); if (input.kind !== "AgentRunConfig") throw new Error(`${sourcePath}: kind must be AgentRunConfig`); const metadata = record(input.metadata); if (stringFieldFromRecord(metadata, "name", "metadata") !== "agentrun") throw new Error(`${sourcePath}: metadata.name must be agentrun`); } function readAgentRunPublicExposureConfig(value: unknown, managerBaseUrl: string, sourcePath: string): AgentRunPublicExposure | null { if (value === undefined || value === null) return null; const exposure = record(value); if (exposure.enabled !== true) return null; const proxyName = stringFieldFromRecord(exposure, "proxyName", "publicExposure"); if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/u.test(proxyName)) throw new Error(`${sourcePath}: publicExposure.proxyName must be a DNS label`); const remotePort = numberFieldFromRecord(exposure, "remotePort", "publicExposure", { min: 1, max: 65_535 }); const publicBaseUrl = normalizedHttpUrl(stringFieldFromRecord(exposure, "publicBaseUrl", "publicExposure"), `${sourcePath}: publicExposure.publicBaseUrl`); const masterBaseUrl = normalizedHttpUrl(stringFieldFromRecord(exposure, "masterBaseUrl", "publicExposure"), `${sourcePath}: publicExposure.masterBaseUrl`); if (!publicBaseUrl.startsWith("https://")) throw new Error(`${sourcePath}: publicExposure.publicBaseUrl must use https://`); if (publicBaseUrl !== managerBaseUrl) throw new Error(`${sourcePath}: manager.baseUrl must match publicExposure.publicBaseUrl`); if (!masterBaseUrl.startsWith(`http://127.0.0.1:${remotePort}`)) throw new Error(`${sourcePath}: publicExposure.masterBaseUrl must point to http://127.0.0.1:${remotePort}`); const masterFrps = record(exposure.masterFrps); const masterCaddy = record(exposure.masterCaddy); const caddyEnabled = masterCaddy.enabled === true; const domain = stringFieldFromRecord(masterCaddy, "domain", "publicExposure.masterCaddy"); if (!/^[a-z0-9.-]+$/u.test(domain) || !domain.endsWith(".nip.io")) throw new Error(`${sourcePath}: publicExposure.masterCaddy.domain must be a nip.io hostname`); const caddyUpstreamBaseUrl = normalizedHttpUrl(stringFieldFromRecord(masterCaddy, "upstreamBaseUrl", "publicExposure.masterCaddy"), `${sourcePath}: publicExposure.masterCaddy.upstreamBaseUrl`); if (caddyUpstreamBaseUrl !== masterBaseUrl) throw new Error(`${sourcePath}: publicExposure.masterCaddy.upstreamBaseUrl must match publicExposure.masterBaseUrl`); return { enabled: true, proxyName, remotePort, publicBaseUrl, masterBaseUrl, masterFrps: { configPath: absolutePathField(masterFrps, "configPath", "publicExposure.masterFrps", sourcePath), containerName: stringFieldFromRecord(masterFrps, "containerName", "publicExposure.masterFrps"), }, masterCaddy: { enabled: caddyEnabled, domain, configPath: absolutePathField(masterCaddy, "configPath", "publicExposure.masterCaddy", sourcePath), serviceName: stringFieldFromRecord(masterCaddy, "serviceName", "publicExposure.masterCaddy"), upstreamBaseUrl: caddyUpstreamBaseUrl, responseHeaderTimeoutSeconds: numberFieldFromRecord(masterCaddy, "responseHeaderTimeoutSeconds", "publicExposure.masterCaddy", { min: 1, max: 600 }), }, }; } function normalizedHttpUrl(value: string, label: string): string { let url: URL; try { url = new URL(value); } catch { throw new Error(`${label} must be an http(s) URL`); } if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`${label} must be an http(s) URL`); return value.replace(/\/+$/u, "/"); } function absolutePathField(obj: Record, key: string, pathValue: string, sourcePath: string): string { const value = stringFieldFromRecord(obj, key, pathValue); if (!value.startsWith("/")) throw new Error(`${sourcePath}: ${pathValue}.${key} must be an absolute path`); return value; } export function resolveAgentRunAuth(config: AgentRunClientConfig, env: NodeJS.ProcessEnv = process.env): AgentRunResolvedAuth { const envValue = env[config.auth.env]?.trim(); if (envValue) return { source: "env", value: envValue }; if (existsSync(config.auth.file)) { const fromFile = readEnvValueFromFile(config.auth.file, config.auth.env); if (fromFile !== null) return { source: "file", value: fromFile }; } throw new AgentRunRestError("auth-missing", `${config.auth.env} is required for direct AgentRun REST access`, { bridge: { mode: "direct-http", clientRole: "render-only", configPath: config.sourcePath, auth: { env: config.auth.env, file: config.auth.file, source: "missing", header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false }, }, }); } function readEnvValueFromFile(pathValue: string, envName: string): string | null { const text = readFileSync(pathValue, "utf8"); for (const line of text.split(/\r?\n/u)) { const trimmed = line.trim(); if (trimmed.length === 0 || trimmed.startsWith("#")) continue; const normalized = trimmed.startsWith("export ") ? trimmed.slice("export ".length).trim() : trimmed; const index = normalized.indexOf("="); if (index <= 0) continue; if (normalized.slice(0, index).trim() !== envName) continue; const raw = normalized.slice(index + 1).trim(); const value = raw.replace(/^['"]|['"]$/gu, ""); return value.length > 0 ? value : null; } return null; } function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRunResolvedAuth, method: AgentRunHttpMethod, pathValue: string): Record { return { mode: "direct-http", clientRole: "render-only", compatibility: "no-hwlab-runtime-proxy-no-ssh-official-cli", configPath: config.sourcePath, baseUrl: config.manager.baseUrl, request: { method, path: pathValue, timeoutMs: config.manager.timeoutMs }, auth: { env: config.auth.env, file: config.auth.file, source: auth.source, header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false }, }; } function agentRunLaneRestBridgeMetadata(config: AgentRunClientConfig, target: AgentRunRestTarget, method: AgentRunHttpMethod, pathValue: string): Record { return { mode: "lane-k8s-service-proxy", clientRole: "render-only", configPath: config.sourcePath, laneConfigPath: target.configPath, node: target.spec.nodeId, lane: target.spec.lane, kubeRoute: target.spec.nodeKubeRoute, namespace: target.spec.runtime.namespace, managerDeployment: target.spec.runtime.managerDeployment, baseUrl: target.spec.runtime.internalBaseUrl, request: { method, path: pathValue, timeoutMs: config.manager.timeoutMs }, auth: { source: "manager-pod-env", env: config.auth.env, fallbackEnv: "AGENTRUN_API_KEY", header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false }, valuesPrinted: false, }; } function addAgentRunNotFoundLookupHint(details: Record, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string, target: AgentRunRestTarget | null): Record { const resource = agentRunResourceFromPath(pathValue); const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath); const candidateLanes = laneConfig?.lanes ?? []; const nextCommands = candidateLanes .filter((lane) => typeof lane.node === "string" && lane.node.length > 0 && typeof lane.lane === "string" && lane.lane.length > 0) .slice(0, 4) .map((lane) => `bun scripts/cli.ts agentrun control-plane status --node ${lane.node} --lane ${lane.lane}`); return { ...details, laneAwareLookup: { kind: "agentrun-resource-not-found-on-current-endpoint", summary: resource === null ? "The request returned 404 on the currently configured AgentRun manager endpoint; a resource from another lane will not be visible through this baseUrl." : `${resource.kind}/${resource.id} returned 404 on the currently configured AgentRun manager endpoint; if it was created on another lane, inspect that lane before treating the resource as lost.`, request: { method, path: pathValue, ...(resource === null ? {} : { resource }), }, currentEndpoint: { transport: target === null ? "direct-http" : "lane-k8s-service-proxy", baseUrl: target === null ? config.manager.baseUrl : target.spec.runtime.internalBaseUrl, configPath: config.sourcePath, defaultNode: laneConfig?.defaultNode ?? null, defaultLane: laneConfig?.defaultLane ?? null, selectedNode: target?.spec.nodeId ?? null, selectedLane: target?.spec.lane ?? null, }, candidateLanes, nextCommands, note: target === null ? "Pass --node --lane to query a YAML-declared non-default AgentRun lane." : "The resource was not found on the selected AgentRun lane; verify the run/session id and the HWLAB runtime lane that created it.", valuesPrinted: false, }, }; } function agentRunResourceFromPath(pathValue: string): Record | null { const run = pathValue.match(/\/runs\/([^/?#]+)/u); if (run?.[1] !== undefined) return { kind: "run", id: decodeURIComponent(run[1]) }; const session = pathValue.match(/\/sessions\/([^/?#]+)/u); if (session?.[1] !== undefined) return { kind: "session", id: decodeURIComponent(session[1]) }; const command = pathValue.match(/\/commands\/([^/?#]+)/u); if (command?.[1] !== undefined) return { kind: "command", id: decodeURIComponent(command[1]) }; const task = pathValue.match(/\/queue\/tasks\/([^/?#]+)/u); if (task?.[1] !== undefined) return { kind: "task", id: decodeURIComponent(task[1]) }; return null; } function readAgentRunLaneLookupConfig(configPath: string): { defaultNode: string | null; defaultLane: string | null; lanes: Record[] } | null { try { const raw = readFileSync(configPath, "utf8"); const parsed = record(Bun.YAML.parse(raw) as unknown); const controlPlane = record(parsed.controlPlane); const defaultTarget = record(controlPlane.default); const lanes = record(controlPlane.lanes); return { defaultNode: stringOrNull(defaultTarget.node), defaultLane: stringOrNull(defaultTarget.lane), lanes: Object.entries(lanes).map(([laneName, rawLane]) => { const lane = record(rawLane); const runtime = record(lane.runtime); return { lane: laneName, node: stringOrNull(lane.node) ?? stringOrNull(lane.nodeId), version: stringOrNull(lane.version), namespace: stringOrNull(runtime.namespace), internalBaseUrl: stringOrNull(runtime.internalBaseUrl), isDefault: laneName === stringOrNull(defaultTarget.lane), }; }), }; } catch { return null; } } function agentRunDryRunPlan(action: string, pathValue: string, body: Record, confirmCommand: string, method: AgentRunHttpMethod = "POST", extra: Record = {}): Record { return { ok: true, action, dryRun: true, mutation: false, transport: "direct-http", clientRole: "render-only", request: { method, path: pathValue, bodyKeys: Object.keys(body), valuesPrinted: false, }, ...extra, next: { confirm: confirmCommand }, valuesPrinted: false, }; } function agentRunQuery(args: string[], names: string[]): string { const params = new URLSearchParams(); const keyMap: Record = { "after-seq": "afterSeq", "backend-profile": "backendProfile", "command": "commandId", "command-id": "commandId", "reader-id": "readerId", "run-id": "runId", "updated-after": "updatedAfter", }; for (const name of names) { const value = agentRunOption(args, name); if (value !== null) params.set(keyMap[name] ?? name, value); } const query = params.toString(); return query.length > 0 ? `?${query}` : ""; } function agentRunOption(args: string[], name: string): string | null { const flag = `--${name}`; for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (arg === flag) { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new AgentRunRestError("validation-failed", `${flag} requires a value`); return value; } if (arg.startsWith(`${flag}=`)) return arg.slice(flag.length + 1); } return null; } function requiredAgentRunOption(args: string[], names: string[], message: string): string { for (const name of names) { const value = agentRunOption(args, name); if (value !== null) return value; } throw new AgentRunRestError("validation-failed", message); } function agentRunHasFlag(args: string[], name: string): boolean { const flag = `--${name}`; return args.includes(flag) || args.some((arg) => arg.startsWith(`${flag}=`)); } async function requiredJsonBody(args: string[], context: string): Promise> { const body = await optionalJsonBody(args); if (Object.keys(body).length === 0) throw new AgentRunRestError("validation-failed", `${context} requires --json-stdin or --json-file `); return body; } async function optionalJsonBody(args: string[]): Promise> { const raw = readJsonTextFromArgs(args, "json"); if (raw === null) return {}; return record(JSON.parse(raw) as unknown); } async function optionalRunnerJsonBody(args: string[]): Promise> { const raw = readJsonTextFromArgs(args, "runner-json"); if (raw === null) return {}; return record(JSON.parse(raw) as unknown); } function readJsonTextFromArgs(args: string[], prefix: "json" | "runner-json"): string | null { if (agentRunHasFlag(args, `${prefix}-stdin`)) return readFileSync(0, "utf8"); const file = agentRunOption(args, `${prefix}-file`); if (file !== null) return file === "-" ? readFileSync(0, "utf8") : readFileSync(file, "utf8"); return null; } function readYamlInputFromArgs(args: string[]): string { if (agentRunHasFlag(args, "yaml-stdin")) return readFileSync(0, "utf8"); const file = agentRunOption(args, "yaml-file"); if (file !== null) return file === "-" ? readFileSync(0, "utf8") : readFileSync(file, "utf8"); throw new AgentRunRestError("validation-failed", "aipod-spec YAML input is required; use --yaml-stdin or --yaml-file "); } async function aipodRenderInputFromArgs(args: string[], trailingPromptStart: number, overrides: Record = {}): Promise> { const input = await optionalJsonBody(args); const prompt = optionalPromptFromArgs(args, trailingPromptStart); if (prompt !== null) input.prompt = prompt; copyAgentRunOptions(args, input, ["tenant-id", "project-id", "queue", "lane", "title", "provider-id", "idempotency-key", "session-id"]); const profile = agentRunOption(args, "profile") ?? agentRunOption(args, "backend-profile"); if (profile) input.backendProfile = profile; const priority = agentRunOption(args, "priority"); if (priority) input.priority = Number(priority); const workspaceRef = jsonObjectOption(args, "workspace-json"); if (workspaceRef !== null) input.workspaceRef = workspaceRef; return { ...input, ...overrides }; } function readPromptFromArgs(args: string[], trailingStart: number): string { const prompt = optionalPromptFromArgs(args, trailingStart); if (prompt === null) throw new AgentRunRestError("validation-failed", "prompt is required; use --prompt-stdin, --prompt-file, --prompt, or a trailing prompt"); return prompt; } function optionalPromptFromArgs(args: string[], trailingStart: number): string | null { if (agentRunHasFlag(args, "prompt-stdin") || agentRunHasFlag(args, "stdin")) return readFileSync(0, "utf8"); const promptFile = agentRunOption(args, "prompt-file"); if (promptFile !== null) return promptFile === "-" ? readFileSync(0, "utf8") : readFileSync(promptFile, "utf8"); const prompt = agentRunOption(args, "prompt"); if (prompt !== null) return prompt; const trailing = args.slice(trailingStart).filter((arg) => !arg.startsWith("-")); return trailing.length > 0 ? trailing.join(" ") : null; } function queueDispatchBodyFromArgs(args: string[]): Record { const body = recordFromMaybeJson(args); copyAgentRunOptions(args, body, ["idempotency-key", "image", "namespace", "attempt-id", "runner-id", "source-commit", "service-account-name"]); const managerUrl = agentRunOption(args, "runner-manager-url"); if (managerUrl) body.managerUrl = managerUrl; return body; } function recordFromMaybeJson(args: string[]): Record { const raw = readJsonTextFromArgs(args, "json"); if (raw === null) return {}; return record(JSON.parse(raw) as unknown); } function cancelBodyFromArgs(args: string[]): Record { const reason = agentRunOption(args, "reason"); return reason === null ? {} : { reason }; } function copyAgentRunOptions(args: string[], target: Record, flagNames: string[]): void { for (const flagName of flagNames) { const value = agentRunOption(args, flagName); if (value === null) continue; target[camelCaseFlag(flagName)] = value; } } function camelCaseFlag(flagName: string): string { return flagName.replace(/-([a-z])/gu, (_, letter: string) => String(letter).toUpperCase()); } function jsonObjectOption(args: string[], flagName: string): Record | null { const value = agentRunOption(args, flagName); if (value === null) return null; return record(JSON.parse(value) as unknown); } function queueSubmitConfirmCommand(args: string[], aipod?: string): string { const dryRunless = args.filter((arg) => arg !== "--dry-run").join(" "); return `bun scripts/cli.ts agentrun queue submit${aipod ? ` --aipod ${aipod}` : ""}${dryRunless.length > 0 ? ` ${dryRunless}` : ""}`.trim(); } function jsonInputDisclosureFromArgs(args: string[]): Record { return { source: agentRunHasFlag(args, "json-stdin") ? "stdin" : agentRunOption(args, "json-file") ?? "", valuesPrinted: false, }; } function defaultAgentRunExecutionPolicy(profile: string, sessionPolicy: AgentRunSessionPolicyConfig = readAgentRunClientConfig().client.sessionPolicy): Record { const { spec } = resolveAgentRunLaneTarget({}); const credentials = agentRunProviderCredentialRefs(spec, profile); if (credentials.length === 0) { throw new AgentRunRestError("validation-failed", `config/agentrun.yaml has no providerCredential Secret binding for backendProfile=${profile} on default lane ${spec.nodeId}/${spec.lane}`); } const providerCredentials = credentials.map((credential) => { if (credential.secretRef.namespace !== spec.runtime.namespace) { throw new AgentRunRestError("validation-failed", `providerCredential ${profile} Secret ${credential.secretRef.name} is in namespace ${credential.secretRef.namespace}, expected default lane namespace ${spec.runtime.namespace}`); } return { profile: credential.profile, secretRef: { name: credential.secretRef.name, keys: credential.secretRef.keys, }, }; }); const basePolicy = cloneJsonRecord(sessionPolicy.executionPolicy); const secretScope = record(basePolicy.secretScope); return { ...basePolicy, secretScope: { ...secretScope, providerCredentials, }, }; } function renderAgentRunSessionPolicyExplanation(): string { const config = readAgentRunClientConfig(); const { spec } = resolveAgentRunLaneTarget({}); const sessionPolicy = config.client.sessionPolicy; const credentials = agentRunProviderCredentialRefs(spec, sessionPolicy.backendProfile); const execution = defaultAgentRunExecutionPolicy(sessionPolicy.backendProfile, sessionPolicy); const credentialSources = credentials.map((credential) => ({ profile: credential.profile, secretRef: credential.secretRef, valuesPrinted: false, })); return [ "KIND: session-policy", `CONFIG: ${config.sourcePath}`, `DEFAULT LANE: ${spec.nodeId}/${spec.lane}`, `DEFAULTS: tenantId=${sessionPolicy.tenantId} projectId=${sessionPolicy.projectId} providerId=${sessionPolicy.providerId} backendProfile=${sessionPolicy.backendProfile}`, `WORKSPACE: ${JSON.stringify(sessionPolicy.workspaceRef)}`, `EXECUTION: ${JSON.stringify(execution)}`, `PROVIDER CREDENTIAL SOURCES: ${JSON.stringify(credentialSources)}`, "VALUES: secret payloads are not printed", ].join("\n"); } function safeAgentRunEnvelope(envelope: Record): Record { return pickCompact(envelope, ["ok", "failureKind", "message", "code", "details", "valuesPrinted"]); } function normalizeAgentRunFailureKind(raw: string | null, httpStatus: number): AgentRunFailureKind { if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw; if (httpStatus === 401 || httpStatus === 403) return "auth-failed"; if (httpStatus === 404) return "not-found"; return raw === "schema-invalid" ? "validation-failed" : "validation-failed"; } function stringFieldFromRecord(obj: Record, key: string, pathValue: string): string { const value = obj[key]; if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${pathValue}.${key} must be a non-empty string`); return value.trim(); } function numberFieldFromRecord(obj: Record, key: string, pathValue: string, bounds: { min: number; max: number }): number { const value = obj[key]; if (typeof value !== "number" || !Number.isFinite(value) || value < bounds.min || value > bounds.max) throw new Error(`${pathValue}.${key} must be a number between ${bounds.min} and ${bounds.max}`); return value; } function positiveIntegerFieldFromRecord(obj: Record, key: string, pathValue: string): number { const value = obj[key]; if (!Number.isInteger(value) || Number(value) <= 0) throw new Error(`${pathValue}.${key} must be a positive integer`); return Number(value); } function booleanFieldFromRecord(obj: Record, key: string, pathValue: string): boolean { const value = obj[key]; if (typeof value !== "boolean") throw new Error(`${pathValue}.${key} must be a boolean`); return value; } function cloneJsonRecord(value: Record): Record { return JSON.parse(JSON.stringify(value)) as Record; } interface AgentRunClientConfig { sourcePath: string; manager: { baseUrl: string; timeoutMs: number; }; auth: { env: string; file: string; header: string; scheme: string; }; client: { role: string; transport: string; sessionPolicy: AgentRunSessionPolicyConfig; }; publicExposure: AgentRunPublicExposure | null; } interface AgentRunSessionPolicyConfig { sourcePath: string; tenantId: string; projectId: string; providerId: string; backendProfile: string; workspaceRef: Record; executionPolicy: Record; } interface AgentRunPublicExposure { enabled: true; proxyName: string; remotePort: number; publicBaseUrl: string; masterBaseUrl: string; masterFrps: { configPath: string; containerName: string; }; masterCaddy: { enabled: boolean; domain: string; configPath: string; serviceName: string; upstreamBaseUrl: string; responseHeaderTimeoutSeconds: number; }; } interface AgentRunResolvedAuth { source: "env" | "file"; value: string; } type AgentRunHttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found"; type AgentRunRestCompatGroup = "queue" | "sessions" | "aipod-specs" | "aipods" | "runs" | "commands" | "runner"; type AgentRunResourceVerb = "get" | "describe" | "events" | "logs" | "result" | "ack" | "cancel" | "dispatch" | "create" | "apply" | "send" | "explain"; type AgentRunResourceKind = "task" | "run" | "command" | "runnerjob" | "session" | "aipodspec"; type AgentRunOutputMode = "human" | "wide" | "name" | "json" | "yaml"; interface AgentRunResourceRef { kind: AgentRunResourceKind; name: string; } interface AgentRunResourceOptions { output: AgentRunOutputMode; full: boolean; raw: boolean; debug: boolean; limit: number; queue: string | null; state: string | null; unread: boolean; readerId: string; taskId: string | null; runId: string | null; commandId: string | null; sessionId: string | null; afterSeq: number | null; tail: number | null; fullText: boolean; reason: string | null; dryRun: boolean; file: string | null; aipod: string | null; idempotencyKey: string | null; promptStdin: boolean; node: string | null; lane: string | null; passthroughArgs: string[]; } interface AgentRunRestTargetOptions { node: string | null; lane: string | null; } interface AgentRunRestTarget { config: UniDeskConfig; configPath: string; spec: AgentRunLaneSpec; } let activeAgentRunRestTarget: AgentRunRestTarget | null = null; type AgentRunBridgeCaptureBackend = "local-backend-core-broker" | "remote-frontend-websocket"; interface LocalBackendCoreStatus { dockerExecutable: boolean; backendCoreContainer: boolean; error: string | null; } interface AgentRunBridgeCapturePlan { backend: AgentRunBridgeCaptureBackend; route: string; reason: string; remoteHost: string | null; localBackendCore: LocalBackendCoreStatus; } type AgentRunBridgeCaptureResult = SshCaptureResult & { bridgeExecution?: AgentRunBridgeCapturePlan }; let localBackendCoreStatusCache: LocalBackendCoreStatus | null = null; async function capture(config: UniDeskConfig, target: string, args: string[]): Promise { const plan = agentRunBridgeCapturePlan(config, target); if (plan.backend === "remote-frontend-websocket" && plan.remoteHost !== null) { return await captureRemote(config, plan, target, args); } const local = attachBridgeExecution(await runSshCommandCapture(config, target, args), plan); const fallbackHost = agentRunBridgeRemoteHost(config); if (local.exitCode !== 0 && fallbackHost !== null && bridgeExecutionFailureKind(local)?.degradedReason === "capture-backend-unavailable") { const fallbackPlan: AgentRunBridgeCapturePlan = { ...plan, backend: "remote-frontend-websocket", remoteHost: fallbackHost, reason: "local-capture-backend-unavailable", }; return await captureRemote(config, fallbackPlan, target, args); } return local; } async function captureRemote(config: UniDeskConfig, plan: AgentRunBridgeCapturePlan, target: string, args: string[]): Promise { if (plan.remoteHost === null) return attachBridgeExecution(remoteBridgeCaptureFailure(new Error("remote host is not configured")), plan); try { return attachBridgeExecution(await runRemoteSshCommandCapture(config, plan.remoteHost, target, args), plan); } catch (error) { return attachBridgeExecution(remoteBridgeCaptureFailure(error), plan); } } function remoteBridgeCaptureFailure(error: unknown): SshCaptureResult { const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error); return { exitCode: 255, stdout: "", stderr: `unidesk remote frontend ssh bridge failed: ${message}\n`, }; } function attachBridgeExecution(result: SshCaptureResult, plan: AgentRunBridgeCapturePlan): AgentRunBridgeCaptureResult { return { ...result, bridgeExecution: plan }; } function agentRunBridgeCapturePlan(config: UniDeskConfig, target: string, env: NodeJS.ProcessEnv = process.env): AgentRunBridgeCapturePlan { const localBackendCore = detectLocalBackendCoreStatus(); const remoteHost = agentRunBridgeRemoteHost(config, env); const runnerEnv = isAgentRunRunnerEnvironment(env); if (runnerEnv && remoteHost !== null) { return { backend: "remote-frontend-websocket", route: target, reason: "runner-environment", remoteHost, localBackendCore }; } if (!localBackendCore.backendCoreContainer && remoteHost !== null) { return { backend: "remote-frontend-websocket", route: target, reason: "local-backend-core-unavailable", remoteHost, localBackendCore }; } return { backend: "local-backend-core-broker", route: target, reason: "main-server-local-backend-core", remoteHost: null, localBackendCore }; } function detectLocalBackendCoreStatus(): LocalBackendCoreStatus { if (localBackendCoreStatusCache !== null) return localBackendCoreStatusCache; const result = spawnSync("docker", ["ps", "--format", "{{.Names}}"], { encoding: "utf8", timeout: 2000 }); if (result.error !== undefined) { localBackendCoreStatusCache = { dockerExecutable: false, backendCoreContainer: false, error: result.error.message, }; return localBackendCoreStatusCache; } const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim(); localBackendCoreStatusCache = { dockerExecutable: result.status === 0, backendCoreContainer: result.status === 0 && String(result.stdout ?? "").split(/\r?\n/u).includes("unidesk-backend-core"), error: result.status === 0 ? null : output || `docker ps exited ${result.status ?? "unknown"}`, }; return localBackendCoreStatusCache; } function isAgentRunRunnerEnvironment(env: NodeJS.ProcessEnv): boolean { return Boolean( env.AGENTRUN_BOOT_MODE || env.AGENTRUN_RUN_ID || env.AGENTRUN_K8S_JOB_NAME || env.CODE_QUEUE_SERVICE_ROLE || env.CODE_QUEUE_INSTANCE_ID || env.KUBERNETES_SERVICE_HOST, ); } function agentRunBridgeRemoteHost(config: UniDeskConfig, env: NodeJS.ProcessEnv = process.env): string | null { return normalizeRemoteHostHint(env.UNIDESK_MAIN_SERVER_IP) ?? normalizeRemoteHostHint(env.UNIDESK_MAIN_SERVER_HOST) ?? normalizeRemoteHostHint(env.CODE_QUEUE_DEV_CONTAINER_MASTER_HOST) ?? normalizeRemoteHostHint(config.network.publicHost); } function normalizeRemoteHostHint(raw: string | undefined): string | null { const value = raw?.trim() ?? ""; if (value.length === 0 || value === "localhost" || value === "127.0.0.1" || value === "::1") return null; return value.replace(/\/+$/u, ""); } function captureBridgeMetadata(result: SshCaptureResult): Record | null { const bridgeExecution = (result as AgentRunBridgeCaptureResult).bridgeExecution; if (bridgeExecution === undefined) return null; return { backend: bridgeExecution.backend, route: bridgeExecution.route, reason: bridgeExecution.reason, remoteHost: bridgeExecution.remoteHost, localBackendCore: bridgeExecution.localBackendCore, }; } function bridgeExecutionFailureKind(result: SshCaptureResult): { degradedReason: string; failureKind: string; recoveryActions: string[] } | null { if (result.exitCode === 0) return null; const stderr = result.stderr; if (/No such container: unidesk-backend-core|failed to start broker|Executable not found.*"docker"|docker: not found|Cannot connect to the Docker daemon/iu.test(stderr)) { return { degradedReason: "capture-backend-unavailable", failureKind: "bridge-execution-environment", recoveryActions: [ "Run the same command from a healthy UniDesk main-server CLI, or provide UNIDESK_MAIN_SERVER_IP/UNIDESK_MAIN_SERVER_HOST so the bridge can use the frontend WebSocket SSH backend.", "For AgentRun runner jobs, request the unidesk-ssh tool credential SecretRef instead of embedding secret values in prompts or payloads.", "After restoring the bridge backend, retry the same bun scripts/cli.ts agentrun ... command; do not use direct kubectl or GitHub API bypasses.", ], }; } if (/frontend login failed|remote frontend ssh bridge|websocket error|timed out waiting for provider session|route-not-allowed/iu.test(stderr)) { return { degradedReason: "bridge-execution-environment-unavailable", failureKind: "bridge-execution-environment", recoveryActions: [ "Verify the UniDesk frontend/backend-core SSH bridge is reachable from this runner and that UNIDESK_MAIN_SERVER_IP or UNIDESK_MAIN_SERVER_HOST points at the main server.", "Verify scoped UNIDESK_SSH_CLIENT_TOKEN route allowlist includes the requested AgentRun route, without printing token values.", "Retry with the same bun scripts/cli.ts agentrun ... command after the controlled bridge path is restored.", ], }; } return null; } function compactCapture(result: SshCaptureResult, options: { full?: boolean; stdoutTailChars?: number; stderrTailChars?: number } = {}): Record { const stdoutTailChars = options.stdoutTailChars ?? 8000; const stderrTailChars = options.stderrTailChars ?? 4000; const full = options.full ?? true; const payload: Record = { exitCode: result.exitCode, stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), stderrBytes: Buffer.byteLength(result.stderr, "utf8"), stdoutTailOmitted: !full && result.exitCode === 0, stderrTailOmitted: !full && result.exitCode === 0, }; const bridgeExecution = captureBridgeMetadata(result); if (bridgeExecution !== null) payload.bridgeExecution = bridgeExecution; const failureKind = bridgeExecutionFailureKind(result); if (failureKind !== null) { payload.degradedReason = failureKind.degradedReason; payload.failureKind = failureKind.failureKind; payload.recoveryActions = failureKind.recoveryActions; } if (full || result.exitCode !== 0) { payload.stdoutTail = tail(result.stdout, stdoutTailChars); payload.stderrTail = tail(result.stderr, stderrTailChars); payload.stdoutTruncated = result.stdout.length > stdoutTailChars; payload.stderrTruncated = result.stderr.length > stderrTailChars; } return payload; } async function timedStatusStage(stage: string, action: () => Promise): Promise> { const startedAtMs = Date.now(); progressEvent("agentrun.control-plane.status.progress", { stage, status: "started" }); try { const value = await action(); const exitCode = isCaptureResult(value) ? value.exitCode : typeof record(value).ok === "boolean" && record(value).ok !== true ? 1 : 0; progressEvent("agentrun.control-plane.status.progress", { stage, status: exitCode === 0 ? "succeeded" : "failed", exitCode, elapsedMs: Date.now() - startedAtMs, }); return { value, elapsedMs: Date.now() - startedAtMs }; } catch (error) { progressEvent("agentrun.control-plane.status.progress", { stage, status: "failed", elapsedMs: Date.now() - startedAtMs, error: error instanceof Error ? error.message : String(error), }); throw error; } } function progressEvent(event: string, payload: Record): void { const stage = displayValue(payload.stage ?? event); const status = displayValue(payload.status ?? "-"); const details = [ payload.exitCode === undefined ? null : `exit=${displayValue(payload.exitCode)}`, payload.elapsedMs === undefined ? null : `elapsed=${displayValue(payload.elapsedMs)}ms`, payload.error === undefined ? null : `error=${truncateOneLine(displayValue(payload.error), 160)}`, ].filter((item): item is string => item !== null); process.stderr.write(`status ${stage} ${status}${details.length === 0 ? "" : ` ${details.join(" ")}`}\n`); } function isCaptureResult(value: unknown): value is SshCaptureResult { return typeof value === "object" && value !== null && "exitCode" in value && typeof (value as { exitCode?: unknown }).exitCode === "number"; } function isAgentRunPipelineRunName(value: string): boolean { return /^agentrun-v[0-9]+-ci-[0-9a-f]{12,40}(?:-[a-z0-9-]+)?$/u.test(value); } function captureJsonPayload(result: SshCaptureResult): Record { const trimmed = result.stdout.trim(); if (trimmed.length === 0) return {}; try { return record(JSON.parse(trimmed) as unknown); } catch { const lastJsonLine = trimmed.split(/\r?\n/u).reverse().find((line) => line.trim().startsWith("{") && line.trim().endsWith("}")); if (lastJsonLine === undefined) return {}; try { return record(JSON.parse(lastJsonLine) as unknown); } catch { return {}; } } } function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function nonNegativeIntegerOrNull(value: unknown): number | null { if (typeof value === "number" && Number.isInteger(value) && value >= 0) return value; if (typeof value === "string" && /^\d+$/u.test(value)) return Number(value); return null; } function isGitSha(value: string): boolean { return /^[0-9a-f]{40}$/u.test(value); } function tail(text: string, maxChars: number): string { return text.length > maxChars ? text.slice(-maxChars) : text; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function shQuote(value: string): string { return `'${value.replace(/'/gu, "'\\''")}'`; } function unsupported(args: string[]): RenderedCliResult { const command = `agentrun ${args.join(" ")}`.trim(); return renderedCliResult(false, command, [ `Error: unsupported AgentRun command: ${command}`, "", "Supported resource commands:", " agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send", "", "Compatibility bridge groups:", " agentrun aipod-specs|queue|runs|commands|runner|sessions", "", "Operations:", " agentrun control-plane status|trigger-current|refresh", " agentrun git-mirror status|sync|flush", ].join("\n")); }