Codex
7170 lines
348 KiB
TypeScript
7170 lines
348 KiB
TypeScript
import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { spawnSync } from "node:child_process";
|
|
import { rootPath, type UniDeskConfig } from "./config";
|
|
import type { RenderedCliResult } from "./output";
|
|
import { applyLocalCaddyManagedSite } from "./pk01-caddy";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
|
import { runRemoteSshCommandCapture } from "./remote";
|
|
import { startJob } from "./jobs";
|
|
import {
|
|
AGENTRUN_CONFIG_PATH,
|
|
agentRunLaneSummary,
|
|
agentRunPipelineRunName,
|
|
agentRunProviderCredentialRefs,
|
|
resolveAgentRunLaneTarget,
|
|
type AgentRunLaneSpec,
|
|
} from "./agentrun-lanes";
|
|
import {
|
|
agentRunImageArtifact,
|
|
placeholderAgentRunImage,
|
|
renderedFilesDigest,
|
|
renderedObjectsDigest,
|
|
renderAgentRunControlPlaneManifests,
|
|
renderAgentRunGitopsFiles,
|
|
type AgentRunArtifactService,
|
|
} from "./agentrun-manifests";
|
|
import { sha256Fingerprint } from "./platform-infra-ops-library";
|
|
|
|
export function agentRunHelp(): unknown {
|
|
return {
|
|
command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|explain",
|
|
output: "human by default; use -o json|yaml or --raw for machine/debug output",
|
|
usage: [
|
|
"bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
|
|
"bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
|
|
"bun scripts/cli.ts agentrun get tasks -o wide",
|
|
"bun scripts/cli.ts agentrun get tasks -o json",
|
|
"bun scripts/cli.ts agentrun describe task/<taskId>",
|
|
"bun scripts/cli.ts agentrun events run/<runId> --after-seq 0 --limit 100",
|
|
"bun scripts/cli.ts agentrun logs session/<sessionId> --tail 100",
|
|
"bun scripts/cli.ts agentrun result run/<runId> --command <commandId>",
|
|
"bun scripts/cli.ts agentrun ack task/<taskId> --reader-id cli",
|
|
"bun scripts/cli.ts agentrun cancel task/<taskId> --reason <text> --dry-run",
|
|
"bun scripts/cli.ts agentrun dispatch task/<taskId>",
|
|
"bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin --idempotency-key <key>",
|
|
"bun scripts/cli.ts agentrun apply -f - --dry-run",
|
|
"bun scripts/cli.ts agentrun send session/<sessionId> --aipod Artificer --prompt-stdin",
|
|
"bun scripts/cli.ts agentrun explain task",
|
|
"bun scripts/cli.ts agentrun explain session-policy",
|
|
"bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02",
|
|
"bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02",
|
|
"bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane status",
|
|
"bun scripts/cli.ts agentrun control-plane status --full",
|
|
"bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-vNN-ci-<short-sha>",
|
|
"bun scripts/cli.ts agentrun control-plane status --source-commit <full-sha>",
|
|
"bun scripts/cli.ts agentrun control-plane expose --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane expose --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane trigger-current --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane trigger-current --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane refresh --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane refresh --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm",
|
|
"bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --dry-run",
|
|
"bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --confirm",
|
|
"bun scripts/cli.ts agentrun git-mirror status",
|
|
"bun scripts/cli.ts agentrun git-mirror status --full",
|
|
"bun scripts/cli.ts agentrun git-mirror sync --confirm",
|
|
"bun scripts/cli.ts agentrun git-mirror flush --confirm",
|
|
],
|
|
resources: ["task/qt", "run", "command/cmd", "runnerjob/rjob", "session/ses", "aipodspec/aps"],
|
|
description: "Operate AgentRun through Kubernetes-style resource verbs. Human output is compact by default; -o json|yaml returns the UniDesk render-only client schema, --raw exposes the REST envelope, and --node/--lane targets a YAML-declared runtime lane.",
|
|
legacyCompatibility: "queue/runs/commands/runner/sessions/aipod-specs remain as compatibility groups backed by direct HTTP; new commander work should use get/describe/events/logs/result/ack/cancel/dispatch/create/apply/send. sessions turn/steer are removed; use send only.",
|
|
};
|
|
}
|
|
|
|
export async function runAgentRunCommand(config: UniDeskConfig | null, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const route = normalizeAgentRunCommandArgs(args);
|
|
const { group, action, actionArgs } = route;
|
|
if (group === undefined || isHelpArg(group)) return renderAgentRunHelp(route.canonicalArgs);
|
|
if (isHelpArg(action) || actionArgs.some(isHelpArg)) return renderAgentRunHelp(route.canonicalArgs);
|
|
if (isResourceVerb(group)) return await runAgentRunResourceCommand(config, group, action, actionArgs, route.canonicalArgs);
|
|
if (isAgentRunRestCompatGroup(group)) {
|
|
return await runAgentRunRestCompatCommand(config, group, route.forwardArgs, route.canonicalArgs);
|
|
}
|
|
if (config === null) throw new Error("agentrun control-plane and git-mirror commands require UniDesk config");
|
|
if (group === "control-plane") {
|
|
if (action === "plan") return await controlPlanePlan(config, parseStatusOptions(actionArgs));
|
|
if (action === "apply") return await controlPlaneApply(config, parseLaneConfirmOptions(actionArgs));
|
|
if (action === "status") {
|
|
const options = parseStatusOptions(actionArgs);
|
|
const result = await status(config, options);
|
|
return options.full || options.raw ? result : renderAgentRunControlPlaneStatusSummary(result);
|
|
}
|
|
if (action === "secret-sync") return await secretSync(config, parseSecretSyncOptions(actionArgs));
|
|
if (action === "restart") return await restartYamlLane(config, parseLaneConfirmOptions(actionArgs));
|
|
if (action === "expose") return await exposeAgentRun(config, parseConfirmOptions(actionArgs));
|
|
if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(actionArgs));
|
|
if (action === "refresh") return await refresh(config, parseRefreshOptions(actionArgs));
|
|
if (action === "cleanup-runners") return await cleanupRunners(config, parseCleanupRunnersOptions(actionArgs));
|
|
if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(actionArgs));
|
|
if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs));
|
|
}
|
|
if (group === "git-mirror") {
|
|
if (action === "status") return await gitMirrorStatus(config, parseGitMirrorStatusOptions(actionArgs));
|
|
if (action === "sync" || action === "flush") {
|
|
const options = parseGitMirrorOptions(actionArgs);
|
|
const { spec } = resolveAgentRunLaneTarget(options);
|
|
if (options.confirm && !options.wait) {
|
|
return startAsyncAgentRunJob(
|
|
`agentrun_${spec.lane}_git_mirror_${action}`,
|
|
["bun", "scripts/cli.ts", "agentrun", "git-mirror", action, "--node", spec.nodeId, "--lane", spec.lane, "--confirm", "--wait", "--timeout-seconds", String(options.timeoutSeconds)],
|
|
`Run AgentRun ${spec.version} git mirror ${action} on ${spec.nodeId}`,
|
|
);
|
|
}
|
|
return await runGitMirrorJob(config, action, options);
|
|
}
|
|
}
|
|
return unsupported(route.canonicalArgs);
|
|
}
|
|
|
|
function isAgentRunRestCompatGroup(group: string | undefined): group is AgentRunRestCompatGroup {
|
|
return group === "queue" || group === "sessions" || group === "aipod-specs" || group === "aipods" || group === "runs" || group === "commands" || group === "runner";
|
|
}
|
|
|
|
interface AgentRunCommandRoute {
|
|
canonicalArgs: string[];
|
|
group: string | undefined;
|
|
action: string | undefined;
|
|
actionArgs: string[];
|
|
forwardArgs: string[];
|
|
}
|
|
|
|
function normalizeAgentRunCommandArgs(args: string[]): AgentRunCommandRoute {
|
|
const canonicalArgs = args[0] === "v01" ? args.slice(1) : args;
|
|
const [group, action, ...actionArgs] = canonicalArgs;
|
|
return {
|
|
canonicalArgs,
|
|
group,
|
|
action,
|
|
actionArgs,
|
|
forwardArgs: canonicalArgs.slice(1),
|
|
};
|
|
}
|
|
|
|
function isHelpArg(value: string | undefined): boolean {
|
|
return value === "help" || value === "--help" || value === "-h";
|
|
}
|
|
|
|
function isResourceVerb(value: string | undefined): value is AgentRunResourceVerb {
|
|
return value === "get"
|
|
|| value === "describe"
|
|
|| value === "events"
|
|
|| value === "logs"
|
|
|| value === "result"
|
|
|| value === "ack"
|
|
|| value === "cancel"
|
|
|| value === "dispatch"
|
|
|| value === "create"
|
|
|| value === "apply"
|
|
|| value === "send"
|
|
|| value === "explain";
|
|
}
|
|
|
|
function renderAgentRunHelp(args: string[]): RenderedCliResult {
|
|
const text = agentRunHelpText(args);
|
|
return renderedCliResult(true, "agentrun help", text);
|
|
}
|
|
|
|
function agentRunHelpText(args: string[]): string {
|
|
const [verb, kind] = args.filter((arg) => !isHelpArg(arg));
|
|
if (verb === "get" && kind !== undefined) return agentRunGetKindHelp(kind);
|
|
if (verb === "get") {
|
|
return [
|
|
"Usage: bun scripts/cli.ts agentrun get <resource> [options]",
|
|
"",
|
|
"Resources: tasks|sessions|runs|commands|runnerjobs|aipodspecs",
|
|
"Output: table by default; use -o wide|name|json|yaml.",
|
|
"",
|
|
"Examples:",
|
|
" bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
|
|
" bun scripts/cli.ts agentrun get sessions --limit 20",
|
|
" bun scripts/cli.ts agentrun get tasks -o json",
|
|
].join("\n");
|
|
}
|
|
if (verb === "describe") {
|
|
return [
|
|
"Usage: bun scripts/cli.ts agentrun describe <kind/name> [--node <node> --lane <lane>] [--full] [-o json|yaml] [--raw]",
|
|
"",
|
|
"Kinds: task|run|command|runnerjob|session|aipodspec",
|
|
"Examples:",
|
|
" bun scripts/cli.ts agentrun describe task/qt_...",
|
|
" bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
|
|
" bun scripts/cli.ts agentrun describe command/cmd_... --run <runId>",
|
|
" bun scripts/cli.ts agentrun describe session/<sessionId> --full",
|
|
].join("\n");
|
|
}
|
|
if (verb === "events") {
|
|
return "Usage: bun scripts/cli.ts agentrun events run/<runId> [--after-seq N] [--limit 100] [-o json|yaml] [--raw]";
|
|
}
|
|
if (verb === "logs") {
|
|
return "Usage: bun scripts/cli.ts agentrun logs session/<sessionId> [--tail 100|--after-seq N] [--limit 100] [--full-text] [-o json|yaml] [--raw]";
|
|
}
|
|
if (verb === "result") {
|
|
return "Usage: bun scripts/cli.ts agentrun result run/<runId> --command <commandId> | agentrun result command/<commandId> --run <runId>";
|
|
}
|
|
if (verb === "ack") {
|
|
return "Usage: bun scripts/cli.ts agentrun ack task/<taskId>|session/<sessionId> [--reader-id cli]";
|
|
}
|
|
if (verb === "cancel") {
|
|
return "Usage: bun scripts/cli.ts agentrun cancel task/<taskId>|session/<sessionId>|run/<runId>|command/<commandId> --reason <text> [--run <runId>] [--dry-run]";
|
|
}
|
|
if (verb === "dispatch") {
|
|
return "Usage: bun scripts/cli.ts agentrun dispatch task/<taskId>";
|
|
}
|
|
if (verb === "create") {
|
|
return "Usage: bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin [--idempotency-key <key>] [--dry-run]";
|
|
}
|
|
if (verb === "apply") {
|
|
return "Usage: bun scripts/cli.ts agentrun apply -f task.yaml|json|- [--dry-run]\nTask manifests use kind: Task and spec: <AgentRun task payload>.";
|
|
}
|
|
if (verb === "send") {
|
|
return "Usage: bun scripts/cli.ts agentrun send session/<sessionId> --aipod Artificer --prompt-stdin\nThe server decides whether this becomes an internal steer or a new turn from session state.";
|
|
}
|
|
if (verb === "explain") return agentRunExplain(kind ?? "task");
|
|
if (verb === "control-plane") {
|
|
return [
|
|
"Usage: bun scripts/cli.ts agentrun control-plane <action> [options]",
|
|
"",
|
|
"Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-released-pvs",
|
|
"Examples:",
|
|
" bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02",
|
|
" bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run",
|
|
" bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02",
|
|
" bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run",
|
|
" bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run",
|
|
" bun scripts/cli.ts agentrun control-plane status",
|
|
" bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-vNN-ci-<short-sha>",
|
|
" bun scripts/cli.ts agentrun control-plane expose --dry-run",
|
|
" bun scripts/cli.ts agentrun control-plane trigger-current --dry-run",
|
|
" bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run",
|
|
" bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
|
].join("\n");
|
|
}
|
|
if (verb === "git-mirror") {
|
|
return [
|
|
"Usage: bun scripts/cli.ts agentrun git-mirror <status|sync|flush> [--full|--raw|--confirm]",
|
|
"",
|
|
"Confirmed sync/flush returns an async job unless --wait is set.",
|
|
].join("\n");
|
|
}
|
|
if (verb !== undefined && isAgentRunRestCompatGroup(verb)) {
|
|
return [
|
|
`Compatibility group: agentrun ${verb} ...`,
|
|
"",
|
|
"Use resource primitives for daily commander work:",
|
|
" bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
|
|
" bun scripts/cli.ts agentrun describe task/<taskId>",
|
|
" bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
|
|
" bun scripts/cli.ts agentrun events run/<runId> --after-seq 0 --limit 100",
|
|
" bun scripts/cli.ts agentrun logs session/<sessionId> --tail 100",
|
|
"",
|
|
"Use --raw on a resource command when you need the direct AgentRun REST envelope.",
|
|
].join("\n");
|
|
}
|
|
return [
|
|
"Usage: bun scripts/cli.ts agentrun <verb> <resource> [options]",
|
|
"",
|
|
"Verbs: get, describe, events, logs, result, ack, cancel, dispatch, create, apply, send, explain",
|
|
"Resources: task/qt, run, command/cmd, runnerjob/rjob, session/ses, aipodspec/aps",
|
|
"",
|
|
"Common:",
|
|
" bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
|
|
" bun scripts/cli.ts agentrun describe task/<taskId>",
|
|
" bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
|
|
" bun scripts/cli.ts agentrun events run/<runId> --after-seq 0 --limit 100",
|
|
" bun scripts/cli.ts agentrun logs session/<sessionId> --tail 100",
|
|
" bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin",
|
|
"",
|
|
"Machine/debug output:",
|
|
" -o json|yaml emits a stable UniDesk resource object without the global JSON envelope.",
|
|
" --raw emits the direct AgentRun REST envelope.",
|
|
" --node/--lane targets a YAML-declared AgentRun runtime lane; default remains the configured public manager.",
|
|
].join("\n");
|
|
}
|
|
|
|
function agentRunGetKindHelp(kindRaw: string): string {
|
|
const kind = parseResourceKind(kindRaw);
|
|
if (kind === "task") return "Usage: bun scripts/cli.ts agentrun get tasks [--queue commander] [--state running,pending,completed,failed] [--unread] [--limit 20] [-o wide|name|json|yaml]";
|
|
if (kind === "session") return "Usage: bun scripts/cli.ts agentrun get sessions [--limit 20] [-o wide|name|json|yaml]";
|
|
if (kind === "run") return "Usage: bun scripts/cli.ts agentrun get runs --task <taskId> [--limit 20] [-o wide|name|json|yaml]";
|
|
if (kind === "command") return "Usage: bun scripts/cli.ts agentrun get commands --run <runId> [--command <commandId>] [-o wide|name|json|yaml]";
|
|
if (kind === "runnerjob") return "Usage: bun scripts/cli.ts agentrun get runnerjobs --run <runId> --command <commandId> [-o wide|name|json|yaml]";
|
|
if (kind === "aipodspec") return "Usage: bun scripts/cli.ts agentrun get aipodspecs [-o wide|name|json|yaml]";
|
|
return "Unknown resource. Supported: tasks, sessions, runs, commands, runnerjobs, aipodspecs.";
|
|
}
|
|
|
|
async function runAgentRunResourceCommand(config: UniDeskConfig | null, verb: AgentRunResourceVerb, action: string | undefined, actionArgs: string[], canonicalArgs: string[]): Promise<RenderedCliResult> {
|
|
if (isHelpArg(action) || actionArgs.some(isHelpArg)) return renderAgentRunHelp(canonicalArgs);
|
|
if (verb === "explain") return renderedCliResult(true, "agentrun explain", agentRunExplain(action ?? "task"));
|
|
const resourceArgs = action === undefined ? actionArgs : [action, ...actionArgs];
|
|
const options = parseResourceOptions(resourceArgs);
|
|
const bridgeActionArgs = stripAgentRunResourceWrapperArgs(actionArgs);
|
|
const command = `agentrun ${canonicalArgs.join(" ")}`.trim();
|
|
try {
|
|
return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, options), async () => {
|
|
if (verb === "get") return await resourceGet(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "describe") return await resourceDescribe(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "events") return await resourceEvents(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "logs") return await resourceLogs(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "result") return await resourceResult(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "ack") return await resourceAck(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "cancel") return await resourceCancel(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "dispatch") return await resourceDispatch(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "create") return await resourceCreate(config, command, action, bridgeActionArgs, options);
|
|
if (verb === "apply") return await resourceApply(config, command, bridgeActionArgs, options);
|
|
if (verb === "send") return await resourceSessionPromptCommand(config, command, action, bridgeActionArgs, options);
|
|
return renderedCliResult(false, command, `Unsupported AgentRun resource command. Try: bun scripts/cli.ts agentrun --help`);
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof AgentRunRestError) return renderAgentRunRestError(command, error, options);
|
|
return renderedCliResult(false, command, `Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
return renderedCliResult(false, command, `Unsupported AgentRun resource command. Try: bun scripts/cli.ts agentrun --help`);
|
|
}
|
|
|
|
export function stripAgentRunResourceWrapperArgs(args: string[]): string[] {
|
|
const result: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (arg === "-o" || arg === "--output") {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--node" || arg === "--lane") {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--output=") || arg.startsWith("-o=")) continue;
|
|
if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue;
|
|
if (arg === "--full" || arg === "--raw" || arg === "--debug" || arg === "--full-text") continue;
|
|
result.push(arg);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function parseResourceOptions(args: string[]): AgentRunResourceOptions {
|
|
const options: AgentRunResourceOptions = {
|
|
output: "human",
|
|
full: false,
|
|
raw: false,
|
|
debug: false,
|
|
limit: 20,
|
|
queue: null,
|
|
state: null,
|
|
unread: false,
|
|
readerId: "cli",
|
|
taskId: null,
|
|
runId: null,
|
|
commandId: null,
|
|
sessionId: null,
|
|
afterSeq: null,
|
|
tail: null,
|
|
fullText: false,
|
|
reason: null,
|
|
dryRun: false,
|
|
file: null,
|
|
aipod: null,
|
|
idempotencyKey: null,
|
|
promptStdin: false,
|
|
node: null,
|
|
lane: null,
|
|
passthroughArgs: [],
|
|
};
|
|
const valueFlags = new Set(["-o", "--output", "--limit", "--queue", "--state", "--reader-id", "--task", "--task-id", "--run", "--run-id", "--command", "--command-id", "--session", "--session-id", "--after-seq", "--tail", "--reason", "-f", "--file", "--filename", "--aipod", "--idempotency-key", "--node", "--lane"]);
|
|
const booleanFlags = new Set(["--full", "--raw", "--debug", "--unread", "--dry-run", "--full-text", "--prompt-stdin", "--stdin"]);
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (!arg.startsWith("-")) continue;
|
|
if (arg.startsWith("--") && arg.includes("=")) {
|
|
const [flag, ...rest] = arg.split("=");
|
|
applyResourceOption(options, flag, rest.join("="));
|
|
continue;
|
|
}
|
|
if (booleanFlags.has(arg)) {
|
|
applyResourceOption(options, arg, null);
|
|
continue;
|
|
}
|
|
if (valueFlags.has(arg)) {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
|
applyResourceOption(options, arg, value);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function applyResourceOption(options: AgentRunResourceOptions, flag: string, value: string | null): void {
|
|
if (flag === "-o" || flag === "--output") {
|
|
if (value !== "human" && value !== "wide" && value !== "name" && value !== "json" && value !== "yaml") throw new Error(`${flag} must be one of human,wide,name,json,yaml`);
|
|
options.output = value;
|
|
return;
|
|
}
|
|
if (flag === "--full") options.full = true;
|
|
else if (flag === "--raw") options.raw = true;
|
|
else if (flag === "--debug") options.debug = true;
|
|
else if (flag === "--unread") options.unread = true;
|
|
else if (flag === "--dry-run") options.dryRun = true;
|
|
else if (flag === "--full-text") options.fullText = true;
|
|
else if (flag === "--prompt-stdin" || flag === "--stdin") options.promptStdin = true;
|
|
else if (flag === "--limit") options.limit = parseNonNegativeInt(value, "--limit", 20, 500);
|
|
else if (flag === "--queue") options.queue = requiredValue(value, flag);
|
|
else if (flag === "--state") options.state = requiredValue(value, flag);
|
|
else if (flag === "--reader-id") options.readerId = requiredValue(value, flag);
|
|
else if (flag === "--task" || flag === "--task-id") options.taskId = requiredValue(value, flag);
|
|
else if (flag === "--run" || flag === "--run-id") options.runId = requiredValue(value, flag);
|
|
else if (flag === "--command" || flag === "--command-id") options.commandId = requiredValue(value, flag);
|
|
else if (flag === "--session" || flag === "--session-id") options.sessionId = requiredValue(value, flag);
|
|
else if (flag === "--after-seq") options.afterSeq = parseNonNegativeInt(value, "--after-seq", 0, Number.MAX_SAFE_INTEGER);
|
|
else if (flag === "--tail") options.tail = parseNonNegativeInt(value, "--tail", 100, 1000);
|
|
else if (flag === "--reason") options.reason = requiredValue(value, flag);
|
|
else if (flag === "-f" || flag === "--file" || flag === "--filename") options.file = requiredValue(value, flag);
|
|
else if (flag === "--aipod") options.aipod = requiredValue(value, flag);
|
|
else if (flag === "--idempotency-key") options.idempotencyKey = requiredValue(value, flag);
|
|
else if (flag === "--node") options.node = requiredValue(value, flag);
|
|
else if (flag === "--lane") options.lane = requiredValue(value, flag);
|
|
}
|
|
|
|
function parseNonNegativeInt(raw: string | null, flag: string, defaultValue: number, maxValue: number): number {
|
|
if (raw === null || raw.length === 0) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${flag} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
function requiredValue(value: string | null, flag: string): string {
|
|
if (value === null || value.length === 0) throw new Error(`${flag} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
async function resourceGet(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const kind = parseResourceKind(action);
|
|
if (kind === null) throw new Error("get requires a resource: tasks|sessions|runs|commands|runnerjobs|aipodspecs");
|
|
let result: Record<string, unknown>;
|
|
if (kind === "task") {
|
|
const taskListArgs = options.unread
|
|
? ["commander", "--reader-id", options.readerId, "--limit", String(options.limit)]
|
|
: [
|
|
"list",
|
|
"--state",
|
|
taskListState(options),
|
|
"--limit",
|
|
String(options.queue === null ? options.limit : Math.max(options.limit, 100)),
|
|
...(options.queue === null ? [] : ["--queue", options.queue]),
|
|
];
|
|
result = await runAgentRunRestCommand(config, "queue", taskListArgs);
|
|
return renderResourceResult(command, result, options, "Task", normalizeTaskItems(innerData(result), options).slice(0, options.limit));
|
|
}
|
|
if (kind === "session") {
|
|
result = await runAgentRunRestCommand(config, "sessions", ["ps", "--limit", String(options.limit)]);
|
|
return renderResourceResult(command, result, options, "Session", normalizeSessionItems(innerData(result)).slice(0, options.limit));
|
|
}
|
|
if (kind === "aipodspec") {
|
|
result = await runAgentRunRestCommand(config, "aipod-specs", ["list"]);
|
|
return renderResourceResult(command, result, options, "AipodSpec", normalizeAipodSpecItems(innerData(result)).slice(0, options.limit));
|
|
}
|
|
if (kind === "run" && options.taskId !== null) {
|
|
result = await runAgentRunRestCommand(config, "queue", ["show", options.taskId]);
|
|
const task = record(innerData(result));
|
|
return renderResourceResult(command, result, options, "Run", normalizeAttemptResources(task, "run"));
|
|
}
|
|
if (kind === "command" && options.runId !== null && options.commandId !== null) {
|
|
result = await runAgentRunRestCommand(config, "commands", ["show", options.commandId, "--run-id", options.runId]);
|
|
return renderResourceResult(command, result, options, "Command", normalizeSingleCommand(innerData(result)));
|
|
}
|
|
if (kind === "runnerjob" && options.runId !== null && options.commandId !== null) {
|
|
result = await runAgentRunRestCommand(config, "runner", ["jobs", "--run-id", options.runId, "--command-id", options.commandId]);
|
|
return renderResourceResult(command, result, options, "RunnerJob", normalizeRunnerJobItems(innerData(result)));
|
|
}
|
|
throw new Error(`get ${kind}s requires more context; use --task, --run/--command, or describe <kind>/<id>`);
|
|
}
|
|
|
|
async function resourceDescribe(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args);
|
|
if (ref.kind === "task") {
|
|
const result = await runAgentRunRestCommand(config, "queue", ["show", ref.name, ...(options.full ? ["--full"] : [])]);
|
|
const data = record(innerData(result));
|
|
const task = unwrapTaskDetail(data);
|
|
if (options.raw) return renderMachine(command, result, "json", result.ok !== false);
|
|
if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: ref.kind, name: ref.name, resource: task }, options.output, result.ok !== false);
|
|
return renderedCliResult(result.ok !== false, command, renderTaskDescription(task, options));
|
|
}
|
|
if (ref.kind === "run") {
|
|
const result = await runAgentRunRestCommand(config, "runs", ["show", ref.name, ...(options.full ? ["--full"] : [])]);
|
|
return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result)));
|
|
}
|
|
if (ref.kind === "command") {
|
|
const runId = options.runId ?? requiredContext("command describe", "--run <runId>");
|
|
const result = await runAgentRunRestCommand(config, "commands", ["show", ref.name, "--run-id", runId, ...(options.full ? ["--full"] : [])]);
|
|
return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result)));
|
|
}
|
|
if (ref.kind === "runnerjob") {
|
|
const runId = options.runId ?? requiredContext("runnerjob describe", "--run <runId>");
|
|
const result = await runAgentRunRestCommand(config, "runner", ["job-status", ref.name, "--run-id", runId]);
|
|
return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result)));
|
|
}
|
|
if (ref.kind === "session") {
|
|
const result = await runAgentRunRestCommand(config, "sessions", ["show", ref.name, ...(options.full ? ["--full"] : [])]);
|
|
return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result)));
|
|
}
|
|
if (ref.kind === "aipodspec") {
|
|
const result = await runAgentRunRestCommand(config, "aipod-specs", ["show", ref.name, ...(options.full ? ["--full"] : [])]);
|
|
return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result)));
|
|
}
|
|
throw new Error(`unsupported describe kind: ${ref.kind}`);
|
|
}
|
|
|
|
async function resourceEvents(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args, "run");
|
|
const runId = ref.kind === "run" ? ref.name : options.runId ?? requiredContext("events", "--run <runId>");
|
|
const eventArgs = ["events", runId, "--after-seq", String(options.afterSeq ?? 0), "--limit", String(options.limit), "--tail-summary"];
|
|
const result = await runAgentRunRestCommand(config, "runs", eventArgs);
|
|
return renderEventLike(command, result, options, "Event", normalizeEventItems(innerData(result)), runId);
|
|
}
|
|
|
|
async function resourceLogs(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args, "session");
|
|
if (ref.kind !== "session") throw new Error("logs currently requires session/<sessionId>");
|
|
const effectiveLimit = options.tail ?? options.limit;
|
|
const logArgs = ["output", ref.name, "--limit", String(effectiveLimit)];
|
|
if (options.afterSeq !== null) logArgs.push("--after-seq", String(options.afterSeq));
|
|
if (options.fullText) logArgs.push("--full-text");
|
|
let result: Record<string, unknown>;
|
|
if (options.afterSeq === null && options.tail !== null) {
|
|
result = await resourceLogsTailResult(config, ref.name, effectiveLimit, options.fullText);
|
|
} else {
|
|
result = await runAgentRunRestCommand(config, "sessions", logArgs);
|
|
}
|
|
return renderEventLike(command, result, { ...options, limit: effectiveLimit }, "Log", normalizeLogItems(innerData(result)), ref.name);
|
|
}
|
|
|
|
async function resourceLogsTailResult(config: UniDeskConfig | null, sessionId: string, limit: number, fullText: boolean): Promise<Record<string, unknown>> {
|
|
const session = record(innerData(await runAgentRunRestCommand(config, "sessions", ["show", sessionId])));
|
|
const lastSeq = nonNegativeIntegerOrNull(session.lastEventSeq) ?? 0;
|
|
let window = Math.max(limit * 4, 100);
|
|
let result: Record<string, unknown> | null = null;
|
|
let data: Record<string, unknown> = {};
|
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
const afterSeq = Math.max(0, lastSeq - window);
|
|
const queryLimit = Math.max(limit, Math.min(1000, window));
|
|
const args = ["output", sessionId, "--limit", String(queryLimit), "--after-seq", String(afterSeq)];
|
|
if (fullText) args.push("--full-text");
|
|
result = await runAgentRunRestCommand(config, "sessions", args);
|
|
data = record(innerData(result));
|
|
const items = arrayRecords(data.items);
|
|
if (items.length >= limit || afterSeq === 0) break;
|
|
window *= 2;
|
|
}
|
|
if (result === null) return await runAgentRunRestCommand(config, "sessions", ["output", sessionId, "--limit", String(limit), "--after-seq", "0"]);
|
|
const items = arrayRecords(data.items);
|
|
const tailedItems = items.slice(-limit);
|
|
const lastReturnedSeq = tailedItems.length > 0 ? nonNegativeIntegerOrNull(tailedItems[tailedItems.length - 1]?.seq) : null;
|
|
return {
|
|
...result,
|
|
data: {
|
|
...data,
|
|
items: tailedItems,
|
|
count: tailedItems.length,
|
|
cursor: lastReturnedSeq === null ? data.cursor : String(lastReturnedSeq),
|
|
clientTail: {
|
|
requested: limit,
|
|
sourceLastEventSeq: lastSeq,
|
|
scanned: items.length,
|
|
valuesPrinted: false,
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
async function resourceResult(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args);
|
|
if (ref.kind === "run") {
|
|
const commandId = options.commandId ?? requiredContext("run result", "--command <commandId>");
|
|
const result = await runAgentRunRestCommand(config, "runs", ["result", ref.name, "--command-id", commandId]);
|
|
return renderResultSummary(command, result, options, ref);
|
|
}
|
|
if (ref.kind === "command") {
|
|
const runId = options.runId ?? requiredContext("command result", "--run <runId>");
|
|
const result = await runAgentRunRestCommand(config, "commands", ["result", ref.name, "--run-id", runId]);
|
|
return renderResultSummary(command, result, options, ref);
|
|
}
|
|
throw new Error("result supports run/<runId> or command/<commandId>");
|
|
}
|
|
|
|
async function resourceAck(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args, "task");
|
|
const result = ref.kind === "task"
|
|
? await runAgentRunRestCommand(config, "queue", ["read", ref.name, "--reader-id", options.readerId])
|
|
: ref.kind === "session"
|
|
? await runAgentRunRestCommand(config, "sessions", ["read", ref.name, "--reader-id", options.readerId])
|
|
: null;
|
|
if (result === null) throw new Error("ack supports task/<taskId> or session/<sessionId>");
|
|
return renderMutationSummary(command, result, options, `Acknowledged ${ref.kind}/${shortId(ref.name)}`);
|
|
}
|
|
|
|
async function resourceCancel(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args, "task");
|
|
const cancelArgs = ["cancel", ref.name];
|
|
if (options.reason !== null) cancelArgs.push("--reason", options.reason);
|
|
if (ref.kind === "command") cancelArgs.push("--run-id", options.runId ?? requiredContext("command cancel", "--run <runId>"));
|
|
if (options.dryRun) {
|
|
const result = agentRunResourceCancelDryRunPlan(ref, options, rerunWithoutDryRun(command));
|
|
return renderMutationSummary(command, result, options, `Planned cancel ${ref.kind}/${shortId(ref.name)}`, [rerunWithoutDryRun(command)]);
|
|
}
|
|
const result = ref.kind === "task"
|
|
? await runAgentRunRestCommand(config, "queue", cancelArgs)
|
|
: ref.kind === "session"
|
|
? await runAgentRunRestCommand(config, "sessions", cancelArgs)
|
|
: ref.kind === "run"
|
|
? await runAgentRunRestCommand(config, "runs", cancelArgs)
|
|
: ref.kind === "command"
|
|
? await runAgentRunRestCommand(config, "commands", cancelArgs)
|
|
: null;
|
|
if (result === null) throw new Error("cancel supports task/<taskId>, session/<sessionId>, run/<runId>, or command/<commandId>");
|
|
return renderMutationSummary(command, result, options, `${options.dryRun ? "Planned cancel" : "Cancel requested"} ${ref.kind}/${shortId(ref.name)}`, options.dryRun ? [rerunWithoutDryRun(command)] : undefined);
|
|
}
|
|
|
|
function agentRunResourceCancelDryRunPlan(ref: AgentRunResourceRef, options: AgentRunResourceOptions, confirmCommand: string): Record<string, unknown> {
|
|
const body: Record<string, unknown> = {};
|
|
if (options.reason !== null) body.reason = options.reason;
|
|
if (ref.kind === "task") return agentRunDryRunPlan("task-cancel", `/api/v1/queue/tasks/${encodeURIComponent(ref.name)}/cancel`, body, confirmCommand);
|
|
if (ref.kind === "session") return agentRunDryRunPlan("session-cancel", `/api/v1/sessions/${encodeURIComponent(ref.name)}/control`, { action: "cancel", ...body }, confirmCommand);
|
|
if (ref.kind === "run") return agentRunDryRunPlan("run-cancel", `/api/v1/runs/${encodeURIComponent(ref.name)}/cancel`, body, confirmCommand);
|
|
if (ref.kind === "command") {
|
|
const runId = options.runId ?? requiredContext("command cancel", "--run <runId>");
|
|
return agentRunDryRunPlan("command-cancel", `/api/v1/commands/${encodeURIComponent(ref.name)}/cancel`, body, confirmCommand, "POST", {
|
|
commandRef: { runId, commandId: ref.name, valuesPrinted: false },
|
|
});
|
|
}
|
|
throw new Error("cancel supports task/<taskId>, session/<sessionId>, run/<runId>, or command/<commandId>");
|
|
}
|
|
|
|
async function resourceDispatch(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args, "task");
|
|
if (ref.kind !== "task") throw new Error("dispatch supports task/<taskId>");
|
|
const result = await runAgentRunRestCommand(config, "queue", ["dispatch", ref.name, ...stripLeadingResource(args, ref.name)]);
|
|
return renderMutationSummary(command, result, options, `Task dispatch submitted ${shortId(ref.name)}`, [
|
|
`bun scripts/cli.ts agentrun describe task/${ref.name}`,
|
|
]);
|
|
}
|
|
|
|
async function resourceCreate(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const kind = parseResourceKind(action);
|
|
if (kind !== "task") throw new Error("create currently supports: create task");
|
|
const submitArgs = ["submit", ...stripLeadingResource(args, "task")];
|
|
const result = await runAgentRunRestCommand(config, "queue", submitArgs);
|
|
return renderMutationSummary(command, result, options, "Task create submitted", options.dryRun ? [rerunWithoutDryRun(command)] : undefined);
|
|
}
|
|
|
|
async function resourceApply(config: UniDeskConfig | null, command: string, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
void config;
|
|
const file = options.file ?? requiredContext("apply", "-f <file>|-");
|
|
const raw = file === "-" ? readFileSync(0, "utf8") : readFileSync(file, "utf8");
|
|
const parsed = parseTaskManifest(raw, file);
|
|
const result = options.dryRun
|
|
? agentRunDryRunPlan("queue-submit", "/api/v1/queue/tasks", parsed, `bun scripts/cli.ts ${command.replace(/\s+--dry-run\b/gu, "").trim()}`)
|
|
: await agentRunRestRequest("agentrun queue submit", "POST", "/api/v1/queue/tasks", parsed);
|
|
return renderMutationSummary(command, result, options, `${options.dryRun ? "Dry-run applied" : "Applied"} task manifest`, options.dryRun ? [rerunWithoutDryRun(command)] : undefined);
|
|
}
|
|
|
|
async function resourceSessionPromptCommand(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise<RenderedCliResult> {
|
|
const ref = parseResourceRef(action, args, "session");
|
|
if (ref.kind !== "session") throw new Error("send requires session/<sessionId>");
|
|
const sessionArgs = ["send", ref.name, ...stripLeadingResource(args, ref.name)];
|
|
const result = await runAgentRunRestCommand(config, "sessions", sessionArgs);
|
|
return renderMutationSummary(command, result, options, options.dryRun ? "Session send plan" : "Session send submitted");
|
|
}
|
|
|
|
function renderedCliResult(ok: boolean, command: string, renderedText: string, contentType: RenderedCliResult["contentType"] = "text/plain"): RenderedCliResult {
|
|
return { ok, command, renderedText, contentType };
|
|
}
|
|
|
|
function renderResourceResult(command: string, raw: Record<string, unknown>, options: AgentRunResourceOptions, kindLabel: string, items: Record<string, unknown>[]): RenderedCliResult {
|
|
if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false);
|
|
const payload = { kind: `${kindLabel}List`, count: items.length, items };
|
|
if (options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output);
|
|
if (options.output === "name") return renderedCliResult(raw.ok !== false, command, items.map((item) => `${String(item.kind ?? kindLabel).toLowerCase()}/${String(item.name ?? "")}`).join("\n"));
|
|
if (items.length === 0) return renderedCliResult(raw.ok !== false, command, `No ${kindLabel.toLowerCase()} resources found.`);
|
|
return renderedCliResult(raw.ok !== false, command, renderResourceTable(items, options.output === "wide"));
|
|
}
|
|
|
|
function renderDescribe(command: string, raw: Record<string, unknown>, options: AgentRunResourceOptions, ref: AgentRunResourceRef, text: string): RenderedCliResult {
|
|
if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false);
|
|
if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: ref.kind, name: ref.name, resource: innerData(raw) }, options.output, raw.ok !== false);
|
|
return renderedCliResult(raw.ok !== false, command, text);
|
|
}
|
|
|
|
function renderEventLike(command: string, raw: Record<string, unknown>, options: AgentRunResourceOptions, kind: string, items: Record<string, unknown>[], sourceName: string): RenderedCliResult {
|
|
if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false);
|
|
const data = record(innerData(raw));
|
|
const payload = { kind: `${kind}List`, source: sourceName, nextAfterSeq: data.nextAfterSeq ?? null, count: items.length, items };
|
|
if (options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output, raw.ok !== false);
|
|
const lines = [
|
|
`${kind}s for ${sourceName}`,
|
|
renderTable(["SEQ", "TYPE", "STATUS", "COMMAND", "SUMMARY"], items.map((item) => [
|
|
String(item.seq ?? "-"),
|
|
String(item.type ?? "-"),
|
|
String(item.status ?? item.phase ?? "-"),
|
|
shortId(String(item.commandId ?? "-")),
|
|
truncateOneLine(String(item.summary ?? item.text ?? "") || "-", 96),
|
|
])),
|
|
];
|
|
if (data.nextAfterSeq !== undefined && data.nextAfterSeq !== null) lines.push(`Next: bun scripts/cli.ts ${nextPagedResourceCommand(command, String(data.nextAfterSeq), options.limit)}`);
|
|
return renderedCliResult(raw.ok !== false, command, lines.join("\n"));
|
|
}
|
|
|
|
function renderResultSummary(command: string, raw: Record<string, unknown>, options: AgentRunResourceOptions, ref: AgentRunResourceRef): RenderedCliResult {
|
|
if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false);
|
|
const data = record(innerData(raw));
|
|
if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: "Result", ref, result: data }, options.output, raw.ok !== false);
|
|
const executionOk = typeof data.executionOk === "boolean"
|
|
? data.executionOk
|
|
: typeof data.ok === "boolean"
|
|
? data.ok
|
|
: raw.ok !== false;
|
|
const lines = [
|
|
`Result: ${ref.kind}/${shortId(ref.name)}`,
|
|
`State: ${displayValue(data.state ?? data.status ?? data.terminalStatus ?? "-")}`,
|
|
`OK: ${String(executionOk)}`,
|
|
];
|
|
const providerTerminalFailure = typeof data.providerTerminalFailure === "boolean" ? data.providerTerminalFailure : null;
|
|
const authority = stringOrNull(data.finalResponseAuthority);
|
|
const fallback = typeof data.finalResponseFallback === "boolean" ? data.finalResponseFallback : null;
|
|
const needsContinuation = typeof data.needsContinuation === "boolean" ? data.needsContinuation : null;
|
|
if (providerTerminalFailure !== null) lines.push(`ProviderTerminalFailure: ${String(providerTerminalFailure)}`);
|
|
if (authority !== null) lines.push(`Authority: ${authority}`);
|
|
if (fallback !== null) lines.push(`Fallback: ${String(fallback)}`);
|
|
if (needsContinuation !== null) lines.push(`NeedsContinuation: ${String(needsContinuation)}`);
|
|
const finalRecord = record(data.finalResponse);
|
|
const final = stringOrNull(data.finalResponse)
|
|
?? stringOrNull(finalRecord.content)
|
|
?? stringOrNull(finalRecord.text)
|
|
?? stringOrNull(finalRecord.message)
|
|
?? stringOrNull(data.output)
|
|
?? stringOrNull(data.summary)
|
|
?? stringOrNull(data.result);
|
|
if (final !== null) lines.push("", truncateMultiline(final, options.fullText ? 8000 : 1600));
|
|
else {
|
|
const failure = renderFailureLines(data);
|
|
lines.push("", ...(failure.length > 0 ? failure : ["No final response is available yet."]));
|
|
}
|
|
return renderedCliResult(raw.ok !== false, command, lines.join("\n"));
|
|
}
|
|
|
|
function renderMutationSummary(command: string, raw: Record<string, unknown>, options: AgentRunResourceOptions, headline: string, overrideNextLines?: string[]): RenderedCliResult {
|
|
if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false);
|
|
if (options.output === "json" || options.output === "yaml") return renderMachine(command, raw, options.output, raw.ok !== false);
|
|
const data = record(innerData(raw));
|
|
const id = stringOrNull(data.id) ?? stringOrNull(data.taskId) ?? stringOrNull(data.sessionId);
|
|
const lines = [
|
|
headline,
|
|
`OK: ${String(raw.ok !== false)}`,
|
|
];
|
|
if (id !== null) lines.push(`Name: ${id}`);
|
|
const decision = stringOrNull(data.decision);
|
|
const internalCommandType = stringOrNull(data.internalCommandType);
|
|
if (data.dryRun !== undefined) lines.push(`DryRun: ${String(data.dryRun)}`);
|
|
if (data.mutation !== undefined) lines.push(`Mutation: ${String(data.mutation)}`);
|
|
if (decision !== null) lines.push(`Decision: ${decision}`);
|
|
if (internalCommandType !== null) lines.push(`InternalCommandType: ${internalCommandType}`);
|
|
const next = record(raw.next ?? data.next);
|
|
const nextLines = (overrideNextLines ?? Object.values(next).map(String)).filter((line) => line.length > 0).slice(0, 5);
|
|
if (nextLines.length > 0) lines.push("", "Next:", ...nextLines.map((line) => ` ${line}`));
|
|
return renderedCliResult(raw.ok !== false, command, lines.join("\n"));
|
|
}
|
|
|
|
function rerunWithoutDryRun(command: string): string {
|
|
return `bun scripts/cli.ts ${command.replace(/\s+--dry-run\b/gu, "").trim()}`;
|
|
}
|
|
|
|
function renderMachine(command: string, value: unknown, mode: "json" | "yaml", ok = true): RenderedCliResult {
|
|
const text = mode === "json" ? `${JSON.stringify(value, null, 2)}\n` : `${Bun.YAML.stringify(value)}\n`;
|
|
return renderedCliResult(ok, command, text, mode === "json" ? "application/json" : "application/yaml");
|
|
}
|
|
|
|
function parseResourceKind(raw: string | undefined): AgentRunResourceKind | null {
|
|
if (raw === undefined) return null;
|
|
const value = raw.toLowerCase().replace(/s$/u, "");
|
|
if (value === "task" || value === "qt" || value === "queue" || value === "queuetask") return "task";
|
|
if (value === "run") return "run";
|
|
if (value === "command" || value === "cmd") return "command";
|
|
if (value === "runnerjob" || value === "runner-job" || value === "rjob" || value === "job") return "runnerjob";
|
|
if (value === "session" || value === "ses") return "session";
|
|
if (value === "aipodspec" || value === "aipod-spec" || value === "aipod-specs" || value === "aps") return "aipodspec";
|
|
return null;
|
|
}
|
|
|
|
function parseResourceRef(action: string | undefined, args: string[], defaultKind: AgentRunResourceKind | null = null): AgentRunResourceRef {
|
|
const raw = action ?? "";
|
|
if (raw.includes("/")) {
|
|
const [kindRaw, ...nameParts] = raw.split("/");
|
|
const kind = parseResourceKind(kindRaw);
|
|
const name = nameParts.join("/");
|
|
if (kind === null || name.length === 0) throw new Error(`invalid resource reference: ${raw}`);
|
|
return { kind, name };
|
|
}
|
|
const kind = parseResourceKind(raw);
|
|
if (kind !== null) {
|
|
const name = args.find((arg) => !arg.startsWith("-") && arg !== raw);
|
|
if (name === undefined) throw new Error(`${kind} reference requires a name`);
|
|
return { kind, name };
|
|
}
|
|
if (defaultKind !== null && raw.length > 0) return { kind: defaultKind, name: raw };
|
|
throw new Error("resource reference required, for example task/<id> or session/<id>");
|
|
}
|
|
|
|
function stripLeadingResource(args: string[], resourceName: string): string[] {
|
|
let skipped = false;
|
|
const result: string[] = [];
|
|
for (const arg of args) {
|
|
if (!skipped && (arg === resourceName || arg.endsWith(`/${resourceName}`) || parseResourceKind(arg) !== null)) {
|
|
skipped = true;
|
|
continue;
|
|
}
|
|
result.push(arg);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function requiredContext(context: string, flag: string): never {
|
|
throw new Error(`${context} requires ${flag}`);
|
|
}
|
|
|
|
function innerData(raw: Record<string, unknown>): unknown {
|
|
const data = raw.data;
|
|
if (typeof data === "object" && data !== null && "data" in data && Object.keys(raw).length <= 3) return record(data).data;
|
|
return data ?? raw;
|
|
}
|
|
|
|
function normalizeTaskItems(data: unknown, options: AgentRunResourceOptions): Record<string, unknown>[] {
|
|
const items = arrayRecords(record(data).items ?? data);
|
|
return items
|
|
.filter((item) => options.queue === null || String(item.queue ?? "") === options.queue)
|
|
.filter((item) => options.state === null || options.state.split(",").includes(String(item.state ?? "")))
|
|
.filter((item) => !options.unread || item.unread === true || item.attention === true)
|
|
.map((item) => {
|
|
const attempt = record(item.latestAttempt);
|
|
return {
|
|
kind: "task",
|
|
name: item.id,
|
|
state: item.state,
|
|
queue: item.queue,
|
|
lane: item.lane,
|
|
profile: item.backendProfile,
|
|
run: attempt.runId,
|
|
command: attempt.commandId,
|
|
runnerjob: attempt.runnerJobId,
|
|
session: attempt.sessionId ?? stringOrNull(record(item.sessionRef).sessionId),
|
|
age: relativeAge(stringOrNull(item.updatedAt) ?? stringOrNull(item.createdAt)),
|
|
title: item.title,
|
|
updatedAt: item.updatedAt,
|
|
attention: item.unread === true || item.attention === true ? "unread" : "",
|
|
};
|
|
});
|
|
}
|
|
|
|
function taskListState(options: AgentRunResourceOptions): string {
|
|
const requested = options.state?.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
return requested?.[0] ?? "running";
|
|
}
|
|
|
|
function normalizeSessionItems(data: unknown): Record<string, unknown>[] {
|
|
const items = arrayRecords(record(data).items ?? data);
|
|
return items.map((item) => ({
|
|
kind: "session",
|
|
name: item.id ?? item.sessionId ?? item.name,
|
|
state: item.state ?? item.status ?? item.phase ?? "-",
|
|
queue: item.queue ?? "",
|
|
lane: item.lane ?? "",
|
|
profile: item.backendProfile ?? item.profile ?? "",
|
|
run: item.runId ?? "",
|
|
command: item.commandId ?? "",
|
|
runnerjob: item.runnerJobId ?? "",
|
|
session: item.id ?? item.sessionId ?? item.name,
|
|
age: relativeAge(stringOrNull(item.updatedAt) ?? stringOrNull(item.createdAt)),
|
|
title: item.title ?? "",
|
|
}));
|
|
}
|
|
|
|
function normalizeAipodSpecItems(data: unknown): Record<string, unknown>[] {
|
|
const items = arrayRecords(record(data).items ?? data);
|
|
return items.map((item) => ({
|
|
kind: "aipodspec",
|
|
name: item.name ?? item.id ?? item.metadata?.toString(),
|
|
state: item.state ?? "available",
|
|
queue: "",
|
|
lane: item.lane ?? "",
|
|
profile: item.backendProfile ?? item.profile ?? "",
|
|
title: item.description ?? item.title ?? "",
|
|
}));
|
|
}
|
|
|
|
function normalizeAttemptResources(task: Record<string, unknown>, kind: "run" | "command" | "runnerjob"): Record<string, unknown>[] {
|
|
const attempt = record(task.latestAttempt);
|
|
const key = kind === "run" ? "runId" : kind === "command" ? "commandId" : "runnerJobId";
|
|
const name = stringOrNull(attempt[key]);
|
|
if (name === null) return [];
|
|
return [{
|
|
kind,
|
|
name,
|
|
state: attempt.state ?? task.state,
|
|
queue: task.queue,
|
|
lane: task.lane,
|
|
run: attempt.runId,
|
|
command: attempt.commandId,
|
|
runnerjob: attempt.runnerJobId,
|
|
session: attempt.sessionId,
|
|
title: task.title,
|
|
}];
|
|
}
|
|
|
|
function normalizeSingleCommand(data: unknown): Record<string, unknown>[] {
|
|
const item = record(data);
|
|
return [{
|
|
kind: "command",
|
|
name: item.id ?? item.commandId ?? "",
|
|
state: item.state ?? item.status ?? "",
|
|
run: item.runId ?? "",
|
|
command: item.id ?? item.commandId ?? "",
|
|
session: item.sessionId ?? "",
|
|
title: item.summary ?? item.commandType ?? "",
|
|
}];
|
|
}
|
|
|
|
function normalizeRunnerJobItems(data: unknown): Record<string, unknown>[] {
|
|
const items = arrayRecords(record(data).items ?? data);
|
|
return items.map((item) => ({
|
|
kind: "runnerjob",
|
|
name: item.id ?? item.runnerJobId ?? item.name ?? item.jobName,
|
|
state: item.phase ?? item.state ?? item.status ?? "",
|
|
run: item.runId ?? "",
|
|
command: item.commandId ?? "",
|
|
runnerjob: item.id ?? item.runnerJobId ?? item.name ?? item.jobName,
|
|
session: item.sessionId ?? "",
|
|
title: item.image ?? item.namespace ?? "",
|
|
}));
|
|
}
|
|
|
|
function normalizeEventItems(data: unknown): Record<string, unknown>[] {
|
|
return arrayRecords(record(data).items ?? data).map((item) => {
|
|
const payload = record(item.payload);
|
|
return {
|
|
seq: item.seq,
|
|
type: item.type,
|
|
status: agentRunEventStatus(item, payload),
|
|
phase: item.phase ?? payload.phase,
|
|
commandId: agentRunEventCommandId(item, payload),
|
|
summary: agentRunEventSummary(item, payload),
|
|
};
|
|
});
|
|
}
|
|
|
|
function normalizeLogItems(data: unknown): Record<string, unknown>[] {
|
|
return arrayRecords(record(data).items ?? data).map((item) => {
|
|
const payload = record(item.payload);
|
|
return {
|
|
seq: item.seq,
|
|
type: item.type ?? item.role ?? "output",
|
|
status: agentRunEventStatus(item, payload),
|
|
commandId: agentRunEventCommandId(item, payload),
|
|
summary: agentRunEventSummary(item, payload),
|
|
};
|
|
});
|
|
}
|
|
|
|
function agentRunEventCommandId(item: Record<string, unknown>, payload: Record<string, unknown>): string {
|
|
return stringOrNull(item.commandId)
|
|
?? stringOrNull(payload.commandId)
|
|
?? stringOrNull(record(payload.summary).commandId)
|
|
?? "-";
|
|
}
|
|
|
|
function agentRunEventStatus(item: Record<string, unknown>, payload: Record<string, unknown>): string {
|
|
return stringOrNull(item.status)
|
|
?? stringOrNull(item.phase)
|
|
?? stringOrNull(payload.status)
|
|
?? stringOrNull(payload.phase)
|
|
?? stringOrNull(payload.failureKind)
|
|
?? (item.type === "error" ? "error" : "");
|
|
}
|
|
|
|
function agentRunEventSummary(item: Record<string, unknown>, payload: Record<string, unknown>): string {
|
|
const type = stringOrNull(item.type);
|
|
const parts: string[] = [];
|
|
const push = (value: unknown): void => {
|
|
const text = eventText(value);
|
|
if (text !== null && !parts.includes(text)) parts.push(text);
|
|
};
|
|
|
|
if (type === "error") {
|
|
push(payload.failureKind);
|
|
const error = record(payload.error);
|
|
push(error.message);
|
|
push(error.additionalDetails);
|
|
if (payload.willRetry === true) push("willRetry=true");
|
|
}
|
|
|
|
if (type === "backend_status") {
|
|
push(payload.phase);
|
|
push(formatCountField("tools", record(payload.tools).count));
|
|
push(formatCountField("bundles", record(payload.bundles).count));
|
|
push(formatCountField("promptRefs", record(payload.promptRefs).count));
|
|
push(payload.jobName);
|
|
push(payload.namespace);
|
|
}
|
|
|
|
if (type === "tool_call") {
|
|
push(payload.summary);
|
|
push(payload.toolName);
|
|
push(payload.command);
|
|
const exitCode = payload.exitCode;
|
|
if (exitCode !== undefined && exitCode !== null) push(`exit=${String(exitCode)}`);
|
|
push(payload.outputSummary);
|
|
}
|
|
|
|
if (type === "command_output") {
|
|
push(commandOutputSummary(payload));
|
|
}
|
|
|
|
if (type === "assistant_message") {
|
|
push(payload.summary);
|
|
push(payload.text);
|
|
}
|
|
|
|
push(item.summary);
|
|
push(item.outputSummary);
|
|
push(item.text);
|
|
if (type !== "command_output") {
|
|
push(payload.summary);
|
|
push(payload.outputSummary);
|
|
push(payload.text);
|
|
}
|
|
push(payload.message);
|
|
push(payload.reason);
|
|
|
|
return parts.join("; ");
|
|
}
|
|
|
|
function commandOutputSummary(payload: Record<string, unknown>): string | null {
|
|
const raw = commandOutputText(payload);
|
|
if (raw === null) return null;
|
|
const structured = parseJsonRecordFromText(raw);
|
|
if (structured !== null) {
|
|
const summary = summarizeStructuredCliOutput(structured);
|
|
if (summary !== null) return summary;
|
|
}
|
|
return summarizePartialJsonCommandOutput(raw) ?? summarizeTextCommandOutput(raw);
|
|
}
|
|
|
|
function commandOutputText(payload: Record<string, unknown>): string | null {
|
|
const summary = record(payload.summary);
|
|
const summaryText = stringOrNull(summary.text);
|
|
const payloadText = stringOrNull(payload.text);
|
|
if (summary.textTruncated === true && payloadText !== null) return payloadText;
|
|
return summaryText
|
|
?? payloadText
|
|
?? stringOrNull(payload.outputSummary)
|
|
?? eventText(payload.summary);
|
|
}
|
|
|
|
function parseJsonRecordFromText(raw: string): Record<string, unknown> | null {
|
|
const trimmed = raw.trim();
|
|
const direct = tryParseJsonRecord(trimmed);
|
|
if (direct !== null) return direct;
|
|
|
|
const firstBrace = trimmed.indexOf("{");
|
|
const lastBrace = trimmed.lastIndexOf("}");
|
|
if (firstBrace >= 0 && lastBrace > firstBrace) {
|
|
const sliced = tryParseJsonRecord(trimmed.slice(firstBrace, lastBrace + 1));
|
|
if (sliced !== null) return sliced;
|
|
}
|
|
|
|
const line = trimmed
|
|
.split(/\r?\n/u)
|
|
.map((item) => item.trim())
|
|
.reverse()
|
|
.find((item) => item.startsWith("{") && item.endsWith("}"));
|
|
return line === undefined ? null : tryParseJsonRecord(line);
|
|
}
|
|
|
|
function tryParseJsonRecord(raw: string): Record<string, unknown> | null {
|
|
try {
|
|
const parsed = JSON.parse(raw) as unknown;
|
|
return isRecord(parsed) ? parsed : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function summarizeStructuredCliOutput(value: Record<string, unknown>): string | null {
|
|
const parts: string[] = [];
|
|
const action = firstPathText(value, ["action", "operation", "command", "kind"]);
|
|
const status = firstPathText(value, ["status", "state", "phase", "body.status"]);
|
|
const failure = firstPathText(value, ["failureKind", "error.failureKind", "body.failureKind"]);
|
|
const message = firstPathText(value, ["error.message", "error.additionalDetails", "message", "reason", "body.message", "body.error", "result.message"]);
|
|
const primary = [
|
|
action,
|
|
status,
|
|
labeledSummaryValue("http", firstPathText(value, ["httpStatus", "request.httpStatus"])),
|
|
].filter((item): item is string => item !== null);
|
|
if (primary.length > 0) pushSummaryPart(parts, primary.join(" "));
|
|
|
|
pushLabeledSummaryPart(parts, "failure", failure);
|
|
if (failure !== null || isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message);
|
|
pushLabeledSummaryPart(parts, "trace", firstPathText(value, ["traceId", "lastTraceId", "trace.traceId", "trace.id", "body.traceId", "body.trace.id", "providerTrace.traceId", "runnerTrace.traceId"]), true);
|
|
pushLabeledSummaryPart(parts, "session", firstPathText(value, ["sessionId", "session.sessionId", "body.sessionId", "workspace.selectedAgentSessionId", "workspace.selectedConversation.sessionId"]), true);
|
|
pushLabeledSummaryPart(parts, "conversation", firstPathText(value, ["conversationId", "session.conversationId", "body.conversationId", "workspace.selectedConversationId", "workspace.selectedConversation.conversationId"]), true);
|
|
pushLabeledSummaryPart(parts, "thread", firstPathText(value, ["threadId", "session.threadId", "body.threadId", "workspace.selectedConversation.threadId"]), true);
|
|
pushLabeledSummaryPart(parts, "provider", firstPathText(value, ["providerProfile", "backendProfile", "profile", "session.providerProfile", "workspace.providerProfile"]));
|
|
pushLabeledSummaryPart(parts, "route", commandOutputRouteSummary(value));
|
|
pushLabeledSummaryPart(parts, "pipeline", firstPathText(value, ["pipelineRun", "pipelineRunName", "pipelineRun.name", "pipeline.runName", "pipeline.name"]), true);
|
|
pushLabeledSummaryPart(parts, "run", firstPathText(value, ["runId", "agentRun.runId"]), true);
|
|
pushLabeledSummaryPart(parts, "cmd", firstPathText(value, ["commandId", "agentRun.commandId", "providerTrace.commandId", "runnerTrace.commandId"]), true);
|
|
pushLabeledSummaryPart(parts, "job", firstPathText(value, ["jobName", "agentRun.jobName", "runnerJobId"]), true);
|
|
pushLabeledSummaryPart(parts, "lane", firstPathText(value, ["runtimeEndpoint.lane", "lane"]));
|
|
pushLabeledSummaryPart(parts, "ns", firstPathText(value, ["runtimeEndpoint.namespace", "namespace"]));
|
|
pushLabeledSummaryPart(parts, "actor", firstPathText(value, ["actor.username", "body.actor.username", "actor.displayName", "body.actor.displayName"]));
|
|
pushLabeledSummaryPart(parts, "auth", firstPathText(value, ["authMethod", "auth.authMethod", "body.authMethod"]));
|
|
pushLabeledSummaryPart(parts, "reply", firstPathText(value, ["reply.content", "result.reply.content", "body.reply.content", "answer", "content", "body.content"]));
|
|
if (failure === null && !isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message);
|
|
|
|
const itemsCount = arrayRecords(value.items).length;
|
|
if (itemsCount > 0) pushSummaryPart(parts, `items=${itemsCount}`);
|
|
const eventsCount = arrayRecords(value.events).length;
|
|
if (eventsCount > 0) pushSummaryPart(parts, `events=${eventsCount}`);
|
|
if (value.sessionUsable === false) pushSummaryPart(parts, "sessionUsable=false");
|
|
if (record(value.summary).textTruncated === true || value.textTruncated === true || value.outputTruncated === true) pushSummaryPart(parts, "truncated=true");
|
|
|
|
return parts.length > 0 ? parts.slice(0, 10).join("; ") : null;
|
|
}
|
|
|
|
function commandOutputRouteSummary(value: Record<string, unknown>): string | null {
|
|
const route = record(value.route);
|
|
const request = record(value.request);
|
|
const method = stringOrNull(route.method) ?? stringOrNull(request.method);
|
|
const path = stringOrNull(route.path) ?? stringOrNull(request.path);
|
|
if (method !== null && path !== null) return `${method} ${path}`;
|
|
return stringOrNull(request.url) ?? stringOrNull(value.baseUrl);
|
|
}
|
|
|
|
function firstPathText(value: Record<string, unknown>, paths: string[]): string | null {
|
|
for (const path of paths) {
|
|
const text = summaryScalar(pathValue(value, path));
|
|
if (text !== null) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function pathValue(value: Record<string, unknown>, path: string): unknown {
|
|
let current: unknown = value;
|
|
for (const key of path.split(".")) {
|
|
if (!isRecord(current)) return undefined;
|
|
current = current[key];
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function summaryScalar(value: unknown): string | null {
|
|
if (typeof value === "string") {
|
|
const trimmed = value.trim();
|
|
return trimmed.length > 0 ? trimmed : null;
|
|
}
|
|
if (typeof value === "number" && Number.isFinite(value)) return String(value);
|
|
if (typeof value === "boolean") return String(value);
|
|
return null;
|
|
}
|
|
|
|
function isFailureLikeStatus(value: string | null): boolean {
|
|
return value !== null && /(?:fail|error|timeout|cancel|interrupt|denied|rejected)/iu.test(value);
|
|
}
|
|
|
|
function pushLabeledSummaryPart(parts: string[], label: string, value: string | null, shorten = false): void {
|
|
const text = labeledSummaryValue(label, value, shorten);
|
|
if (text !== null) pushSummaryPart(parts, text);
|
|
}
|
|
|
|
function labeledSummaryValue(label: string, value: string | null, shorten = false): string | null {
|
|
if (value === null) return null;
|
|
const rendered = shorten ? shortId(value) : truncateOneLine(value, 80);
|
|
return `${label}=${rendered}`;
|
|
}
|
|
|
|
function pushSummaryPart(parts: string[], value: string): void {
|
|
const text = truncateOneLine(value, 160);
|
|
if (text.length > 0 && !parts.includes(text)) parts.push(text);
|
|
}
|
|
|
|
function summarizeTextCommandOutput(raw: string): string | null {
|
|
const lines = raw
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter((line) => line.length > 0 && !isLowSignalCommandOutputLine(line));
|
|
const signal = lines.find((line) => /(?:trace|session|conversation|thread|status|failure|error|route|http|pipeline|provider|commit|succeeded|failed|created)/iu.test(line))
|
|
?? lines[0]
|
|
?? null;
|
|
if (signal === null) return null;
|
|
return truncateOneLine(cleanCommandOutputLine(signal), 220);
|
|
}
|
|
|
|
function summarizePartialJsonCommandOutput(raw: string): string | null {
|
|
const fields = new Map<string, string>();
|
|
for (const line of raw.split(/\r?\n/u)) {
|
|
const field = jsonLineField(line);
|
|
if (field === null || isLowSignalJsonField(field.key)) continue;
|
|
if (!fields.has(field.key) && field.value.length > 0) fields.set(field.key, field.value);
|
|
}
|
|
|
|
const parts: string[] = [];
|
|
const action = fields.get("action") ?? fields.get("operation") ?? fields.get("command") ?? fields.get("kind") ?? null;
|
|
const status = fields.get("status") ?? fields.get("state") ?? fields.get("phase") ?? null;
|
|
const http = fields.get("httpStatus") ?? null;
|
|
const failure = fields.get("failureKind") ?? null;
|
|
const message = fields.get("message") ?? fields.get("reason") ?? fields.get("error") ?? null;
|
|
const primary = [action, status, labeledSummaryValue("http", http)].filter((item): item is string => item !== null);
|
|
if (primary.length > 0) pushSummaryPart(parts, primary.join(" "));
|
|
|
|
pushLabeledSummaryPart(parts, "failure", failure);
|
|
if (failure !== null || isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message);
|
|
pushLabeledSummaryPart(parts, "trace", fields.get("traceId") ?? fields.get("lastTraceId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "session", fields.get("sessionId") ?? fields.get("selectedAgentSessionId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "conversation", fields.get("conversationId") ?? fields.get("selectedConversationId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "thread", fields.get("threadId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "provider", fields.get("providerProfile") ?? fields.get("backendProfile") ?? fields.get("profile") ?? null);
|
|
pushLabeledSummaryPart(parts, "route", partialRouteSummary(fields));
|
|
pushLabeledSummaryPart(parts, "pipeline", fields.get("pipelineRun") ?? fields.get("pipelineRunName") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "run", fields.get("runId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "cmd", fields.get("commandId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "job", fields.get("jobName") ?? fields.get("runnerJobId") ?? null, true);
|
|
pushLabeledSummaryPart(parts, "lane", fields.get("lane") ?? null);
|
|
pushLabeledSummaryPart(parts, "ns", fields.get("namespace") ?? null);
|
|
pushLabeledSummaryPart(parts, "actor", fields.get("username") ?? fields.get("displayName") ?? null);
|
|
pushLabeledSummaryPart(parts, "auth", fields.get("authMethod") ?? null);
|
|
if (failure === null && !isFailureLikeStatus(status)) pushLabeledSummaryPart(parts, "message", message);
|
|
|
|
return parts.length > 0 ? parts.slice(0, 10).join("; ") : null;
|
|
}
|
|
|
|
function partialRouteSummary(fields: Map<string, string>): string | null {
|
|
const method = fields.get("method") ?? null;
|
|
const path = fields.get("path") ?? null;
|
|
if (method !== null && path !== null) return `${method} ${path}`;
|
|
return fields.get("url") ?? fields.get("baseUrl") ?? null;
|
|
}
|
|
|
|
function jsonLineField(line: string): { key: string; value: string } | null {
|
|
const match = line.trim().replace(/,$/u, "").match(/^"([^"]+)":\s*(.+)$/u);
|
|
if (match === null) return null;
|
|
return { key: match[1], value: cleanJsonScalar(match[2]) };
|
|
}
|
|
|
|
function cleanJsonScalar(raw: string): string {
|
|
const trimmed = raw.trim();
|
|
if (trimmed === "null" || trimmed === "{" || trimmed === "[" || trimmed === "}" || trimmed === "]") return "";
|
|
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return typeof parsed === "string" ? parsed.trim() : String(parsed);
|
|
} catch {
|
|
return trimmed.slice(1, -1).trim();
|
|
}
|
|
}
|
|
return trimmed;
|
|
}
|
|
|
|
function cleanCommandOutputLine(line: string): string {
|
|
const trimmed = line.replace(/,$/u, "").trim();
|
|
const jsonField = trimmed.match(/^"([^"]+)":\s*(.+)$/u);
|
|
if (jsonField === null) return trimmed;
|
|
const key = jsonField[1];
|
|
const value = jsonField[2].replace(/^"|"$/gu, "").trim();
|
|
return `${key}=${value}`;
|
|
}
|
|
|
|
function isLowSignalCommandOutputLine(line: string): boolean {
|
|
const trimmed = line.replace(/,$/u, "").trim();
|
|
if (trimmed === "{" || trimmed === "}" || trimmed === "}," || trimmed === "];" || trimmed === "[") return true;
|
|
const field = jsonLineField(trimmed);
|
|
return field !== null && isLowSignalJsonField(field.key);
|
|
}
|
|
|
|
function isLowSignalJsonField(key: string): boolean {
|
|
return key === "generatedAt" || key === "cli" || key === "version" || key === "valuesRedacted" || key === "secretMaterialStored";
|
|
}
|
|
|
|
function eventText(value: unknown): string | null {
|
|
const direct = stringOrNull(value);
|
|
if (direct !== null && direct.trim().length > 0) return direct.trim();
|
|
const nested = record(value);
|
|
return stringOrNull(nested.text)
|
|
?? stringOrNull(nested.message)
|
|
?? stringOrNull(nested.reason)
|
|
?? stringOrNull(nested.summary);
|
|
}
|
|
|
|
function formatCountField(label: string, value: unknown): string | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return `${label}=${value}`;
|
|
if (typeof value === "string" && value.length > 0) return `${label}=${value}`;
|
|
return null;
|
|
}
|
|
|
|
function renderResourceTable(items: Record<string, unknown>[], wide: boolean): string {
|
|
const headers = wide
|
|
? ["NAME", "STATE", "QUEUE", "LANE", "PROFILE", "RUN", "CMD", "RJOB", "SESSION", "AGE", "TITLE"]
|
|
: ["NAME", "STATE", "QUEUE", "LANE", "RUN", "CMD", "RJOB", "SESSION", "AGE", "ATTENTION"];
|
|
const rows = items.map((item) => {
|
|
if (wide) {
|
|
return [
|
|
resourceName(item),
|
|
displayValue(item.state),
|
|
displayValue(item.queue),
|
|
displayValue(item.lane),
|
|
displayValue(item.profile),
|
|
shortId(stringOrDash(item.run)),
|
|
shortId(stringOrDash(item.command)),
|
|
shortId(stringOrDash(item.runnerjob)),
|
|
shortId(stringOrDash(item.session)),
|
|
displayValue(item.age),
|
|
truncateOneLine(displayValue(item.title), 48),
|
|
];
|
|
}
|
|
return [
|
|
resourceName(item),
|
|
displayValue(item.state),
|
|
displayValue(item.queue),
|
|
displayValue(item.lane),
|
|
shortId(stringOrDash(item.run)),
|
|
shortId(stringOrDash(item.command)),
|
|
shortId(stringOrDash(item.runnerjob)),
|
|
shortId(stringOrDash(item.session)),
|
|
displayValue(item.age),
|
|
displayValue(item.attention),
|
|
];
|
|
});
|
|
return renderTable(headers, rows);
|
|
}
|
|
|
|
function renderTaskDescription(task: Record<string, unknown>, options: AgentRunResourceOptions): string {
|
|
const attempt = record(task.latestAttempt);
|
|
const taskId = displayValue(task.id);
|
|
const runId = stringOrNull(attempt.runId);
|
|
const commandId = stringOrNull(attempt.commandId);
|
|
const runnerJobId = stringOrNull(attempt.runnerJobId);
|
|
const sessionId = stringOrNull(attempt.sessionId) ?? stringOrNull(record(task.sessionRef).sessionId);
|
|
const lines = [
|
|
`Name: task/${taskId}`,
|
|
`State: ${displayValue(task.state)} Attention: ${displayValue(task.unread === true ? "unread" : "")}`,
|
|
`Queue: ${displayValue(task.queue)}`,
|
|
`Lane: ${displayValue(task.lane)} Profile: ${displayValue(task.backendProfile)} Provider: ${displayValue(task.providerId)}`,
|
|
`Title: ${displayValue(task.title)}`,
|
|
"",
|
|
"Latest Attempt:",
|
|
` Attempt: ${displayValue(attempt.attemptId)} State: ${displayValue(attempt.state)}`,
|
|
` Run: ${runId === null ? "-" : `run/${runId}`}`,
|
|
` Command: ${commandId === null ? "-" : `command/${commandId}`}`,
|
|
` Runner: ${runnerJobId === null ? "-" : `runnerjob/${runnerJobId}`}`,
|
|
` Session: ${sessionId === null ? "-" : `session/${sessionId}`}`,
|
|
"",
|
|
"Conditions:",
|
|
` Terminal: ${displayValue(task.state)}`,
|
|
` Failure: ${displayValue(task.failureKind ?? task.degradedReason ?? "none")}`,
|
|
" Values: redacted",
|
|
"",
|
|
"Next:",
|
|
];
|
|
const next = [
|
|
runId === null ? null : `bun scripts/cli.ts agentrun events run/${runId} --after-seq 0 --limit 100`,
|
|
sessionId === null ? null : `bun scripts/cli.ts agentrun logs session/${sessionId} --tail 100`,
|
|
runId === null || commandId === null ? null : `bun scripts/cli.ts agentrun result run/${runId} --command ${commandId}`,
|
|
`bun scripts/cli.ts agentrun ack task/${taskId}`,
|
|
options.full ? null : `bun scripts/cli.ts agentrun describe task/${taskId} --full`,
|
|
].filter((item): item is string => item !== null).slice(0, 5);
|
|
lines.push(...next.map((item) => ` ${item}`));
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function unwrapTaskDetail(data: Record<string, unknown>): Record<string, unknown> {
|
|
const task = record(data.task);
|
|
if (Object.keys(task).length > 0) return task;
|
|
return data;
|
|
}
|
|
|
|
function renderGenericDescription(ref: AgentRunResourceRef, data: unknown): string {
|
|
const value = record(data);
|
|
const lines = [
|
|
`Name: ${ref.kind}/${ref.name}`,
|
|
`State: ${displayValue(value.state ?? value.status ?? value.phase ?? value.executionState ?? value.terminalStatus ?? "-")}`,
|
|
];
|
|
const identity = renderResourceIdentityLines(ref, value);
|
|
if (identity.length > 0) lines.push("", ...identity);
|
|
const supervisor = renderSupervisorLines(ref, value);
|
|
if (supervisor.length > 0) lines.push("", "Supervisor:", ...supervisor.map((line) => ` ${line}`));
|
|
const failure = renderFailureLines(value);
|
|
if (failure.length > 0) lines.push("", ...failure);
|
|
const next = renderResourceNextLines(ref, value);
|
|
if (next.length > 0) lines.push("", "Next:", ...next.map((line) => ` ${line}`));
|
|
if (identity.length === 0 && failure.length === 0 && next.length === 0) {
|
|
lines.push(
|
|
"",
|
|
"Summary:",
|
|
JSON.stringify(pickCompact(value, ["id", "name", "runId", "commandId", "runnerJobId", "sessionId", "state", "status", "phase", "reason", "failureKind", "valuesPrinted"]), null, 2),
|
|
);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function renderResourceIdentityLines(ref: AgentRunResourceRef, value: Record<string, unknown>): string[] {
|
|
const supervisor = record(value.supervisor);
|
|
const fields: [string, unknown][] = [
|
|
["Run", firstKnownString(value.runId, value.activeRunId, value.lastRunId, supervisor.runId, supervisor.activeRunId, supervisor.lastRunId)],
|
|
["Command", firstKnownString(value.commandId, value.activeCommandId, value.lastCommandId, supervisor.commandId, supervisor.activeCommandId, supervisor.lastCommandId)],
|
|
["RunnerJob", value.runnerJobId ?? (ref.kind === "runnerjob" ? value.id : null)],
|
|
["Session", firstKnownString(value.sessionId, supervisor.sessionId)],
|
|
["Namespace", value.namespace],
|
|
["Job", value.jobName],
|
|
["Image", value.image],
|
|
["LogPath", value.logPath],
|
|
["Started", value.startedAt],
|
|
["Finished", value.finishedAt],
|
|
];
|
|
if (ref.kind !== "runnerjob") {
|
|
return fields
|
|
.filter(([label]) => label === "Run" || label === "Command" || label === "RunnerJob" || label === "Session")
|
|
.map(([label, item]) => formatResourceField(label, item))
|
|
.filter((line): line is string => line !== null);
|
|
}
|
|
return fields.map(([label, item]) => formatResourceField(label, item)).filter((line): line is string => line !== null);
|
|
}
|
|
|
|
function firstKnownString(...values: unknown[]): string | null {
|
|
for (const value of values) {
|
|
const text = stringOrNull(value);
|
|
if (text !== null && text.length > 0) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function formatResourceField(label: string, value: unknown): string | null {
|
|
const text = stringOrNull(value);
|
|
if (text === null || text.length === 0) return null;
|
|
return `${label}: ${text}`;
|
|
}
|
|
|
|
function renderSupervisorLines(ref: AgentRunResourceRef, value: Record<string, unknown>): string[] {
|
|
if (ref.kind !== "session" && ref.kind !== "task") return [];
|
|
const supervisor = record(value.supervisor);
|
|
const liveness = record(value.liveness);
|
|
const source = Object.keys(supervisor).length > 0 ? supervisor : liveness;
|
|
if (Object.keys(source).length === 0) return [];
|
|
const timeoutBudget = record(source.timeoutBudget);
|
|
const terminalClassification = record(source.terminalClassification);
|
|
const runId = firstKnownString(source.activeRunId, source.runId, source.lastRunId, value.activeRunId, value.runId, value.lastRunId);
|
|
const commandId = firstKnownString(source.activeCommandId, source.commandId, source.lastCommandId, value.activeCommandId, value.commandId, value.lastCommandId);
|
|
const lines: string[] = [];
|
|
const executionState = firstKnownString(source.executionState, value.executionState);
|
|
if (executionState !== null) lines.push(`Execution: ${executionState}`);
|
|
if (typeof source.active === "boolean") lines.push(`Active: ${String(source.active)}`);
|
|
if (runId !== null) lines.push(`Run: run/${runId}`);
|
|
if (commandId !== null) lines.push(`Command: command/${commandId}`);
|
|
const phase = firstKnownString(source.phase, liveness.phase);
|
|
const lastEventAgeMs = typeof source.lastEventAgeMs === "number" ? source.lastEventAgeMs : typeof liveness.lastEventAgeMs === "number" ? liveness.lastEventAgeMs : null;
|
|
if (phase !== null || lastEventAgeMs !== null) lines.push(`Liveness: ${displayValue(phase)}${lastEventAgeMs === null ? "" : ` ageMs=${String(lastEventAgeMs)}`}`);
|
|
const timeoutKind = firstKnownString(timeoutBudget.timeoutKind);
|
|
const timeoutState = firstKnownString(timeoutBudget.state);
|
|
const remainingMs = typeof timeoutBudget.remainingMs === "number" ? timeoutBudget.remainingMs : null;
|
|
if (timeoutKind !== null || timeoutState !== null || remainingMs !== null) lines.push(`Timeout: ${displayValue(timeoutKind)} / ${displayValue(timeoutState)}${remainingMs === null ? "" : ` remainingMs=${String(remainingMs)}`}`);
|
|
const category = firstKnownString(terminalClassification.category);
|
|
const reason = firstKnownString(terminalClassification.reason);
|
|
if (category !== null || reason !== null) lines.push(`Classification: ${displayValue(category)}${reason === null ? "" : `; ${reason}`}`);
|
|
return lines;
|
|
}
|
|
|
|
function renderFailureLines(value: Record<string, unknown>): string[] {
|
|
const terminalClassification = record(value.terminalClassification);
|
|
const nestedTerminalClassification = Object.keys(terminalClassification).length > 0
|
|
? terminalClassification
|
|
: record(record(value.liveness).terminalClassification);
|
|
const failureKind = stringOrNull(value.failureKind)
|
|
?? stringOrNull(nestedTerminalClassification.failureKind)
|
|
?? stringOrNull(value.degradedReason);
|
|
const failureMessage = stringOrNull(value.failureMessage)
|
|
?? stringOrNull(nestedTerminalClassification.failureMessage)
|
|
?? stringOrNull(value.message)
|
|
?? stringOrNull(value.reason);
|
|
const category = stringOrNull(nestedTerminalClassification.category);
|
|
const timeoutKind = stringOrNull(nestedTerminalClassification.timeoutKind);
|
|
const timeoutState = stringOrNull(nestedTerminalClassification.timeoutState);
|
|
const providerInterruption = stringOrNull(nestedTerminalClassification.providerInterruption);
|
|
const lastActivity = stringOrNull(nestedTerminalClassification.lastActivityKind);
|
|
const lastActivitySeq = nestedTerminalClassification.lastActivitySeq;
|
|
const lines: string[] = [];
|
|
if (failureKind !== null) lines.push(`Failure: ${failureKind}`);
|
|
if (failureMessage !== null) lines.push(`Message: ${failureMessage}`);
|
|
if (category !== null) lines.push(`Category: ${category}`);
|
|
if (timeoutKind !== null || timeoutState !== null) lines.push(`Timeout: ${displayValue(timeoutKind)} / ${displayValue(timeoutState)}`);
|
|
if (providerInterruption !== null) lines.push(`Provider: ${providerInterruption}`);
|
|
if (lastActivity !== null) lines.push(`LastActivity: ${lastActivity}${lastActivitySeq === undefined ? "" : ` seq=${String(lastActivitySeq)}`}`);
|
|
return lines;
|
|
}
|
|
|
|
function renderResourceNextLines(ref: AgentRunResourceRef, value: Record<string, unknown>): string[] {
|
|
const supervisor = record(value.supervisor);
|
|
const runId = firstKnownString(value.runId, value.activeRunId, value.lastRunId, supervisor.runId, supervisor.activeRunId, supervisor.lastRunId);
|
|
const commandId = firstKnownString(value.commandId, value.activeCommandId, value.lastCommandId, supervisor.commandId, supervisor.activeCommandId, supervisor.lastCommandId);
|
|
const runnerJobId = stringOrNull(value.runnerJobId) ?? (ref.kind === "runnerjob" ? ref.name : null);
|
|
const sessionId = firstKnownString(value.sessionId, supervisor.sessionId);
|
|
const lines = [
|
|
runId === null ? null : `bun scripts/cli.ts agentrun events run/${runId} --after-seq 0 --limit 100`,
|
|
sessionId === null ? null : `bun scripts/cli.ts agentrun logs session/${sessionId} --tail 100`,
|
|
runId === null || commandId === null ? null : `bun scripts/cli.ts agentrun result run/${runId} --command ${commandId}`,
|
|
runId === null || commandId === null ? null : `bun scripts/cli.ts agentrun describe command/${commandId} --run ${runId} --full`,
|
|
runId === null || runnerJobId === null ? null : `bun scripts/cli.ts agentrun describe runnerjob/${runnerJobId} --run ${runId} --full`,
|
|
ref.kind !== "session" || sessionId === null ? null : `bun scripts/cli.ts agentrun send session/${sessionId} --prompt-stdin`,
|
|
];
|
|
const unique: string[] = [];
|
|
for (const line of lines) {
|
|
if (line !== null && !unique.includes(line)) unique.push(line);
|
|
}
|
|
return unique.slice(0, 5);
|
|
}
|
|
|
|
function parseTaskManifest(raw: string, source: string): Record<string, unknown> {
|
|
const parsed = source.endsWith(".json") ? JSON.parse(raw) as unknown : Bun.YAML.parse(raw) as unknown;
|
|
const input = record(parsed);
|
|
const kind = stringOrNull(input.kind);
|
|
if (kind !== null && kind.toLowerCase() !== "task") throw new Error("apply currently supports kind: Task only");
|
|
const spec = record(input.spec);
|
|
const payload = Object.keys(spec).length > 0 ? spec : input;
|
|
if (!isRecord(payload.payload) && !isRecord(payload.executionPolicy) && stringOrNull(payload.title) === null) {
|
|
throw new Error("task manifest must contain spec with an AgentRun task payload");
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
function nextPagedResourceCommand(command: string, nextAfterSeq: string, limit: number): string {
|
|
const parts = command
|
|
.replace(/\s+--after-seq\s+\S+/gu, "")
|
|
.replace(/\s+--tail(?:\s+\S+)?/gu, "")
|
|
.replace(/\s+--limit\s+\S+/gu, "")
|
|
.trim();
|
|
return `${parts} --after-seq ${nextAfterSeq} --limit ${limit}`;
|
|
}
|
|
|
|
function agentRunExplain(kindRaw: string): string {
|
|
if (kindRaw === "session-policy" || kindRaw === "provider-policy") return renderAgentRunSessionPolicyExplanation();
|
|
const kind = parseResourceKind(kindRaw);
|
|
if (kind === "task") {
|
|
return [
|
|
"KIND: task",
|
|
"FIELDS:",
|
|
" metadata.name: optional client name/idempotency hint",
|
|
" spec.tenantId, spec.projectId, spec.queue, spec.title",
|
|
" spec.payload.prompt or spec.payload.<structured fields>",
|
|
" spec.executionPolicy, spec.resourceBundleRef, spec.workspaceRef",
|
|
"",
|
|
"Apply example:",
|
|
" kind: Task",
|
|
" spec:",
|
|
" tenantId: unidesk",
|
|
" projectId: pikasTech/unidesk",
|
|
" queue: commander",
|
|
" title: Example",
|
|
" payload:",
|
|
" prompt: Do the work",
|
|
].join("\n");
|
|
}
|
|
return `KIND: ${kindRaw}\nUse get/describe with -o json to inspect the current stable UniDesk resource schema.`;
|
|
}
|
|
|
|
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
|
if (Array.isArray(value)) return value.map(record).filter((item) => Object.keys(item).length > 0);
|
|
if (typeof value === "object" && value !== null && Array.isArray((value as { items?: unknown }).items)) return arrayRecords((value as { items: unknown }).items);
|
|
return [];
|
|
}
|
|
|
|
function renderTable(headers: string[], rows: string[][]): string {
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length)));
|
|
const line = (values: string[]): string => values.map((value, index) => value.padEnd(widths[index] ?? value.length)).join(" ").trimEnd();
|
|
return [line(headers), ...rows.map(line)].join("\n");
|
|
}
|
|
|
|
function resourceName(item: Record<string, unknown>): string {
|
|
const kind = String(item.kind ?? "resource");
|
|
return `${kind}/${shortId(displayValue(item.name))}`;
|
|
}
|
|
|
|
function shortId(value: string): string {
|
|
if (value === "-" || value.length <= 18) return value;
|
|
const known = ["qt_", "run_", "cmd_", "rjob_"];
|
|
const prefix = known.find((item) => value.startsWith(item));
|
|
if (prefix !== undefined) return `${prefix}${value.slice(prefix.length, prefix.length + 8)}...`;
|
|
if (/^\w+-\d+/u.test(value)) return value.length > 28 ? `${value.slice(0, 25)}...` : value;
|
|
return `${value.slice(0, 14)}...`;
|
|
}
|
|
|
|
function displayValue(value: unknown): string {
|
|
if (value === undefined || value === null || value === "") return "-";
|
|
return String(value);
|
|
}
|
|
|
|
function stringOrDash(value: unknown): string {
|
|
return displayValue(value);
|
|
}
|
|
|
|
function truncateOneLine(value: string, maxChars: number): string {
|
|
const oneLine = value.replace(/\s+/gu, " ").trim();
|
|
return oneLine.length <= maxChars ? oneLine : `${oneLine.slice(0, Math.max(0, maxChars - 3))}...`;
|
|
}
|
|
|
|
function truncateMultiline(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : `${value.slice(0, maxChars)}\n...[truncated ${value.length - maxChars} chars; rerun with --full-text or --full]`;
|
|
}
|
|
|
|
function relativeAge(iso: string | null): string {
|
|
if (iso === null) return "-";
|
|
const ms = Date.now() - Date.parse(iso);
|
|
if (!Number.isFinite(ms) || ms < 0) return "-";
|
|
const minutes = Math.floor(ms / 60000);
|
|
if (minutes < 60) return `${minutes}m`;
|
|
const hours = Math.floor(minutes / 60);
|
|
if (hours < 48) return `${hours}h`;
|
|
return `${Math.floor(hours / 24)}d`;
|
|
}
|
|
|
|
function pickCompact(value: Record<string, unknown>, keys: string[]): Record<string, unknown> {
|
|
const picked: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
if (value[key] !== undefined) picked[key] = value[key];
|
|
}
|
|
return picked;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
interface TriggerOptions {
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
node: string | null;
|
|
lane: string | null;
|
|
wait: boolean;
|
|
}
|
|
|
|
interface ConfirmOptions {
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface SecretSyncOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
}
|
|
|
|
interface LaneConfirmOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
}
|
|
|
|
interface RefreshOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
}
|
|
|
|
interface GitMirrorStatusOptions extends DisclosureOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
}
|
|
|
|
interface GitMirrorOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
timeoutSeconds: number;
|
|
wait: boolean;
|
|
}
|
|
|
|
interface CleanupRunnersOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
timeoutSeconds: number;
|
|
forceActive: boolean;
|
|
}
|
|
|
|
interface CleanupRunsOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
minAgeMinutes: number;
|
|
limit: number;
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface CleanupReleasedPvOptions extends ConfirmOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
limit: number;
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface DisclosureOptions {
|
|
full: boolean;
|
|
raw: boolean;
|
|
}
|
|
|
|
interface StatusOptions extends DisclosureOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
sourceCommit: string | null;
|
|
pipelineRun: string | null;
|
|
targetMode: "latest-source-head" | "source-commit" | "pipeline-run";
|
|
}
|
|
|
|
interface TimedValue<T> {
|
|
value: T;
|
|
elapsedMs: number;
|
|
}
|
|
|
|
function parseDisclosureOptions(args: string[]): DisclosureOptions {
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === "--node" || arg === "--lane") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg !== "--full" && arg !== "--raw") throw new Error(`unsupported status option: ${arg}`);
|
|
}
|
|
const raw = args.includes("--raw");
|
|
return { full: raw || args.includes("--full"), raw };
|
|
}
|
|
|
|
function parseGitMirrorStatusOptions(args: string[]): GitMirrorStatusOptions {
|
|
validateOptions(args, new Set(["--full", "--raw"]), new Set(["--node", "--lane"]));
|
|
const raw = args.includes("--raw");
|
|
return {
|
|
full: raw || args.includes("--full"),
|
|
raw,
|
|
node: optionValue(args, "--node") ?? null,
|
|
lane: optionValue(args, "--lane") ?? null,
|
|
};
|
|
}
|
|
|
|
function parseStatusOptions(args: string[]): StatusOptions {
|
|
let node: string | null = null;
|
|
let lane: string | null = null;
|
|
let sourceCommit: string | null = null;
|
|
let pipelineRun: string | null = null;
|
|
let full = false;
|
|
let raw = false;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === "--full") {
|
|
full = true;
|
|
continue;
|
|
}
|
|
if (arg === "--raw") {
|
|
raw = true;
|
|
full = true;
|
|
continue;
|
|
}
|
|
if (arg === "--node") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value");
|
|
node = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--lane") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value");
|
|
lane = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--source-commit") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--source-commit requires a value");
|
|
if (!isGitSha(value)) throw new Error("--source-commit must be a full 40-character git SHA");
|
|
sourceCommit = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--pipeline-run") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--pipeline-run requires a value");
|
|
if (!isAgentRunPipelineRunName(value)) throw new Error("--pipeline-run must be an agentrun-vNN-ci-<12+ hex> PipelineRun name");
|
|
pipelineRun = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported status option: ${arg}`);
|
|
}
|
|
if (sourceCommit !== null && pipelineRun !== null) throw new Error("control-plane status accepts only one of --source-commit or --pipeline-run");
|
|
return {
|
|
full,
|
|
raw,
|
|
node,
|
|
lane,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
targetMode: pipelineRun !== null ? "pipeline-run" : sourceCommit !== null ? "source-commit" : "latest-source-head",
|
|
};
|
|
}
|
|
|
|
|
|
function parseTriggerOptions(args: string[]): TriggerOptions {
|
|
const base = parseConfirmOptions(args);
|
|
let node: string | null = null;
|
|
let lane: string | null = null;
|
|
let wait = false;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === "--confirm" || arg === "--dry-run") continue;
|
|
if (arg === "--wait") {
|
|
wait = true;
|
|
continue;
|
|
}
|
|
if (arg === "--node") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value");
|
|
node = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--lane") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value");
|
|
lane = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported trigger-current option: ${arg}`);
|
|
}
|
|
return { ...base, node, lane, wait };
|
|
}
|
|
|
|
function parseSecretSyncOptions(args: string[]): SecretSyncOptions {
|
|
const base = parseConfirmOptions(args);
|
|
let node: string | null = null;
|
|
let lane: string | null = null;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === "--confirm" || arg === "--dry-run") continue;
|
|
if (arg === "--node") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value");
|
|
node = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--lane") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value");
|
|
lane = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported secret-sync option: ${arg}`);
|
|
}
|
|
return { ...base, node, lane };
|
|
}
|
|
|
|
function parseLaneConfirmOptions(args: string[]): LaneConfirmOptions {
|
|
const base = parseConfirmOptions(args);
|
|
let node: string | null = null;
|
|
let lane: string | null = null;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === "--confirm" || arg === "--dry-run") continue;
|
|
if (arg === "--node") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value");
|
|
node = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--lane") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value");
|
|
lane = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported control-plane option: ${arg}`);
|
|
}
|
|
if (node === null && lane === null) throw new Error("control-plane apply requires --node and --lane");
|
|
return { ...base, node, lane };
|
|
}
|
|
|
|
function parseRefreshOptions(args: string[]): RefreshOptions {
|
|
const base = parseConfirmOptions(args);
|
|
let node: string | null = null;
|
|
let lane: string | null = null;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === "--confirm" || arg === "--dry-run") continue;
|
|
if (arg === "--node") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value");
|
|
node = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--lane") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value");
|
|
lane = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported refresh option: ${arg}`);
|
|
}
|
|
return { ...base, node, lane };
|
|
}
|
|
|
|
function parseConfirmOptions(args: string[]): ConfirmOptions {
|
|
if (args.includes("--confirm") && args.includes("--dry-run")) throw new Error("accepts only one of --confirm or --dry-run");
|
|
return {
|
|
confirm: args.includes("--confirm"),
|
|
dryRun: args.includes("--dry-run") || !args.includes("--confirm"),
|
|
};
|
|
}
|
|
|
|
function parseGitMirrorOptions(args: string[]): GitMirrorOptions {
|
|
validateOptions(args, new Set(["--confirm", "--dry-run", "--wait"]), new Set(["--timeout-seconds", "--node", "--lane"]));
|
|
const base = parseConfirmOptions(args);
|
|
const timeoutIndex = args.indexOf("--timeout-seconds");
|
|
const timeoutSeconds = timeoutIndex >= 0 ? Number(args[timeoutIndex + 1]) : 300;
|
|
if (!Number.isFinite(timeoutSeconds) || timeoutSeconds < 30) throw new Error("--timeout-seconds must be a number >= 30");
|
|
return {
|
|
...base,
|
|
node: optionValue(args, "--node") ?? null,
|
|
lane: optionValue(args, "--lane") ?? null,
|
|
timeoutSeconds,
|
|
wait: args.includes("--wait"),
|
|
};
|
|
}
|
|
|
|
function parseCleanupRunnersOptions(args: string[]): CleanupRunnersOptions {
|
|
validateOptions(args, new Set(["--confirm", "--dry-run", "--force-active"]), new Set(["--timeout-seconds", "--node", "--lane"]));
|
|
const base = parseConfirmOptions(args);
|
|
return {
|
|
...base,
|
|
node: optionValue(args, "--node") ?? null,
|
|
lane: optionValue(args, "--lane") ?? null,
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600),
|
|
forceActive: args.includes("--force-active"),
|
|
};
|
|
}
|
|
|
|
function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions {
|
|
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds", "--node", "--lane"]));
|
|
const base = parseConfirmOptions(args);
|
|
return {
|
|
...base,
|
|
node: optionValue(args, "--node") ?? null,
|
|
lane: optionValue(args, "--lane") ?? null,
|
|
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
|
limit: positiveIntegerOption(args, "--limit", 20, 500),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600),
|
|
};
|
|
}
|
|
|
|
function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPvOptions {
|
|
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds", "--node", "--lane"]));
|
|
const base = parseConfirmOptions(args);
|
|
return {
|
|
...base,
|
|
node: optionValue(args, "--node") ?? null,
|
|
lane: optionValue(args, "--lane") ?? null,
|
|
limit: positiveIntegerOption(args, "--limit", 20, 500),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
|
};
|
|
}
|
|
|
|
function validateOptions(args: string[], booleanOptions: Set<string>, valueOptions: Set<string>): void {
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (booleanOptions.has(arg)) continue;
|
|
if (valueOptions.has(arg)) {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function optionValue(args: string[], name: string): string | undefined {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return undefined;
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const raw = optionValue(args, name);
|
|
if (raw === undefined) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
async function controlPlanePlan(_config: UniDeskConfig, options: StatusOptions): Promise<Record<string, unknown>> {
|
|
const target = resolveAgentRunLaneTarget(options);
|
|
const spec = target.spec;
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane plan",
|
|
mode: "yaml-declared-node-lane",
|
|
configPath: target.configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plannedChecks: [
|
|
"source-branch-exists",
|
|
"source-worktree-exists-and-clean",
|
|
"git-mirror-services-ready",
|
|
"ci-namespace-pipeline-serviceaccount",
|
|
"argo-application-alignment",
|
|
"runtime-namespace-manager-service",
|
|
"database-secretref-present",
|
|
"local-postgres-absent-when-external",
|
|
],
|
|
deploymentBoundary: {
|
|
mutation: false,
|
|
note: "plan/status are read-only. Long writes must use controlled AgentRun/Platform DB commands and this YAML target.",
|
|
},
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null,
|
|
postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null,
|
|
controlPlaneApply: `bun scripts/cli.ts agentrun control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --dry-run`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function controlPlaneApply(config: UniDeskConfig, options: LaneConfirmOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const objects = renderAgentRunControlPlaneManifests(spec);
|
|
const manifestYaml = `${objects.map((object) => Bun.YAML.stringify(object).trim()).join("\n---\n")}\n`;
|
|
const objectRefs = objects.map((object) => manifestObjectRef(object));
|
|
const plan = {
|
|
node: spec.nodeId,
|
|
kubeRoute: spec.nodeKubeRoute,
|
|
lane: spec.lane,
|
|
version: spec.version,
|
|
objectCount: objects.length,
|
|
objects: objectRefs,
|
|
manifestBytes: Buffer.byteLength(manifestYaml, "utf8"),
|
|
manifestDigest: renderedObjectsDigest(objects),
|
|
fieldManager: "unidesk-agentrun-control-plane",
|
|
valuesPrinted: false,
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane apply",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const applied = await capture(config, spec.nodeKubeRoute, ["sh", "--", applyYamlScript(manifestYaml, "unidesk-agentrun-control-plane", false)]);
|
|
const payload = captureJsonPayload(applied);
|
|
return {
|
|
ok: applied.exitCode === 0 && payload.ok !== false,
|
|
command: "agentrun control-plane apply",
|
|
mode: "confirmed-apply",
|
|
mutation: true,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
result: payload,
|
|
capture: compactCapture(applied, { full: applied.exitCode !== 0, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
next: {
|
|
secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function status(config: UniDeskConfig, options: StatusOptions): Promise<Record<string, unknown>> {
|
|
return await statusYamlLane(config, options, resolveAgentRunLaneTarget(options));
|
|
}
|
|
|
|
async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise<Record<string, unknown>> {
|
|
const spec = target.spec;
|
|
const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)]));
|
|
const sourcePayload = captureJsonPayload(sourceProbe.value);
|
|
const branchTipCommit = stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead);
|
|
const initialSourceCommit = options.sourceCommit
|
|
?? stringOrNull(sourcePayload.remoteBranchCommit)
|
|
?? stringOrNull(sourcePayload.localHead);
|
|
const pipelineRunName = options.pipelineRun ?? (initialSourceCommit ? agentRunPipelineRunName(spec, initialSourceCommit) : null);
|
|
const [runtimeProbe, mirrorProbe] = await Promise.all([
|
|
timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])),
|
|
timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)])),
|
|
]);
|
|
const runtimePayload = captureJsonPayload(runtimeProbe.value);
|
|
const mirrorPayload = captureJsonPayload(mirrorProbe.value);
|
|
const pipeline = record(runtimePayload.pipeline);
|
|
const pipelineRunStatus = record(runtimePayload.pipelineRun);
|
|
const argo = record(runtimePayload.argo);
|
|
const manager = record(runtimePayload.manager);
|
|
const database = record(runtimePayload.database);
|
|
const secrets = record(runtimePayload.secrets);
|
|
const localPostgres = record(runtimePayload.localPostgres);
|
|
const expectedGitopsRevision = stringOrNull(mirrorPayload.gitopsCommit);
|
|
const mirrorSourceCommit = stringOrNull(mirrorPayload.sourceCommit);
|
|
const pipelineRunSourceCommit = stringOrNull(pipelineRunStatus.sourceCommit);
|
|
const sourceCommit = options.sourceCommit
|
|
?? (options.pipelineRun !== null ? pipelineRunSourceCommit ?? initialSourceCommit : initialSourceCommit);
|
|
const managerSourceMatchesExpected = Boolean(sourceCommit && manager.sourceCommit === sourceCommit);
|
|
const argoSyncedToGitops = Boolean(expectedGitopsRevision && argo.revision === expectedGitopsRevision);
|
|
const pipelineSucceeded = pipelineRunStatus.status === "True";
|
|
const sourceWorktreeDetached = sourcePayload.branch === "HEAD";
|
|
const targetMode = options.pipelineRun !== null ? "pipeline-run" : options.sourceCommit !== null ? "source-commit" : "latest-source-head";
|
|
const sourceBranchAdvanced = targetMode !== "latest-source-head"
|
|
&& sourceCommit !== null
|
|
&& branchTipCommit !== null
|
|
&& sourceCommit !== branchTipCommit;
|
|
const branchDrift = {
|
|
targetMode,
|
|
sourceBranch: spec.source.branch,
|
|
currentBranchTipCommit: branchTipCommit,
|
|
targetSourceCommit: sourceCommit,
|
|
pipelineRunSourceCommit,
|
|
sourceBranchAdvanced,
|
|
targetSupersededByCurrentBranch: sourceBranchAdvanced,
|
|
closeoutHint: sourceBranchAdvanced
|
|
? "target PipelineRun/source commit is not the current branch tip; verify the branch tip contains the fix, then trigger-current again for latest-tip closeout"
|
|
: null,
|
|
valuesPrinted: false,
|
|
};
|
|
const warnings = [
|
|
...(sourceWorktreeDetached ? ["source-worktree-detached"] : []),
|
|
...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []),
|
|
];
|
|
const blockers = [
|
|
...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]),
|
|
...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]),
|
|
...(sourcePayload.workspaceClean === true || sourcePayload.workspaceExists !== true ? [] : ["source-worktree-dirty"]),
|
|
...(mirrorPayload.readReady === true ? [] : ["git-mirror-read-not-ready"]),
|
|
...(mirrorPayload.writeReady === true ? [] : ["git-mirror-write-not-ready"]),
|
|
...(mirrorPayload.cachePvcExists === true ? [] : ["git-mirror-cache-pvc-missing"]),
|
|
...(runtimePayload.ciNamespaceExists === true ? [] : ["ci-namespace-missing"]),
|
|
...(pipeline.exists === true ? [] : ["pipeline-missing"]),
|
|
...(pipelineRunName !== null && pipelineRunStatus.exists !== true ? ["pipelinerun-missing"] : []),
|
|
...(pipelineRunStatus.exists !== true || pipelineRunStatus.status === undefined || pipelineRunStatus.status === null || pipelineRunStatus.status === "True" ? [] : ["pipelinerun-not-succeeded"]),
|
|
...(runtimePayload.serviceAccountExists === true ? [] : ["ci-serviceaccount-missing"]),
|
|
...(argo.exists === true ? [] : ["argo-application-missing"]),
|
|
...(mirrorPayload.readReady === true && expectedGitopsRevision === null ? ["gitops-revision-unresolved"] : []),
|
|
...(argo.exists !== true || expectedGitopsRevision === null || argoSyncedToGitops ? [] : ["argo-revision-stale"]),
|
|
...(runtimePayload.runtimeNamespaceExists === true ? [] : ["runtime-namespace-missing"]),
|
|
...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]),
|
|
...(manager.deploymentExists !== true || sourceCommit === null || managerSourceMatchesExpected ? [] : ["manager-source-stale"]),
|
|
...(manager.serviceExists === true ? [] : ["manager-service-missing"]),
|
|
...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []),
|
|
...(secrets.ready === true ? [] : ["lane-secret-missing"]),
|
|
...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []),
|
|
];
|
|
const ciBlockerNames = new Set(["ci-namespace-missing", "pipeline-missing", "pipelinerun-missing", "pipelinerun-not-succeeded", "ci-serviceaccount-missing"]);
|
|
const ciBlockers = blockers.filter((blocker) => ciBlockerNames.has(blocker));
|
|
const runtimeBlockers = blockers.filter((blocker) => !ciBlockerNames.has(blocker));
|
|
const aligned = blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected;
|
|
const runtimeAligned = runtimeBlockers.length === 0 && argoSyncedToGitops && managerSourceMatchesExpected;
|
|
const ciEvidenceMissing = pipelineRunName !== null && pipelineRunStatus.exists !== true;
|
|
const mirrorAlreadySynced = mirrorPayload.readReady === true
|
|
&& mirrorPayload.writeReady === true
|
|
&& sourceCommit !== null
|
|
&& mirrorSourceCommit === sourceCommit
|
|
&& expectedGitopsRevision !== null;
|
|
const runtimeAlignment = {
|
|
pipelineSucceeded,
|
|
argoRevision: stringOrNull(argo.revision),
|
|
argoSyncedToGitops,
|
|
managerSourceCommit: stringOrNull(manager.sourceCommit),
|
|
managerSourceMatchesExpected,
|
|
runtimeAligned,
|
|
};
|
|
const statusFullCommand = agentRunControlPlaneStatusCommand(spec, options, true);
|
|
const triggerCurrentCommand = `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`;
|
|
const nextAction = aligned
|
|
? { code: "none", summary: "lane aligned; no action required", command: null }
|
|
: ciEvidenceMissing && runtimeAligned && mirrorAlreadySynced
|
|
? { code: "ci-evidence-missing", summary: "runtime and git mirror are aligned, but the expected PipelineRun evidence is missing; rerun trigger-current to recreate CI evidence", command: triggerCurrentCommand }
|
|
: sourceBranchAdvanced
|
|
? { code: "target-superseded", summary: "target commit is no longer the branch tip; trigger-current against latest source before closeout", command: triggerCurrentCommand }
|
|
: ciBlockers.length > 0
|
|
? { code: "ci-blocked", summary: `CI evidence is blocked: ${ciBlockers.join(", ")}`, command: statusFullCommand }
|
|
: runtimeBlockers.length > 0
|
|
? { code: "runtime-blocked", summary: `runtime alignment is blocked: ${runtimeBlockers.join(", ")}`, command: statusFullCommand }
|
|
: { code: "inspect-full-status", summary: "alignment is inconclusive; inspect full status details", command: statusFullCommand };
|
|
const compactSourceStatus = {
|
|
workspaceExists: sourcePayload.workspaceExists ?? false,
|
|
workspaceClean: sourcePayload.workspaceClean ?? null,
|
|
branch: sourcePayload.branch ?? null,
|
|
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
|
|
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
|
|
workspaceDetached: sourceWorktreeDetached,
|
|
};
|
|
const compactGitMirrorStatus = {
|
|
readReady: mirrorPayload.readReady ?? false,
|
|
writeReady: mirrorPayload.writeReady ?? false,
|
|
cachePvcExists: mirrorPayload.cachePvcExists ?? false,
|
|
sourceCommit: mirrorSourceCommit,
|
|
gitopsCommit: expectedGitopsRevision,
|
|
alreadySynced: mirrorAlreadySynced,
|
|
};
|
|
const compactCiStatus = {
|
|
namespaceExists: runtimePayload.ciNamespaceExists ?? false,
|
|
serviceAccountExists: runtimePayload.serviceAccountExists ?? false,
|
|
pipelineExists: pipeline.exists ?? false,
|
|
pipelineRun: {
|
|
name: pipelineRunName,
|
|
exists: pipelineRunStatus.exists ?? false,
|
|
status: pipelineRunStatus.status ?? null,
|
|
reason: pipelineRunStatus.reason ?? null,
|
|
sourceCommit: pipelineRunSourceCommit,
|
|
},
|
|
evidenceMissing: ciEvidenceMissing,
|
|
blockers: ciBlockers,
|
|
};
|
|
const compactArgoStatus = {
|
|
exists: argo.exists ?? false,
|
|
application: argo.application ?? null,
|
|
revision: stringOrNull(argo.revision),
|
|
syncStatus: argo.syncStatus ?? null,
|
|
healthStatus: argo.healthStatus ?? null,
|
|
syncedToGitops: argoSyncedToGitops,
|
|
};
|
|
const compactRuntimeStatus = {
|
|
namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
|
|
manager: {
|
|
deploymentExists: manager.deploymentExists ?? false,
|
|
serviceExists: manager.serviceExists ?? false,
|
|
sourceCommit: stringOrNull(manager.sourceCommit),
|
|
sourceMatchesExpected: managerSourceMatchesExpected,
|
|
},
|
|
databaseSecretPresent: database.secretPresent ?? null,
|
|
secretsReady: secrets.ready ?? null,
|
|
localPostgresAbsent: localPostgres.absent ?? null,
|
|
blockers: runtimeBlockers,
|
|
};
|
|
const detailedSummary = {
|
|
aligned,
|
|
runtimeAligned,
|
|
ciEvidenceMissing,
|
|
mirrorAlreadySynced,
|
|
blockers,
|
|
warnings,
|
|
sourceCommit,
|
|
expectedPipelineRun: pipelineRunName,
|
|
expectedGitopsRevision,
|
|
runtimeAlignment,
|
|
branchDrift,
|
|
source: {
|
|
workspaceExists: sourcePayload.workspaceExists ?? false,
|
|
workspaceClean: sourcePayload.workspaceClean ?? null,
|
|
branch: sourcePayload.branch ?? null,
|
|
workspaceDetached: sourceWorktreeDetached,
|
|
localHead: sourcePayload.localHead ?? null,
|
|
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
|
|
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
|
|
},
|
|
gitMirror: {
|
|
readReady: mirrorPayload.readReady ?? false,
|
|
writeReady: mirrorPayload.writeReady ?? false,
|
|
cachePvcExists: mirrorPayload.cachePvcExists ?? false,
|
|
repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null,
|
|
sourceCommit: mirrorSourceCommit,
|
|
gitopsCommit: expectedGitopsRevision,
|
|
alreadySynced: mirrorAlreadySynced,
|
|
},
|
|
ci: {
|
|
namespaceExists: runtimePayload.ciNamespaceExists ?? false,
|
|
serviceAccountExists: runtimePayload.serviceAccountExists ?? false,
|
|
pipeline,
|
|
pipelineRun: pipelineRunStatus,
|
|
evidenceMissing: ciEvidenceMissing,
|
|
blockers: ciBlockers,
|
|
},
|
|
argo,
|
|
runtime: {
|
|
namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
|
|
manager: {
|
|
deploymentExists: manager.deploymentExists ?? false,
|
|
serviceExists: manager.serviceExists ?? false,
|
|
deployment: manager.deployment ?? null,
|
|
service: manager.service ?? null,
|
|
image: manager.image ?? null,
|
|
sourceCommit: stringOrNull(manager.sourceCommit),
|
|
sourceMatchesExpected: managerSourceMatchesExpected,
|
|
},
|
|
database,
|
|
secrets: compactLaneSecretsStatus(secrets),
|
|
localPostgres,
|
|
},
|
|
nextAction,
|
|
};
|
|
const commanderSummary = {
|
|
aligned,
|
|
runtimeAligned,
|
|
ciEvidenceMissing,
|
|
mirrorAlreadySynced,
|
|
blockers,
|
|
warnings,
|
|
sourceCommit,
|
|
expectedPipelineRun: pipelineRunName,
|
|
expectedGitopsRevision,
|
|
runtimeAlignment,
|
|
branchDrift,
|
|
source: compactSourceStatus,
|
|
gitMirror: compactGitMirrorStatus,
|
|
ci: compactCiStatus,
|
|
argo: compactArgoStatus,
|
|
runtime: compactRuntimeStatus,
|
|
nextAction,
|
|
};
|
|
const result: Record<string, unknown> = {
|
|
ok: sourceProbe.value.exitCode === 0 && runtimeProbe.value.exitCode === 0 && mirrorProbe.value.exitCode === 0 && blockers.length === 0,
|
|
command: "agentrun control-plane status",
|
|
mode: "yaml-declared-node-lane",
|
|
configPath: target.configPath,
|
|
target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec),
|
|
summary: options.full || options.raw ? detailedSummary : commanderSummary,
|
|
alignment: {
|
|
aligned,
|
|
runtimeAligned,
|
|
ciEvidenceMissing,
|
|
mirrorAlreadySynced,
|
|
blockers,
|
|
warnings,
|
|
sourceCommit,
|
|
expectedPipelineRun: pipelineRunName,
|
|
expectedGitopsRevision,
|
|
runtimeAlignment,
|
|
branchDrift,
|
|
},
|
|
timings: {
|
|
sourceMs: sourceProbe.elapsedMs,
|
|
runtimeMs: runtimeProbe.elapsedMs,
|
|
gitMirrorMs: mirrorProbe.elapsedMs,
|
|
totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs),
|
|
},
|
|
disclosure: {
|
|
output: options.full || options.raw ? "full" : "compact-summary",
|
|
full: options.full,
|
|
raw: options.raw,
|
|
omittedWhenCompact: options.full || options.raw ? [] : ["full target YAML summary", "detailed source/runtime/gitMirror objects", "capture stdout/stderr tails for successful probes"],
|
|
fullCommand: statusFullCommand,
|
|
},
|
|
next: {
|
|
action: nextAction,
|
|
plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null,
|
|
postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null,
|
|
secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
statusFull: statusFullCommand,
|
|
triggerCurrent: ciEvidenceMissing || sourceBranchAdvanced ? triggerCurrentCommand : null,
|
|
triggerLatest: sourceBranchAdvanced ? `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
if (options.full || options.raw || sourceProbe.value.exitCode !== 0 || runtimeProbe.value.exitCode !== 0 || mirrorProbe.value.exitCode !== 0) {
|
|
result.captures = {
|
|
source: compactCapture(sourceProbe.value, { full: options.full || options.raw || sourceProbe.value.exitCode !== 0 }),
|
|
runtime: compactCapture(runtimeProbe.value, { full: options.full || options.raw || runtimeProbe.value.exitCode !== 0 }),
|
|
gitMirror: compactCapture(mirrorProbe.value, { full: options.full || options.raw || mirrorProbe.value.exitCode !== 0 }),
|
|
};
|
|
}
|
|
if (options.full || options.raw) {
|
|
result.source = sourcePayload;
|
|
result.runtime = runtimePayload;
|
|
result.gitMirror = mirrorPayload;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function agentRunControlPlaneStatusCommand(spec: AgentRunLaneSpec, options: Pick<StatusOptions, "sourceCommit" | "pipelineRun">, full: boolean): string {
|
|
return [
|
|
"bun scripts/cli.ts agentrun control-plane status",
|
|
`--node ${spec.nodeId}`,
|
|
`--lane ${spec.lane}`,
|
|
options.pipelineRun === null ? null : `--pipeline-run ${options.pipelineRun}`,
|
|
options.sourceCommit === null ? null : `--source-commit ${options.sourceCommit}`,
|
|
full ? "--full" : null,
|
|
].filter(Boolean).join(" ");
|
|
}
|
|
|
|
function renderAgentRunControlPlaneStatusSummary(result: Record<string, unknown>): RenderedCliResult {
|
|
const summary = record(result.summary);
|
|
const target = record(result.target);
|
|
const node = record(target.node);
|
|
const runtimeTarget = record(target.runtime);
|
|
const ciTarget = record(target.ci);
|
|
const source = record(summary.source);
|
|
const gitMirror = record(summary.gitMirror);
|
|
const ci = record(summary.ci);
|
|
const ciRun = record(ci.pipelineRun);
|
|
const argo = record(summary.argo);
|
|
const runtime = record(summary.runtime);
|
|
const runtimeManager = record(runtime.manager);
|
|
const nextAction = record(summary.nextAction);
|
|
const next = record(result.next);
|
|
const disclosure = record(result.disclosure);
|
|
const timings = record(result.timings);
|
|
const blockers = Array.isArray(summary.blockers) ? summary.blockers.map(String) : [];
|
|
const warnings = Array.isArray(summary.warnings) ? summary.warnings.map(String) : [];
|
|
const lines = [
|
|
"AGENTRUN CONTROL-PLANE STATUS",
|
|
renderTable(
|
|
["NODE", "LANE", "NAMESPACE", "SOURCE", "PIPELINE", "ALIGNED", "RUNTIME", "BLOCKERS"],
|
|
[[
|
|
displayValue(node.id ?? target.node ?? "-"),
|
|
displayValue(target.lane ?? "-"),
|
|
displayValue(runtimeTarget.namespace ?? "-"),
|
|
shortSha(summary.sourceCommit),
|
|
displayValue(summary.expectedPipelineRun ?? "-"),
|
|
yesNo(summary.aligned),
|
|
yesNo(summary.runtimeAligned),
|
|
blockers.length === 0 ? "-" : blockers.join(","),
|
|
]],
|
|
),
|
|
"",
|
|
renderTable(
|
|
["COMPONENT", "STATUS", "DETAIL"],
|
|
[
|
|
["source", yesNo(source.workspaceClean), `branch=${displayValue(source.branch)} commit=${shortSha(source.remoteBranchCommit)} detached=${yesNo(source.workspaceDetached)}`],
|
|
["git-mirror", yesNo(gitMirror.alreadySynced), `read=${yesNo(gitMirror.readReady)} write=${yesNo(gitMirror.writeReady)} gitops=${shortSha(gitMirror.gitopsCommit)}`],
|
|
["ci", yesNo(ciRun.status === "True"), `run=${displayValue(ciRun.name)} status=${displayValue(ciRun.status)} reason=${displayValue(ciRun.reason)} evidenceMissing=${yesNo(ci.evidenceMissing)}`],
|
|
["argo", yesNo(argo.syncedToGitops), `sync=${displayValue(argo.syncStatus)} health=${displayValue(argo.healthStatus)} revision=${shortSha(argo.revision)}`],
|
|
["runtime", yesNo(runtimeManager.sourceMatchesExpected), `manager=${shortSha(runtimeManager.sourceCommit)} secrets=${yesNo(runtime.secretsReady)} dbSecret=${yesNo(runtime.databaseSecretPresent)} localPgAbsent=${yesNo(runtime.localPostgresAbsent)}`],
|
|
],
|
|
),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((warning) => `- ${warning}`)].join("\n"),
|
|
"",
|
|
"NEXT",
|
|
` action: ${displayValue(nextAction.summary ?? "-")}`,
|
|
nextAction.command ? ` run: ${displayValue(nextAction.command)}` : null,
|
|
` full: ${displayValue(next.statusFull ?? disclosure.fullCommand ?? "bun scripts/cli.ts agentrun control-plane status --full")}`,
|
|
"",
|
|
`TIMINGS source=${displayValue(timings.sourceMs)}ms runtime=${displayValue(timings.runtimeMs)}ms gitMirror=${displayValue(timings.gitMirrorMs)}ms total=${displayValue(timings.totalMs)}ms`,
|
|
].filter((line): line is string => line !== null);
|
|
return renderedCliResult(result.ok !== false, "agentrun control-plane status", `${lines.join("\n")}\n`);
|
|
}
|
|
|
|
function yesNo(value: unknown): string {
|
|
if (value === true) return "yes";
|
|
if (value === false) return "no";
|
|
if (value === null || value === undefined) return "-";
|
|
return displayValue(value);
|
|
}
|
|
|
|
function shortSha(value: unknown): string {
|
|
const text = displayValue(value);
|
|
if (/^[0-9a-f]{40}$/u.test(text)) return text.slice(0, 12);
|
|
return text;
|
|
}
|
|
|
|
function compactLaneSecretsStatus(secrets: Record<string, unknown>): Record<string, unknown> {
|
|
const items = Array.isArray(secrets.items) ? secrets.items.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
|
const missing = items
|
|
.filter((item) => item.present !== true || item.keyPresent !== true)
|
|
.map((item) => ({
|
|
namespace: item.namespace ?? null,
|
|
name: item.name ?? null,
|
|
key: item.key ?? null,
|
|
present: item.present ?? false,
|
|
keyPresent: item.keyPresent ?? false,
|
|
}));
|
|
return {
|
|
ready: secrets.ready ?? false,
|
|
count: secrets.count ?? items.length,
|
|
missingCount: missing.length,
|
|
missing,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactAgentRunLaneStatusTarget(spec: AgentRunLaneSpec): Record<string, unknown> {
|
|
return {
|
|
node: {
|
|
id: spec.nodeId,
|
|
route: spec.nodeRoute,
|
|
kubeRoute: spec.nodeKubeRoute,
|
|
},
|
|
lane: spec.lane,
|
|
version: spec.version,
|
|
source: {
|
|
repository: spec.source.repository,
|
|
branch: spec.source.branch,
|
|
},
|
|
runtime: {
|
|
namespace: spec.runtime.namespace,
|
|
managerDeployment: spec.runtime.managerDeployment,
|
|
managerService: spec.runtime.managerService,
|
|
},
|
|
ci: {
|
|
namespace: spec.ci.namespace,
|
|
pipeline: spec.ci.pipeline,
|
|
pipelineRunPrefix: spec.ci.pipelineRunPrefix,
|
|
},
|
|
gitops: {
|
|
branch: spec.gitops.branch,
|
|
argoApplication: spec.gitops.argoApplication,
|
|
},
|
|
gitMirror: {
|
|
namespace: spec.gitMirror.namespace,
|
|
readService: spec.gitMirror.readService,
|
|
writeService: spec.gitMirror.writeService,
|
|
repositoryCount: spec.gitMirror.repositories.length,
|
|
},
|
|
database: {
|
|
mode: spec.database.mode,
|
|
provider: spec.database.provider,
|
|
configRef: spec.database.configRef,
|
|
localPostgresExpectedAbsent: spec.database.localPostgresExpectedAbsent,
|
|
valuesPrinted: false,
|
|
},
|
|
secretCount: spec.secrets.length,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function restartYamlLane(config: UniDeskConfig, options: LaneConfirmOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const plan = {
|
|
node: spec.nodeId,
|
|
kubeRoute: spec.nodeKubeRoute,
|
|
lane: spec.lane,
|
|
namespace: spec.runtime.namespace,
|
|
deployment: spec.runtime.managerDeployment,
|
|
annotation: "kubectl.kubernetes.io/restartedAt",
|
|
reason: "reload-lane-secrets-or-runtime-env",
|
|
valuesPrinted: false,
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane restart",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", restartYamlLaneScript(spec)]);
|
|
const payload = captureJsonPayload(result);
|
|
return {
|
|
ok: result.exitCode === 0 && payload.ok !== false,
|
|
command: "agentrun control-plane restart",
|
|
mode: "confirmed-rollout-restart",
|
|
mutation: true,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
result: payload,
|
|
capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }),
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const sources = collectLaneSecretSources(spec);
|
|
if (sources.length === 0) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane secret-sync",
|
|
mode: "yaml-declared-node-lane",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
degradedReason: "lane-secret-sources-not-declared",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const values = sources.map((source) => ({ spec: source, value: readSecretSourceValue(spec, source) }));
|
|
const plan = values.map(({ spec: item, value }) => ({
|
|
id: item.id,
|
|
namespace: item.targetRef.namespace,
|
|
secret: item.targetRef.name,
|
|
key: item.targetRef.key,
|
|
sourceRef: item.sourceRef,
|
|
sourceMode: item.sourceMode,
|
|
sourceKey: item.sourceKey,
|
|
sourcePath: value.redactedPath,
|
|
fingerprint: value.fingerprint,
|
|
valueBytes: value.valueBytes,
|
|
valuesPrinted: false,
|
|
}));
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane secret-sync",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan: { secretCount: plan.length, items: plan, valuesPrinted: false },
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", secretSyncScript(spec, values.map(({ spec: item, value }) => ({ targetRef: item.targetRef, value: value.value })))]);
|
|
const payload = captureJsonPayload(result);
|
|
return {
|
|
ok: result.exitCode === 0 && payload.ok !== false,
|
|
command: "agentrun control-plane secret-sync",
|
|
mode: "confirmed-sync",
|
|
mutation: true,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan: { secretCount: plan.length, items: plan, valuesPrinted: false },
|
|
result: payload,
|
|
capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function exposeAgentRun(_config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const clientConfig = readAgentRunClientConfig();
|
|
const exposure = clientConfig.publicExposure;
|
|
if (!exposure?.enabled) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane expose",
|
|
action: "disabled-by-yaml",
|
|
publicExposure: exposureSummary(exposure),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane expose",
|
|
dryRun: true,
|
|
mutation: false,
|
|
publicExposure: exposureSummary(exposure),
|
|
plan: {
|
|
masterFrps: {
|
|
action: "ensure-allow-port",
|
|
configPath: exposure.masterFrps.configPath,
|
|
containerName: exposure.masterFrps.containerName,
|
|
remotePort: exposure.remotePort,
|
|
},
|
|
masterCaddy: {
|
|
action: "ensure-https-vhost",
|
|
domain: exposure.masterCaddy.domain,
|
|
configPath: exposure.masterCaddy.configPath,
|
|
serviceName: exposure.masterCaddy.serviceName,
|
|
upstreamBaseUrl: exposure.masterCaddy.upstreamBaseUrl,
|
|
responseHeaderTimeoutSeconds: exposure.masterCaddy.responseHeaderTimeoutSeconds,
|
|
},
|
|
},
|
|
next: { confirm: "bun scripts/cli.ts agentrun control-plane expose --confirm" },
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const masterFrps = applyAgentRunMasterFrpsAllowPort(exposure);
|
|
const masterCaddy = applyAgentRunMasterCaddySite(exposure);
|
|
const ok = masterFrps.ok === true && masterCaddy.ok === true;
|
|
return {
|
|
ok,
|
|
command: "agentrun control-plane expose",
|
|
mutation: true,
|
|
publicExposure: exposureSummary(exposure),
|
|
masterFrps,
|
|
masterCaddy,
|
|
validation: {
|
|
publicProbe: `curl -fsS ${exposure.publicBaseUrl.replace(/\/+$/u, "")}/health/readiness`,
|
|
directClient: "bun scripts/cli.ts agentrun get tasks --queue commander -o json",
|
|
valuesPrinted: false,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function exposureSummary(exposure: AgentRunPublicExposure | null | undefined): Record<string, unknown> {
|
|
if (!exposure) return { enabled: false, valuesPrinted: false };
|
|
return {
|
|
enabled: exposure.enabled,
|
|
proxyName: exposure.proxyName,
|
|
remotePort: exposure.remotePort,
|
|
publicBaseUrl: exposure.publicBaseUrl,
|
|
masterBaseUrl: exposure.masterBaseUrl,
|
|
frps: {
|
|
configPath: exposure.masterFrps.configPath,
|
|
containerName: exposure.masterFrps.containerName,
|
|
},
|
|
caddy: {
|
|
enabled: exposure.masterCaddy.enabled,
|
|
domain: exposure.masterCaddy.domain,
|
|
configPath: exposure.masterCaddy.configPath,
|
|
serviceName: exposure.masterCaddy.serviceName,
|
|
upstreamBaseUrl: exposure.masterCaddy.upstreamBaseUrl,
|
|
responseHeaderTimeoutSeconds: exposure.masterCaddy.responseHeaderTimeoutSeconds,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function applyAgentRunMasterFrpsAllowPort(exposure: AgentRunPublicExposure): Record<string, unknown> {
|
|
const pathValue = exposure.masterFrps.configPath;
|
|
const port = exposure.remotePort;
|
|
const container = exposure.masterFrps.containerName;
|
|
if (!existsSync(pathValue)) return { ok: false, error: "master-frps-config-missing", path: pathValue, valuesPrinted: false };
|
|
const before = readFileSync(pathValue, "utf8");
|
|
const alreadyAllowed = frpsAllowPortExists(before, port);
|
|
let backupPath: string | null = null;
|
|
let action = "kept-existing";
|
|
if (!alreadyAllowed) {
|
|
const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z");
|
|
backupPath = `${pathValue}.bak-agentrun-${stamp}`;
|
|
copyFileSync(pathValue, backupPath);
|
|
writeFileSync(pathValue, `${before.replace(/\s*$/u, "")}\n\n[[allowPorts]]\nstart = ${port}\nend = ${port}\n`, "utf8");
|
|
chmodSync(pathValue, statSync(backupPath).mode & 0o777);
|
|
action = "added-allow-port";
|
|
}
|
|
const restart = alreadyAllowed ? null : spawnSync("docker", ["restart", container], { encoding: "utf8" });
|
|
const inspect = spawnSync("docker", ["inspect", "-f", "{{.State.Running}}", container], { encoding: "utf8" });
|
|
const ok = alreadyAllowed || (restart?.status === 0 && inspect.status === 0 && String(inspect.stdout).trim() === "true");
|
|
return {
|
|
ok,
|
|
action,
|
|
path: pathValue,
|
|
backupPath,
|
|
remotePort: port,
|
|
containerName: container,
|
|
restart: restart === null ? null : { exitCode: restart.status, stdoutTail: String(restart.stdout).slice(-1000), stderrTail: String(restart.stderr).slice(-1000) },
|
|
inspect: { exitCode: inspect.status, running: String(inspect.stdout).trim() === "true", stderrTail: String(inspect.stderr).slice(-1000) },
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function applyAgentRunMasterCaddySite(exposure: AgentRunPublicExposure): Record<string, unknown> {
|
|
const caddy = exposure.masterCaddy;
|
|
if (!caddy.enabled) return { ok: true, action: "disabled-by-yaml", valuesPrinted: false };
|
|
const pathValue = caddy.configPath;
|
|
if (!existsSync(pathValue)) return { ok: false, error: "master-caddy-config-missing", path: pathValue, valuesPrinted: false };
|
|
const desiredBlock = renderAgentRunCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds);
|
|
const applied = applyLocalCaddyManagedSite({
|
|
serviceId: "agentrun",
|
|
markerName: "agentrun",
|
|
configPath: pathValue,
|
|
serviceName: caddy.serviceName,
|
|
siteBlock: desiredBlock,
|
|
legacySiteDomain: caddy.domain,
|
|
});
|
|
return {
|
|
...applied,
|
|
domain: caddy.domain,
|
|
upstreamBaseUrl: caddy.upstreamBaseUrl,
|
|
responseHeaderTimeoutSeconds: caddy.responseHeaderTimeoutSeconds,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function renderAgentRunCaddySiteBlock(domain: string, upstreamBaseUrl: string, responseHeaderTimeoutSeconds: number): string {
|
|
const upstream = new URL(upstreamBaseUrl);
|
|
const upstreamHost = `${upstream.hostname}${upstream.port ? `:${upstream.port}` : ""}`;
|
|
return `${domain} {
|
|
encode zstd gzip
|
|
reverse_proxy ${upstreamHost} {
|
|
header_up Host {host}
|
|
header_up X-Real-IP {remote_host}
|
|
transport http {
|
|
dial_timeout 5s
|
|
response_header_timeout ${responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
}`;
|
|
}
|
|
|
|
function frpsAllowPortExists(toml: string, port: number): boolean {
|
|
const sections = toml.split(/(?=\[\[allowPorts\]\])/u);
|
|
return sections.some((section) => {
|
|
if (!section.includes("[[allowPorts]]")) return false;
|
|
const start = section.match(/^\s*start\s*=\s*(\d+)\s*$/mu);
|
|
const end = section.match(/^\s*end\s*=\s*(\d+)\s*$/mu);
|
|
return Number(start?.[1]) === port && Number(end?.[1]) === port;
|
|
});
|
|
}
|
|
|
|
async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): Promise<Record<string, unknown>> {
|
|
return await triggerCurrentYamlLane(config, options, resolveAgentRunLaneTarget(options));
|
|
}
|
|
|
|
async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise<Record<string, unknown>> {
|
|
const spec = target.spec;
|
|
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapProbeScript(spec)]);
|
|
const source = captureJsonPayload(probe);
|
|
const sourceCommit = stringOrNull(source.sourceCommit);
|
|
const remoteBranchExists = source.remoteBranchExists === true;
|
|
const pipelineRun = sourceCommit !== null && isGitSha(sourceCommit) ? agentRunPipelineRunName(spec, sourceCommit) : null;
|
|
const placeholderImage = sourceCommit === null ? null : placeholderAgentRunImage(spec, sourceCommit);
|
|
const renderedFiles = placeholderImage === null ? [] : renderAgentRunGitopsFiles(spec, { sourceCommit, image: placeholderImage });
|
|
const plan = {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
version: spec.version,
|
|
source: {
|
|
workspace: spec.source.workspace,
|
|
remote: spec.source.remote,
|
|
branch: spec.source.branch,
|
|
bootstrapFromBranch: spec.source.bootstrapFromBranch,
|
|
bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds,
|
|
bootstrapPollSeconds: spec.source.bootstrapPollSeconds,
|
|
remoteBranchExists,
|
|
sourceCommit,
|
|
},
|
|
deploymentFormat: spec.deployment.format,
|
|
deploymentTruth: "config/agentrun.yaml",
|
|
removedServiceDeployJson: true,
|
|
pipelineRun,
|
|
imageBuild: {
|
|
repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`,
|
|
containerfile: spec.deployment.manager.imageBuild.containerfile,
|
|
buildArgNames: Object.keys(spec.deployment.manager.imageBuild.buildArgs).sort(),
|
|
timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds,
|
|
pollSeconds: spec.deployment.manager.imageBuild.pollSeconds,
|
|
proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null,
|
|
},
|
|
gitops: {
|
|
branch: spec.gitops.branch,
|
|
path: spec.gitops.path,
|
|
renderedFileCount: renderedFiles.length,
|
|
renderedFilesDigest: renderedFiles.length === 0 ? null : renderedFilesDigest(renderedFiles),
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath: target.configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
sourceProbe: compactCapture(probe, { full: probe.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
if (options.wait) {
|
|
return await triggerCurrentYamlLaneConfirmed(config, spec, target.configPath, true);
|
|
}
|
|
return startAsyncAgentRunJob(
|
|
`agentrun_${spec.lane}_trigger_current`,
|
|
["bun", "scripts/cli.ts", "agentrun", "control-plane", "trigger-current", "--node", spec.nodeId, "--lane", spec.lane, "--confirm", "--wait"],
|
|
`Run AgentRun ${spec.version} YAML-only trigger-current on ${spec.nodeId}`,
|
|
);
|
|
}
|
|
|
|
async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise<Record<string, unknown>> {
|
|
const result = await triggerCurrentYamlLaneConfirmedSteps(config, spec, configPath, waited);
|
|
const restore = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceRestoreScript(spec)]);
|
|
const restorePayload = captureJsonPayload(restore);
|
|
const restoreOk = restore.exitCode === 0 && restorePayload.ok === true;
|
|
const existingWarnings = Array.isArray(result.warnings) ? result.warnings.filter((item): item is string => typeof item === "string") : [];
|
|
return {
|
|
...result,
|
|
warnings: restoreOk ? existingWarnings : Array.from(new Set([...existingWarnings, "source-worktree-restore-failed"])),
|
|
sourceWorkspaceRestore: restorePayload,
|
|
sourceWorkspaceRestoreCapture: compactCapture(restore, { full: !restoreOk, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise<Record<string, unknown>> {
|
|
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
status: "submitting",
|
|
});
|
|
const bootstrapSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapSubmitScript(spec)]);
|
|
const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit);
|
|
if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "source-bootstrap-submit",
|
|
degradedReason: "yaml-lane-source-bootstrap-submit-failed",
|
|
result: bootstrapSubmitPayload,
|
|
capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId));
|
|
const bootstrapPayload = bootstrap.payload;
|
|
const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit);
|
|
if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "source-bootstrap",
|
|
degradedReason: "yaml-lane-source-bootstrap-failed",
|
|
result: bootstrapPayload,
|
|
bootstrapStatus: bootstrap,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const buildSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageSubmitScript(spec, sourceCommit)]);
|
|
const buildSubmitPayload = captureJsonPayload(buildSubmit);
|
|
if (buildSubmit.exitCode !== 0 || buildSubmitPayload.ok === false) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "image-build-submit",
|
|
sourceCommit,
|
|
degradedReason: "yaml-lane-image-build-submit-failed",
|
|
sourceBootstrap: bootstrapPayload,
|
|
result: buildSubmitPayload,
|
|
capture: compactCapture(buildSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const build = await waitForYamlLaneBuildImage(config, spec, sourceCommit, stringOrNull(buildSubmitPayload.jobId));
|
|
const buildPayload = build.payload;
|
|
const digest = stringOrNull(buildPayload.digest);
|
|
const envIdentity = stringOrNull(buildPayload.envIdentity);
|
|
if (build.ok !== true || digest === null || envIdentity === null) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "image-build",
|
|
sourceCommit,
|
|
degradedReason: "yaml-lane-image-build-failed",
|
|
sourceBootstrap: bootstrapPayload,
|
|
buildSubmit: buildSubmitPayload,
|
|
result: buildPayload,
|
|
buildStatus: build,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
|
|
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image });
|
|
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceCommit,
|
|
status: "submitting",
|
|
});
|
|
const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]);
|
|
const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit);
|
|
if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "gitops-publish-submit",
|
|
sourceCommit,
|
|
image,
|
|
degradedReason: "yaml-lane-gitops-publish-submit-failed",
|
|
sourceBootstrap: bootstrapPayload,
|
|
imageBuild: buildPayload,
|
|
result: gitopsSubmitPayload,
|
|
capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId));
|
|
const gitopsPayload = gitops.payload;
|
|
if (gitops.ok !== true || gitopsPayload.ok === false) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "gitops-publish",
|
|
sourceCommit,
|
|
image,
|
|
degradedReason: "yaml-lane-gitops-publish-failed",
|
|
sourceBootstrap: bootstrapPayload,
|
|
imageBuild: buildPayload,
|
|
gitopsPublishSubmit: gitopsSubmitPayload,
|
|
result: gitopsPayload,
|
|
gitopsPublishStatus: gitops,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const mirror = await runYamlLaneGitMirrorSyncJob(config, spec);
|
|
if (mirror.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
phase: "git-mirror-sync",
|
|
sourceCommit,
|
|
image,
|
|
gitops: gitopsPayload,
|
|
degradedReason: "yaml-lane-git-mirror-sync-failed",
|
|
result: mirror,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const pipelineRun = agentRunPipelineRunName(spec, sourceCommit);
|
|
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLanePipelineRunCreateScript(spec, sourceCommit, pipelineRun)]);
|
|
const createPayload = captureJsonPayload(created);
|
|
return {
|
|
ok: created.exitCode === 0 && createPayload.ok !== false,
|
|
command: "agentrun control-plane trigger-current",
|
|
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
|
mutation: true,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
sourceCommit,
|
|
pipelineRun,
|
|
image,
|
|
sourceBootstrap: bootstrapPayload,
|
|
imageBuildSubmit: buildSubmitPayload,
|
|
imageBuild: buildPayload,
|
|
gitopsPublishSubmit: gitopsSubmitPayload,
|
|
renderedFiles: {
|
|
count: renderedFiles.length,
|
|
digest: renderedFilesDigest(renderedFiles),
|
|
},
|
|
gitops: gitopsPayload,
|
|
gitMirror: mirror,
|
|
created: createPayload,
|
|
capture: compactCapture(created, { full: created.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }),
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
statusByPipelineRun: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise<Record<string, unknown>> {
|
|
return await refreshYamlLane(config, options);
|
|
}
|
|
|
|
async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const plan = {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
version: spec.version,
|
|
argoNamespace: spec.gitops.argoNamespace,
|
|
argoApplication: spec.gitops.argoApplication,
|
|
gitopsBranch: spec.gitops.branch,
|
|
mutation: "argocd-hard-refresh",
|
|
valuesPrinted: false,
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun control-plane refresh",
|
|
mode: "dry-run",
|
|
mutation: false,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const refreshed = await capture(config, spec.nodeKubeRoute, ["sh", "--", refreshYamlLaneScript(spec)]);
|
|
const payload = captureJsonPayload(refreshed);
|
|
return {
|
|
ok: refreshed.exitCode === 0 && payload.ok !== false,
|
|
command: "agentrun control-plane refresh",
|
|
mode: "confirmed-yaml-lane",
|
|
mutation: true,
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
plan,
|
|
result: payload,
|
|
refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function cleanupRunners(config: UniDeskConfig, options: CleanupRunnersOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun control-plane cleanup-runners",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: spec.runtime.namespace,
|
|
retention: spec.deployment.runner.retention,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`,
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun control-plane cleanup-runs",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: spec.ci.namespace,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`,
|
|
diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun control-plane cleanup-released-pvs",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: spec.ci.namespace,
|
|
limit: options.limit,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`,
|
|
diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string {
|
|
const retention = spec.deployment.runner.retention;
|
|
const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64");
|
|
const jobNamePrefixesB64 = Buffer.from(JSON.stringify(retention.selectors.jobNamePrefixes), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
`max_runners=${String(retention.maxRunners)}`,
|
|
`cleanup_order=${shQuote(retention.cleanupOrder)}`,
|
|
`active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`,
|
|
`age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`,
|
|
`age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
`force_active=${options.forceActive ? "true" : "false"}`,
|
|
`match_labels_json_b64=${shQuote(matchLabelsB64)}`,
|
|
`job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`,
|
|
"match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)",
|
|
"job_name_prefixes_json=$(printf '%s' \"$job_name_prefixes_json_b64\" | base64 -d)",
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
|
|
"facts_exit=0",
|
|
"set +e",
|
|
"kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'",
|
|
cleanupRunnersFactsNodeScript(),
|
|
"NODE",
|
|
"facts_exit=$?",
|
|
"set -e",
|
|
"if [ \"$facts_exit\" -ne 0 ]; then",
|
|
" printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"",
|
|
"fi",
|
|
"MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupRunnersPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then",
|
|
" xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupRunnersFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(namespace)}`,
|
|
`pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`,
|
|
`min_age_minutes=${String(options.minAgeMinutes)}`,
|
|
`limit=${String(options.limit)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"",
|
|
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
|
|
"NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupRunsPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-names.txt\" ]; then",
|
|
" xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupRunsFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(namespace)}`,
|
|
`limit=${String(options.limit)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
|
"NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupReleasedPvsPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then",
|
|
" xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupReleasedPvsFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`expected_workspace=${shQuote(spec.source.workspace)}`,
|
|
`source_branch=${shQuote(spec.source.branch)}`,
|
|
"workspace_exists=false",
|
|
"workspace_clean=null",
|
|
"local_head=null",
|
|
"branch=null",
|
|
"remote_url=null",
|
|
"remote_branch_exists=false",
|
|
"remote_branch_commit=null",
|
|
"status_short=''",
|
|
"if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then",
|
|
" actual_workspace=$(pwd)",
|
|
" workspace_exists=true",
|
|
" branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
" remote_url=$(git remote get-url origin 2>/dev/null || true)",
|
|
" local_head=$(git rev-parse HEAD 2>/dev/null || true)",
|
|
" status_short=$(git status --short 2>/dev/null || true)",
|
|
" if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi",
|
|
" git fetch origin \"$source_branch\" >/dev/null 2>&1 || true",
|
|
" remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)",
|
|
" if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi",
|
|
"else",
|
|
" actual_workspace=$(pwd)",
|
|
"fi",
|
|
"export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace",
|
|
"node <<'NODE'",
|
|
"function nullable(value) { return value && value !== 'null' ? value : null; }",
|
|
"function booleanValue(value) { if (value === 'true') return true; if (value === 'false') return false; return null; }",
|
|
"const env = process.env;",
|
|
"console.log(JSON.stringify({",
|
|
" ok: env.workspace_exists === 'true',",
|
|
" expectedWorkspace: env.expected_workspace,",
|
|
" actualWorkspace: env.actual_workspace || null,",
|
|
" workspaceExists: env.workspace_exists === 'true',",
|
|
" workspaceClean: booleanValue(env.workspace_clean),",
|
|
" branch: nullable(env.branch),",
|
|
" remoteUrl: nullable(env.remote_url),",
|
|
" localHead: nullable(env.local_head),",
|
|
" remoteBranch: env.source_branch,",
|
|
" remoteBranchExists: env.remote_branch_exists === 'true',",
|
|
" remoteBranchCommit: nullable(env.remote_branch_commit),",
|
|
" statusShort: nullable(env.status_short),",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string {
|
|
return [
|
|
"set +e",
|
|
`runtime_namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`ci_namespace=${shQuote(spec.ci.namespace)}`,
|
|
`pipeline_name=${shQuote(spec.ci.pipeline)}`,
|
|
`pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`,
|
|
`service_account=${shQuote(spec.ci.serviceAccountName)}`,
|
|
`argo_namespace=${shQuote(spec.gitops.argoNamespace)}`,
|
|
`argo_application=${shQuote(spec.gitops.argoApplication)}`,
|
|
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
`manager_service=${shQuote(spec.runtime.managerService)}`,
|
|
`database_secret=${shQuote(spec.database.secretRef.name)}`,
|
|
`database_key=${shQuote(spec.database.secretRef.key)}`,
|
|
`secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`,
|
|
"export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json",
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null",
|
|
"runtime_ns_exit=$?",
|
|
"kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null",
|
|
"ci_ns_exit=$?",
|
|
"kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null",
|
|
"pipeline_exit=$?",
|
|
"kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null",
|
|
"sa_exit=$?",
|
|
"if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi",
|
|
"kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null",
|
|
"argo_exit=$?",
|
|
"kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null",
|
|
"manager_deploy_exit=$?",
|
|
"kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null",
|
|
"manager_svc_exit=$?",
|
|
"kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null",
|
|
"db_secret_exit=$?",
|
|
"SECRET_REFS_JSON=\"$secrets_json\" NODE_TMP=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/secrets.json\"",
|
|
"const fs = require('node:fs');",
|
|
"const cp = require('node:child_process');",
|
|
"const refs = JSON.parse(process.env.SECRET_REFS_JSON || '[]');",
|
|
"const result = [];",
|
|
"for (const ref of refs) {",
|
|
" const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });",
|
|
" let keyPresent = false;",
|
|
" if (out.status === 0) {",
|
|
" try { keyPresent = Object.prototype.hasOwnProperty.call((JSON.parse(out.stdout).data || {}), ref.key); } catch {}",
|
|
" }",
|
|
" result.push({ namespace: ref.namespace, name: ref.name, key: ref.key, present: out.status === 0, keyPresent, valuesPrinted: false });",
|
|
"}",
|
|
"console.log(JSON.stringify({ ready: result.every((item) => item.present && item.keyPresent), count: result.length, items: result, valuesPrinted: false }));",
|
|
"NODE",
|
|
"kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null",
|
|
"NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const dir = process.env.NODE_TMP;",
|
|
"function readJson(name) { try { return JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); } catch { return null; } }",
|
|
"function exists(exitName) { return process.env[exitName] === '0'; }",
|
|
"function condition(obj) { return obj?.status?.conditions?.[0] || null; }",
|
|
"function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }",
|
|
"function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }",
|
|
"const pipelineRun = readJson('pipelinerun.json');",
|
|
"const argo = readJson('argo.json');",
|
|
"const managerDeploy = readJson('manager-deploy.json');",
|
|
"const managerSvc = readJson('manager-svc.json');",
|
|
"const dbSecret = readJson('db-secret.json');",
|
|
"const secrets = readJson('secrets.json') || { ready: true, count: 0, items: [], valuesPrinted: false };",
|
|
"let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}",
|
|
"const c = condition(pipelineRun);",
|
|
"console.log(JSON.stringify({",
|
|
" ok: exists('RUNTIME_NS_EXIT') && exists('CI_NS_EXIT'),",
|
|
" runtimeNamespaceExists: exists('RUNTIME_NS_EXIT'),",
|
|
" ciNamespaceExists: exists('CI_NS_EXIT'),",
|
|
" serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),",
|
|
" pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },",
|
|
" pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },",
|
|
" argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },",
|
|
" manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },",
|
|
" database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },",
|
|
" secrets,",
|
|
" localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`remote=${shQuote(spec.source.remote)}`,
|
|
`branch=${shQuote(spec.source.branch)}`,
|
|
"workspace_exists=false",
|
|
"remote_branch_exists=false",
|
|
"source_commit=null",
|
|
"if [ -d \"$workspace/.git\" ]; then",
|
|
" workspace_exists=true",
|
|
" cd \"$workspace\"",
|
|
" remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)",
|
|
" if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi",
|
|
"else",
|
|
" remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)",
|
|
" if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi",
|
|
"fi",
|
|
"export workspace_exists remote_branch_exists source_commit",
|
|
"node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string {
|
|
const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch;
|
|
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set -eu",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`remote=${shQuote(spec.source.remote)}`,
|
|
`branch=${shQuote(spec.source.branch)}`,
|
|
`bootstrap_branch=${shQuote(bootstrap)}`,
|
|
`state_dir=${shQuote(stateDir)}`,
|
|
"mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"",
|
|
"job_id=\"source-bootstrap-$(date +%s)-$$\"",
|
|
"status_file=\"$state_dir/$job_id.json\"",
|
|
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
|
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
|
"cat > \"$status_file\" <<EOF",
|
|
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"workspace\":\"$workspace\",\"branch\":\"$branch\",\"valuesPrinted\":false}",
|
|
"EOF",
|
|
"(",
|
|
" set -eu",
|
|
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 60 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
|
|
"const code = Number(process.env.CODE || 1);",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi; exit \"$code\"; }",
|
|
" trap write_failed_status EXIT",
|
|
" if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
|
" :",
|
|
" else",
|
|
" rm -rf \"$workspace\"",
|
|
" git clone --no-checkout \"$remote\" \"$workspace\"",
|
|
" fi",
|
|
" cd \"$workspace\"",
|
|
" git remote set-url origin \"$remote\" || git remote add origin \"$remote\"",
|
|
" git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"",
|
|
" if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then",
|
|
" git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"",
|
|
" else",
|
|
" git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"",
|
|
" fi",
|
|
" if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi",
|
|
" git add -A deploy/deploy.json 2>/dev/null || true",
|
|
" if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then",
|
|
" git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'",
|
|
" fi",
|
|
" git push -u origin \"$branch\"",
|
|
" source_commit=$(git rev-parse HEAD)",
|
|
" status_short=$(git status --short)",
|
|
" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
|
|
"console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));",
|
|
"NODE",
|
|
" trap - EXIT",
|
|
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
|
"pid=$!",
|
|
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
|
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } };
|
|
const startedAt = Date.now();
|
|
const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000;
|
|
const pollMs = spec.source.bootstrapPollSeconds * 1000;
|
|
let lastPayload: Record<string, unknown> = {};
|
|
let polls = 0;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]);
|
|
const payload = captureJsonPayload(probe);
|
|
lastPayload = payload;
|
|
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
jobId,
|
|
polls,
|
|
status: stringOrNull(payload.status) ?? "unknown",
|
|
sourceCommit: stringOrNull(payload.sourceCommit),
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (payload.status === "succeeded" && stringOrNull(payload.sourceCommit) !== null) return { ok: payload.ok === true, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
await sleep(pollMs);
|
|
}
|
|
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
|
}
|
|
|
|
function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
|
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set +e",
|
|
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
|
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`branch=${shQuote(spec.source.branch)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"workspace_exists=false",
|
|
"if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi",
|
|
"if [ \"$workspace_exists\" != true ]; then",
|
|
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));",
|
|
"NODE",
|
|
" exit 0",
|
|
"fi",
|
|
"before_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
"before_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
|
"status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
|
"if [ -n \"$status_short\" ]; then",
|
|
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" exit 0",
|
|
"fi",
|
|
"git -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"",
|
|
"fetch_exit=$?",
|
|
"remote_branch_commit=$(git -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)",
|
|
"checkout_exit=1",
|
|
"if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then",
|
|
" git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"",
|
|
" checkout_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/checkout.out\"",
|
|
" : > \"$tmp_dir/checkout.err\"",
|
|
"fi",
|
|
"after_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
|
"after_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
|
"after_status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
|
"fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
|
|
"checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
|
|
"WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'",
|
|
"const fetchExit = Number(process.env.FETCH_EXIT || '1');",
|
|
"const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');",
|
|
"const afterClean = !process.env.AFTER_STATUS_SHORT;",
|
|
"const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;",
|
|
"let failureKind = null;",
|
|
"if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';",
|
|
"else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';",
|
|
"else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';",
|
|
"else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';",
|
|
"else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';",
|
|
"console.log(JSON.stringify({",
|
|
" ok,",
|
|
" status: ok ? 'restored' : 'failed',",
|
|
" failureKind,",
|
|
" workspace: process.env.WORKSPACE,",
|
|
" branch: process.env.BRANCH,",
|
|
" before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },",
|
|
" after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },",
|
|
" remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,",
|
|
" fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },",
|
|
" checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string {
|
|
const build = spec.deployment.manager.imageBuild;
|
|
const noProxy = build.noProxy.join(",");
|
|
const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`;
|
|
const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`;
|
|
const buildArgs = Object.entries(build.buildArgs)
|
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
.map(([key, value]) => `${key}=${value}`);
|
|
const script = [
|
|
"set -eu",
|
|
`workspace=${shQuote(spec.source.workspace)}`,
|
|
`source_commit=${shQuote(sourceCommit)}`,
|
|
`state_dir=${shQuote(stateDir)}`,
|
|
`containerfile=${shQuote(build.containerfile)}`,
|
|
`context_dir=${shQuote(build.context)}`,
|
|
`image_repository=${shQuote(imageRepository)}`,
|
|
`network=${shQuote(build.network)}`,
|
|
`http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`,
|
|
`https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`,
|
|
`no_proxy_value=${shQuote(noProxy)}`,
|
|
`env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`,
|
|
`build_args_json=${shQuote(JSON.stringify(buildArgs))}`,
|
|
"mkdir -p \"$state_dir\"",
|
|
"cd \"$workspace\"",
|
|
"git checkout \"$source_commit\"",
|
|
"env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'",
|
|
"const { createHash } = require('node:crypto');",
|
|
"const { readFileSync, existsSync } = require('node:fs');",
|
|
"const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');",
|
|
"const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
|
|
"const hash = createHash('sha256');",
|
|
"for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }",
|
|
"for (const file of files) { hash.update(file); hash.update('\\0'); if (existsSync(file)) hash.update(readFileSync(file)); hash.update('\\0'); }",
|
|
"process.stdout.write(hash.digest('hex').slice(0, 24));",
|
|
"NODE",
|
|
")",
|
|
"source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)",
|
|
"job_id=\"$source_short-$env_identity\"",
|
|
"status_file=\"$state_dir/$job_id.json\"",
|
|
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
|
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
|
"cat > \"$status_file\" <<EOF",
|
|
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"sourceCommit\":\"$source_commit\",\"envIdentity\":\"$env_identity\",\"image\":\"$image_repository:$env_identity\",\"valuesPrinted\":false}",
|
|
"EOF",
|
|
"(",
|
|
" set -eu",
|
|
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 40 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"",
|
|
"const code = Number(process.env.CODE || 1);",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi; exit \"$code\"; }",
|
|
" trap write_failed_status EXIT",
|
|
" cd \"$workspace\"",
|
|
" image=\"$image_repository:$env_identity\"",
|
|
" args=\"--network $network\"",
|
|
" if [ -n \"$http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$http_proxy_value --build-arg http_proxy=$http_proxy_value\"; fi",
|
|
" if [ -n \"$https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$https_proxy_value --build-arg https_proxy=$https_proxy_value\"; fi",
|
|
" if [ -n \"$no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$no_proxy_value --build-arg no_proxy=$no_proxy_value\"; fi",
|
|
" build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'",
|
|
"const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
|
|
"for (const value of values) console.log(value);",
|
|
"NODE",
|
|
" )",
|
|
" while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done <<EOF",
|
|
"$build_arg_values",
|
|
"EOF",
|
|
" if docker image inspect \"$image\" >/dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi",
|
|
" manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'",
|
|
" fetch_manifest_digest() {",
|
|
" ref=\"$1\"",
|
|
" attempt=1",
|
|
" max_attempts=30",
|
|
" while [ \"$attempt\" -le \"$max_attempts\" ]; do",
|
|
" headers=$(mktemp)",
|
|
" if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$ref\" >\"$headers\"; then",
|
|
" found_digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)",
|
|
" rm -f \"$headers\"",
|
|
" if [ -n \"$found_digest\" ]; then printf '%s' \"$found_digest\"; return 0; fi",
|
|
" else",
|
|
" rm -f \"$headers\"",
|
|
" fi",
|
|
" printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"retrying\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$attempt\" \"$max_attempts\" >&2",
|
|
" sleep 1",
|
|
" attempt=$((attempt + 1))",
|
|
" done",
|
|
" printf '{\"event\":\"agentrun-registry-manifest-probe\",\"status\":\"exhausted\",\"ref\":\"%s\",\"attempt\":\"%s/%s\"}\\n' \"$ref\" \"$max_attempts\" \"$max_attempts\" >&2",
|
|
" return 1",
|
|
" }",
|
|
" docker push \"$image\"",
|
|
" digest=$(fetch_manifest_digest \"$env_identity\" || true)",
|
|
" if [ -n \"$digest\" ]; then verified_digest=$(fetch_manifest_digest \"$digest\" || true); if [ \"$verified_digest\" != \"$digest\" ]; then digest=; fi; fi",
|
|
" STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"",
|
|
"const digest = process.env.DIGEST || null;",
|
|
"console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" trap - EXIT",
|
|
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
|
"pid=$!",
|
|
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
return script;
|
|
}
|
|
|
|
async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
|
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } };
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000;
|
|
let lastPayload: Record<string, unknown> = {};
|
|
let polls = 0;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageStatusScript(spec, jobId)]);
|
|
const payload = captureJsonPayload(probe);
|
|
lastPayload = payload;
|
|
progressEvent("agentrun.yaml-lane.image-build.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceCommit,
|
|
jobId,
|
|
polls,
|
|
status: stringOrNull(payload.status) ?? "unknown",
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000);
|
|
}
|
|
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
|
}
|
|
|
|
function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
|
const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set +e",
|
|
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
|
`stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`,
|
|
`stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`,
|
|
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
|
"if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
|
|
const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
|
|
const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({
|
|
path: file.path,
|
|
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
|
}))), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`state_dir=${shQuote(stateDir)}`,
|
|
`remote=${shQuote(spec.source.remote)}`,
|
|
`gitops_branch=${shQuote(spec.gitops.branch)}`,
|
|
`gitops_root=${shQuote(spec.deployment.gitopsRoot)}`,
|
|
`artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`,
|
|
`files_b64=${shQuote(filesB64)}`,
|
|
"mkdir -p \"$state_dir\"",
|
|
"job_id=\"gitops-publish-$(date +%s)-$$\"",
|
|
"status_file=\"$state_dir/$job_id.json\"",
|
|
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
|
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
|
"publish_root=\"$state_dir/worktrees\"",
|
|
"publish_worktree=\"$publish_root/$job_id\"",
|
|
"cat > \"$status_file\" <<EOF",
|
|
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"publishWorkspace\":\"$publish_worktree\",\"gitopsBranch\":\"$gitops_branch\",\"valuesPrinted\":false}",
|
|
"EOF",
|
|
"(",
|
|
" set -eu",
|
|
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 80 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" PUBLISH_WORKSPACE=\"$publish_worktree\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"",
|
|
"const code = Number(process.env.CODE || 1);",
|
|
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, publishWorkspace: process.env.PUBLISH_WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
" fi; exit \"$code\"; }",
|
|
" trap write_failed_status EXIT",
|
|
" mkdir -p \"$publish_root\"",
|
|
" rm -rf \"$publish_worktree\"",
|
|
" git clone --no-checkout \"$remote\" \"$publish_worktree\"",
|
|
" cd \"$publish_worktree\"",
|
|
" git fetch origin \"$gitops_branch\" || true",
|
|
" if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
|
|
" git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
|
|
" else",
|
|
" git checkout --orphan \"$gitops_branch\"",
|
|
" git rm -rf . >/dev/null 2>&1 || true",
|
|
" fi",
|
|
" git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
|
|
" rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
|
|
" FILES_B64=\"$files_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
|
|
"for (const file of files) {",
|
|
" const target = path.resolve(process.cwd(), file.path);",
|
|
" if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);",
|
|
" fs.mkdirSync(path.dirname(target), { recursive: true });",
|
|
" fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));",
|
|
"}",
|
|
"NODE",
|
|
" git add source.json \"$artifact_catalog\" \"$gitops_root\"",
|
|
" if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
|
|
" git push -u origin \"$gitops_branch\"",
|
|
" gitops_commit=$(git rev-parse HEAD)",
|
|
" rm -rf \"$publish_worktree\"",
|
|
" CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" node <<'NODE' > \"$status_file\"",
|
|
"console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
|
|
"NODE",
|
|
" trap - EXIT",
|
|
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
|
"pid=$!",
|
|
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
|
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } };
|
|
const startedAt = Date.now();
|
|
const timeoutMs = 300_000;
|
|
let lastPayload: Record<string, unknown> = {};
|
|
let polls = 0;
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]);
|
|
const payload = captureJsonPayload(probe);
|
|
lastPayload = payload;
|
|
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sourceCommit,
|
|
jobId,
|
|
polls,
|
|
status: stringOrNull(payload.status) ?? "unknown",
|
|
gitopsCommit: stringOrNull(payload.gitopsCommit),
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
|
await sleep(5_000);
|
|
}
|
|
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
|
}
|
|
|
|
function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
|
const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
|
|
return [
|
|
"set +e",
|
|
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
|
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
const AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS = 5;
|
|
const AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN =
|
|
/(kex_exchange_identification|connection closed by remote host|connection closed by unknown|could not read from remote repository|ssh\.github\.com|github\.com|fetch-pack|early eof|econnreset|read econnreset|proxy-connect|tunnel socket error)/i;
|
|
|
|
function agentRunGitMirrorRetryDelayMs(attempt: number): number {
|
|
return Math.min(15_000, 1_000 * Math.pow(2, Math.max(0, attempt - 1)));
|
|
}
|
|
|
|
function agentRunGitMirrorFailureText(value: unknown): string {
|
|
try {
|
|
return JSON.stringify(value);
|
|
} catch {
|
|
return String(value);
|
|
}
|
|
}
|
|
|
|
function agentRunGitMirrorRetryableFailure(
|
|
spec: AgentRunLaneSpec,
|
|
action: "sync" | "flush",
|
|
attempt: number,
|
|
value: unknown,
|
|
): Record<string, unknown> | null {
|
|
const text = agentRunGitMirrorFailureText(value);
|
|
if (!AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN.test(text)) return null;
|
|
const retryExhausted = attempt >= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS;
|
|
const retryBackoffMs = retryExhausted ? 0 : agentRunGitMirrorRetryDelayMs(attempt);
|
|
return {
|
|
retryable: true,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS,
|
|
retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`,
|
|
retryExhausted,
|
|
retryBackoffMs,
|
|
backoffPolicy: "exponential",
|
|
failureKind: "git-mirror-upstream-transient",
|
|
upstream: "git-mirror-ssh",
|
|
action,
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
message: retryExhausted
|
|
? "AgentRun git-mirror upstream transient failure exhausted retries; stopping instead of continuing silently."
|
|
: "AgentRun git-mirror upstream transient failure; retrying with exponential backoff.",
|
|
next: retryExhausted
|
|
? { stopped: true, reason: "retry-exhausted" }
|
|
: {
|
|
retry: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
|
delayMs: retryBackoffMs,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function agentRunGitMirrorRetrySummary(attempts: Record<string, unknown>[]): Record<string, unknown> {
|
|
const last = attempts.length > 0 ? record(attempts[attempts.length - 1]) : {};
|
|
return {
|
|
policy: "exponential",
|
|
maxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS,
|
|
attempts,
|
|
exhausted: last.retryExhausted === true,
|
|
stopped: last.retryExhausted === true,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise<Record<string, unknown>> {
|
|
const attempts: Record<string, unknown>[] = [];
|
|
for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
|
const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
|
|
const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName);
|
|
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
|
if (created.exitCode !== 0) {
|
|
const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false };
|
|
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
|
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
|
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
|
|
await sleep(agentRunGitMirrorRetryDelayMs(attempt));
|
|
continue;
|
|
}
|
|
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
|
|
}
|
|
const startedAt = Date.now();
|
|
let polls = 0;
|
|
let lastProbe: SshCaptureResult | null = null;
|
|
while (Date.now() - startedAt < 300_000) {
|
|
polls += 1;
|
|
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
|
|
const payload = captureJsonPayload(lastProbe);
|
|
progressEvent("agentrun.yaml-lane.git-mirror.progress", {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
jobName,
|
|
polls,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS,
|
|
retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`,
|
|
succeeded: payload.succeeded === true,
|
|
failed: payload.failed === true,
|
|
elapsedMs: Date.now() - startedAt,
|
|
});
|
|
if (payload.succeeded === true) {
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, polls, elapsedMs: Date.now() - startedAt });
|
|
return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
|
|
}
|
|
if (payload.failed === true) {
|
|
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
|
|
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
|
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
|
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
|
|
await sleep(agentRunGitMirrorRetryDelayMs(attempt));
|
|
break;
|
|
}
|
|
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
|
|
}
|
|
await sleep(5_000);
|
|
}
|
|
if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue;
|
|
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
|
|
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
|
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
|
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
|
|
await sleep(agentRunGitMirrorRetryDelayMs(attempt));
|
|
continue;
|
|
}
|
|
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
|
|
}
|
|
return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
|
|
}
|
|
|
|
function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string {
|
|
const manifest = {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRun,
|
|
namespace: spec.ci.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "agentrun",
|
|
"agentrun.pikastech.local/lane": spec.version,
|
|
"agentrun.pikastech.local/node": spec.nodeId,
|
|
"agentrun.pikastech.local/source-commit": sourceCommit,
|
|
"agentrun.pikastech.local/trigger": "unidesk-yaml-only",
|
|
},
|
|
},
|
|
spec: {
|
|
pipelineRef: { name: spec.ci.pipeline },
|
|
taskRunTemplate: {
|
|
serviceAccountName: spec.ci.serviceAccountName,
|
|
podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } },
|
|
},
|
|
params: [
|
|
{ name: "git-url", value: spec.source.remote },
|
|
{ name: "git-read-url", value: spec.gitMirror.readUrl },
|
|
{ name: "git-write-url", value: spec.gitMirror.writeUrl },
|
|
{ name: "source-branch", value: spec.source.branch },
|
|
{ name: "gitops-branch", value: spec.gitops.branch },
|
|
{ name: "revision", value: sourceCommit },
|
|
{ name: "registry-prefix", value: spec.ci.registryPrefix },
|
|
{ name: "tools-image", value: spec.ci.toolsImage },
|
|
],
|
|
workspaces: [
|
|
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "4Gi" } } } } },
|
|
{ name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName } },
|
|
],
|
|
},
|
|
};
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(spec.ci.namespace)}`,
|
|
`pipeline_run=${shQuote(pipelineRun)}`,
|
|
`manifest_b64=${shQuote(manifestB64)}`,
|
|
"tmp=$(mktemp)",
|
|
"trap 'rm -f \"$tmp\"' EXIT",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"",
|
|
"existing_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
|
|
"if [ \"$existing_status\" = False ]; then kubectl -n \"$namespace\" delete pipelinerun \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true; fi",
|
|
"if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then created=false; else kubectl create -f \"$tmp\"; created=true; fi",
|
|
"status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)",
|
|
"CREATED=\"$created\" STATUS=\"$status\" PIPELINE_RUN=\"$pipeline_run\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, pipelineRun: process.env.PIPELINE_RUN, created: process.env.CREATED === 'true', status: process.env.STATUS || null, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function createYamlLaneJobScript(namespace: string, jobName: string, manifest: Record<string, unknown>): string {
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(namespace)}`,
|
|
`job=${shQuote(jobName)}`,
|
|
`manifest_b64=${shQuote(manifestB64)}`,
|
|
"tmp=$(mktemp)",
|
|
"trap 'rm -f \"$tmp\"' EXIT",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"",
|
|
"kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
|
|
"kubectl create -f \"$tmp\"",
|
|
"JOB=\"$job\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, jobName: process.env.JOB, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneJobProbeScript(namespace: string, jobName: string): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shQuote(namespace)}`,
|
|
`job=${shQuote(jobName)}`,
|
|
"kubectl -n \"$namespace\" get job \"$job\" -o json > /tmp/agentrun-job.json 2>/dev/null",
|
|
"job_exit=$?",
|
|
"kubectl -n \"$namespace\" logs \"job/$job\" --tail=120 > /tmp/agentrun-job.log 2>/dev/null",
|
|
"JOB_EXIT=\"$job_exit\" JOB=\"$job\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"let job = null; try { job = JSON.parse(fs.readFileSync('/tmp/agentrun-job.json', 'utf8')); } catch {}",
|
|
"let log = ''; try { log = fs.readFileSync('/tmp/agentrun-job.log', 'utf8'); } catch {}",
|
|
"const succeeded = Number(job?.status?.succeeded || 0) > 0;",
|
|
"const failed = Number(job?.status?.failed || 0) > 0;",
|
|
"console.log(JSON.stringify({ ok: process.env.JOB_EXIT === '0', jobName: process.env.JOB, succeeded, failed, active: job?.status?.active || 0, logsTail: log.slice(-4000), valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneGitMirrorCacheVolume(spec: AgentRunLaneSpec): Record<string, unknown> {
|
|
if (spec.gitMirror.cacheHostPath !== null) {
|
|
return { name: "cache", hostPath: { path: spec.gitMirror.cacheHostPath, type: "DirectoryOrCreate" } };
|
|
}
|
|
return { name: "cache", persistentVolumeClaim: { claimName: spec.gitMirror.cachePvc } };
|
|
}
|
|
|
|
function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync" | "flush", name: string): Record<string, unknown> {
|
|
const proxyHostNeedsHostNetwork = spec.gitMirror.githubProxy.host === "127.0.0.1" || spec.gitMirror.githubProxy.host === "localhost";
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name,
|
|
namespace: spec.gitMirror.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "agentrun",
|
|
"agentrun.pikastech.local/lane": spec.version,
|
|
"agentrun.pikastech.local/node": spec.nodeId,
|
|
"agentrun.pikastech.local/component": action,
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 600,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "agentrun",
|
|
"agentrun.pikastech.local/lane": spec.version,
|
|
"agentrun.pikastech.local/node": spec.nodeId,
|
|
"agentrun.pikastech.local/component": action,
|
|
},
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
...(proxyHostNeedsHostNetwork ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}),
|
|
volumes: [
|
|
yamlLaneGitMirrorCacheVolume(spec),
|
|
{ name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName, defaultMode: 0o400 } },
|
|
],
|
|
containers: [{
|
|
name: action,
|
|
image: spec.gitMirror.toolsImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", action === "sync" ? yamlLaneGitMirrorSyncShell(spec) : yamlLaneGitMirrorFlushShell(spec)],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function yamlLaneGitMirrorSshSetupShellLines(spec: AgentRunLaneSpec): string[] {
|
|
const proxyHost = spec.gitMirror.githubProxy.host;
|
|
const proxyPort = spec.gitMirror.githubProxy.port;
|
|
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
|
const proxySummary = `agentrun git-mirror-egress-proxy node=${spec.nodeId} lane=${spec.lane} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
|
|
const proxyCommand = `ProxyCommand=node /tmp/agentrun-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
|
|
"cat > /tmp/agentrun-github-proxy-connect.cjs <<'NODE_PROXY'",
|
|
"#!/usr/bin/env node",
|
|
"const net = require('node:net');",
|
|
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
|
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
|
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
|
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
|
|
" console.error('agentrun git-mirror proxy-connect: invalid ProxyCommand arguments');",
|
|
" process.exit(64);",
|
|
"}",
|
|
"let settled = false;",
|
|
"let tunnelEstablished = false;",
|
|
"function finish(code, message) {",
|
|
" if (settled) return;",
|
|
" settled = true;",
|
|
" if (message) console.error('agentrun git-mirror proxy-connect: ' + message);",
|
|
" process.exit(code);",
|
|
"}",
|
|
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
|
"let buffer = Buffer.alloc(0);",
|
|
"socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
|
|
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
|
|
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
|
|
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
|
|
"function onData(chunk) {",
|
|
" buffer = Buffer.concat([buffer, chunk]);",
|
|
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
|
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
|
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
|
|
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
|
|
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
|
|
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
|
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
|
|
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
|
|
" socket.destroy();",
|
|
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
|
|
" return;",
|
|
" }",
|
|
" socket.off('data', onData);",
|
|
" socket.setTimeout(0);",
|
|
" tunnelEstablished = true;",
|
|
" const rest = buffer.slice(headerEnd + 4);",
|
|
" if (rest.length) process.stdout.write(rest);",
|
|
" process.stdin.on('error', () => {});",
|
|
" process.stdout.on('error', () => {});",
|
|
" process.stdin.pipe(socket);",
|
|
" socket.pipe(process.stdout);",
|
|
"}",
|
|
"socket.on('data', onData);",
|
|
"NODE_PROXY",
|
|
"chmod 0700 /tmp/agentrun-github-proxy-connect.cjs",
|
|
"cat > /tmp/agentrun-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/agentrun-git-ssh-proxy.sh",
|
|
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
|
|
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
|
|
`export ALL_PROXY=${shQuote(proxyUrl)}`,
|
|
`export http_proxy=${shQuote(proxyUrl)}`,
|
|
`export https_proxy=${shQuote(proxyUrl)}`,
|
|
`export all_proxy=${shQuote(proxyUrl)}`,
|
|
"export GIT_SSH=/tmp/agentrun-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
|
|
function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set -eu",
|
|
...yamlLaneGitMirrorSshSetupShellLines(spec),
|
|
`repository=${shQuote(spec.source.repository)}`,
|
|
`source_branch=${shQuote(spec.source.branch)}`,
|
|
`gitops_branch=${shQuote(spec.gitops.branch)}`,
|
|
"repo=\"/cache/${repository}.git\"",
|
|
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
|
"mkdir -p \"$(dirname \"$repo\")\"",
|
|
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
|
|
" git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"",
|
|
"else",
|
|
" rm -rf \"$repo\"",
|
|
" git init --bare \"$repo\"",
|
|
" git --git-dir=\"$repo\" remote add origin \"$remote\"",
|
|
"fi",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
|
|
"git --git-dir=\"$repo\" config http.uploadpack true",
|
|
"git --git-dir=\"$repo\" config http.receivepack true",
|
|
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"",
|
|
"source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")",
|
|
"git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"",
|
|
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"",
|
|
"gitops_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\")",
|
|
"git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$gitops_sha\"",
|
|
"git --git-dir=\"$repo\" update-server-info",
|
|
"SOURCE_SHA=\"$source_sha\" GITOPS_SHA=\"$gitops_sha\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: true, localSource: process.env.SOURCE_SHA, localGitops: process.env.GITOPS_SHA, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneGitMirrorFlushShell(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set -eu",
|
|
...yamlLaneGitMirrorSshSetupShellLines(spec),
|
|
`repository=${shQuote(spec.source.repository)}`,
|
|
`gitops_branch=${shQuote(spec.gitops.branch)}`,
|
|
"repo=\"/cache/${repository}.git\"",
|
|
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
|
"test -d \"$repo\"",
|
|
"git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"",
|
|
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
|
|
"if [ -n \"$local_gitops\" ]; then",
|
|
" git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"",
|
|
" git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"",
|
|
"fi",
|
|
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
|
|
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
|
|
"LOCAL_GITOPS=\"$local_gitops\" GITHUB_GITOPS=\"$github_gitops\" PENDING=\"$pending\" node <<'NODE'",
|
|
"console.log(JSON.stringify({ ok: process.env.PENDING !== 'true', localGitops: process.env.LOCAL_GITOPS || null, githubGitops: process.env.GITHUB_GITOPS || null, pendingFlush: process.env.PENDING === 'true', valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shQuote(spec.gitMirror.namespace)}`,
|
|
`read_deployment=${shQuote(spec.gitMirror.readDeployment)}`,
|
|
`read_service=${shQuote(spec.gitMirror.readService)}`,
|
|
`write_service=${shQuote(spec.gitMirror.writeService)}`,
|
|
`cache_pvc=${shQuote(spec.gitMirror.cachePvc)}`,
|
|
`repository=${shQuote(spec.source.repository)}`,
|
|
`source_branch=${shQuote(spec.source.branch)}`,
|
|
`gitops_branch=${shQuote(spec.gitops.branch)}`,
|
|
`repositories_json=${shQuote(JSON.stringify(spec.gitMirror.repositories))}`,
|
|
"kubectl -n \"$namespace\" get svc \"$read_service\" -o json >/tmp/agentrun-gitmirror-read.json 2>/dev/null",
|
|
"read_exit=$?",
|
|
"kubectl -n \"$namespace\" get svc \"$write_service\" -o json >/tmp/agentrun-gitmirror-write.json 2>/dev/null",
|
|
"write_exit=$?",
|
|
"kubectl -n \"$namespace\" get pvc \"$cache_pvc\" -o json >/tmp/agentrun-gitmirror-cache.json 2>/dev/null",
|
|
"cache_exit=$?",
|
|
"kubectl -n \"$namespace\" get deploy,svc,pvc -o name > /tmp/agentrun-gitmirror-names.txt 2>/dev/null",
|
|
"repo_path=\"/cache/${repository}.git\"",
|
|
"rm -f /tmp/agentrun-gitmirror-refs.txt",
|
|
"if [ \"$read_exit\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; gitops_branch=\"$3\"; printf \"sourceCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$source_branch\" 2>/dev/null || true; printf \"gitopsCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$gitops_branch\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" \"$gitops_branch\" > /tmp/agentrun-gitmirror-refs.txt 2>/dev/null",
|
|
"fi",
|
|
"NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" REPOSITORY=\"$repository\" SOURCE_BRANCH=\"$source_branch\" GITOPS_BRANCH=\"$gitops_branch\" REPOSITORIES_JSON=\"$repositories_json\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');",
|
|
"let names = ''; try { names = fs.readFileSync('/tmp/agentrun-gitmirror-names.txt', 'utf8'); } catch {}",
|
|
"let refs = ''; try { refs = fs.readFileSync('/tmp/agentrun-gitmirror-refs.txt', 'utf8'); } catch {}",
|
|
"const refValue = (key) => refs.split(/\\r?\\n/).find((line) => line.startsWith(`${key}=`))?.slice(key.length + 1).trim() || null;",
|
|
"const readReady = process.env.READ_EXIT === '0';",
|
|
"const writeReady = process.env.WRITE_EXIT === '0';",
|
|
"const cachePvcExists = process.env.CACHE_EXIT === '0';",
|
|
"console.log(JSON.stringify({",
|
|
" ok: readReady && writeReady && cachePvcExists,",
|
|
" namespace: process.env.NAMESPACE,",
|
|
" readReady,",
|
|
" writeReady,",
|
|
" cachePvcExists,",
|
|
" resources: names.split(/\\r?\\n/).filter(Boolean).slice(0, 40),",
|
|
" repository: process.env.REPOSITORY,",
|
|
" sourceBranch: process.env.SOURCE_BRANCH,",
|
|
" gitopsBranch: process.env.GITOPS_BRANCH,",
|
|
" sourceCommit: refValue('sourceCommit'),",
|
|
" gitopsCommit: refValue('gitopsCommit'),",
|
|
" repositories,",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
type LaneSecretSource = {
|
|
id: string;
|
|
sourceRef: string;
|
|
sourceMode: "env" | "file";
|
|
sourceKey: string | null;
|
|
targetRef: { namespace: string; name: string; key: string };
|
|
};
|
|
|
|
function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource): { redactedPath: string; value: string; valueBytes: number; fingerprint: string } {
|
|
const { sourceRef } = source;
|
|
if (sourceRef.includes("..")) throw new Error(`secret sourceRef must not contain ..: ${sourceRef}`);
|
|
const sourcePath = sourceRef.startsWith("/")
|
|
? sourceRef
|
|
: join(resolveSecretSourceRoot(spec), ...sourceRef.split("/"));
|
|
if (!existsSync(sourcePath)) throw new Error(`secret source ${sourceRef} is missing`);
|
|
let value: string;
|
|
if (source.sourceMode === "file") {
|
|
value = readFileSync(sourcePath, "utf8");
|
|
} else {
|
|
if (source.sourceKey === null) throw new Error(`secret source ${sourceRef} is missing sourceKey`);
|
|
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
|
|
const envValue = values.get(source.sourceKey);
|
|
if (envValue === undefined || envValue.length === 0) throw new Error(`secret source ${sourceRef} is missing required key ${source.sourceKey}`);
|
|
value = envValue;
|
|
}
|
|
if (value.length === 0) throw new Error(`secret source ${sourceRef} is empty`);
|
|
return {
|
|
redactedPath: sourceRef.startsWith("/") ? redactAbsoluteSecretPath(sourceRef) : `.state/secrets/${sourceRef}`,
|
|
value,
|
|
valueBytes: Buffer.byteLength(value, "utf8"),
|
|
fingerprint: sha256Fingerprint(value),
|
|
};
|
|
}
|
|
|
|
function redactAbsoluteSecretPath(sourceRef: string): string {
|
|
const parts = sourceRef.split("/").filter(Boolean);
|
|
return parts.length === 0 ? "/" : `/${parts.slice(0, -1).join("/")}/<redacted>`;
|
|
}
|
|
|
|
function resolveSecretSourceRoot(spec: AgentRunLaneSpec): string {
|
|
if (spec.database.configRef === null) return rootPath(".state", "secrets");
|
|
const configPath = spec.database.configRef.startsWith("/") ? spec.database.configRef : rootPath(spec.database.configRef);
|
|
const configRoot = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
|
|
const secrets = record(configRoot.secrets);
|
|
const root = stringOrNull(secrets.root);
|
|
if (root === null || !root.startsWith("/") || root.includes("..")) throw new Error(`${spec.database.configRef}.secrets.root must be an absolute path without ..`);
|
|
return root;
|
|
}
|
|
|
|
function parseEnvFile(text: string): Map<string, string> {
|
|
const result = new Map<string, string>();
|
|
for (const rawLine of text.split(/\r?\n/u)) {
|
|
let line = rawLine.trim();
|
|
if (line.length === 0 || line.startsWith("#")) continue;
|
|
if (line.startsWith("export ")) line = line.slice("export ".length).trim();
|
|
const index = line.indexOf("=");
|
|
if (index <= 0) continue;
|
|
const key = line.slice(0, index).trim();
|
|
let value = line.slice(index + 1).trim();
|
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
result.set(key, value);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function collectLaneSecretSources(spec: AgentRunLaneSpec): LaneSecretSource[] {
|
|
const result: LaneSecretSource[] = [];
|
|
if (spec.database.secretSourceRef !== null) {
|
|
result.push({
|
|
id: "database",
|
|
sourceRef: spec.database.secretSourceRef,
|
|
sourceMode: "env",
|
|
sourceKey: spec.database.secretRef.key,
|
|
targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: spec.database.secretRef.key },
|
|
});
|
|
}
|
|
for (const secret of spec.secrets) {
|
|
result.push({
|
|
id: secret.id,
|
|
sourceRef: secret.sourceRef,
|
|
sourceMode: secret.sourceMode,
|
|
sourceKey: secret.sourceKey,
|
|
targetRef: secret.targetRef,
|
|
});
|
|
}
|
|
const seen = new Set<string>();
|
|
return result.filter((item) => {
|
|
const key = `${item.targetRef.namespace}/${item.targetRef.name}/${item.targetRef.key}`;
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ targetRef: { namespace: string; name: string; key: string }; value: string }>): string {
|
|
const encoded = Buffer.from(JSON.stringify(values.map((item) => ({
|
|
targetRef: item.targetRef,
|
|
valueBase64: Buffer.from(item.value, "utf8").toString("base64"),
|
|
}))), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`payload_b64=${shQuote(encoded)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"payload_json=\"$tmp_dir/payload.json\"",
|
|
"printf '%s' \"$payload_b64\" | base64 -d > \"$payload_json\"",
|
|
"PAYLOAD_JSON=\"$payload_json\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const cp = require('node:child_process');",
|
|
"const crypto = require('node:crypto');",
|
|
"const items = JSON.parse(fs.readFileSync(process.env.PAYLOAD_JSON, 'utf8'));",
|
|
"const results = [];",
|
|
"const groups = new Map();",
|
|
"function run(argv, input) {",
|
|
" const out = cp.spawnSync(argv[0], argv.slice(1), { input, encoding: 'utf8' });",
|
|
" if (out.status !== 0) throw new Error(`${argv.join(' ')} failed: ${out.stderr || out.stdout}`);",
|
|
" return out;",
|
|
"}",
|
|
"function tryRun(argv) {",
|
|
" return cp.spawnSync(argv[0], argv.slice(1), { encoding: 'utf8' });",
|
|
"}",
|
|
"for (let index = 0; index < items.length; index += 1) {",
|
|
" const item = items[index];",
|
|
" const ref = item.targetRef;",
|
|
" const groupKey = `${ref.namespace}\\u0000${ref.name}`;",
|
|
" if (!groups.has(groupKey)) groups.set(groupKey, { namespace: ref.namespace, name: ref.name, items: [] });",
|
|
" groups.get(groupKey).items.push({ key: ref.key, valueBase64: item.valueBase64 });",
|
|
"}",
|
|
"for (const group of groups.values()) {",
|
|
" const ns = run(['kubectl', 'create', 'namespace', group.namespace, '--dry-run=client', '-o', 'yaml']).stdout;",
|
|
" run(['kubectl', 'apply', '--server-side', '--field-manager=unidesk-agentrun-secret-sync', '-f', '-'], ns);",
|
|
" const existing = tryRun(['kubectl', '-n', group.namespace, 'get', 'secret', group.name]);",
|
|
" if (existing.status !== 0) run(['kubectl', '-n', group.namespace, 'create', 'secret', 'generic', group.name]);",
|
|
" const patch = { data: {} };",
|
|
" for (const entry of group.items) patch.data[entry.key] = entry.valueBase64;",
|
|
" const patchFile = `${process.env.TMP_DIR}/${group.namespace}-${group.name}-patch.json`;",
|
|
" fs.writeFileSync(patchFile, JSON.stringify(patch));",
|
|
" run(['kubectl', '-n', group.namespace, 'patch', 'secret', group.name, '--type=merge', `--patch-file=${patchFile}`]);",
|
|
" const fetched = JSON.parse(run(['kubectl', '-n', group.namespace, 'get', 'secret', group.name, '-o', 'json']).stdout);",
|
|
" for (const entry of group.items) {",
|
|
" const raw = fetched.data?.[entry.key] || '';",
|
|
" const decoded = raw ? Buffer.from(raw, 'base64') : Buffer.alloc(0);",
|
|
" results.push({ namespace: group.namespace, secret: group.name, key: entry.key, ok: raw.length > 0, valueBytes: decoded.length, fingerprint: raw ? 'sha256:' + crypto.createHash('sha256').update(decoded).digest('hex') : null, valuesPrinted: false });",
|
|
" }",
|
|
"}",
|
|
"console.log(JSON.stringify({ ok: results.every((item) => item.ok), secretCount: results.length, items: results, valuesPrinted: false }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function restartYamlLaneScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
"ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"kubectl -n \"$namespace\" patch deployment \"$deployment\" --type='merge' -p \"{\\\"spec\\\":{\\\"template\\\":{\\\"metadata\\\":{\\\"annotations\\\":{\\\"kubectl.kubernetes.io/restartedAt\\\":\\\"$ts\\\"}}}}}\" >/tmp/agentrun-restart-patch.out 2>/tmp/agentrun-restart-patch.err",
|
|
"patch_rc=$?",
|
|
"if [ \"$patch_rc\" -eq 0 ]; then",
|
|
" kubectl -n \"$namespace\" rollout status deployment/\"$deployment\" --timeout=180s >/tmp/agentrun-rollout-status.out 2>/tmp/agentrun-rollout-status.err",
|
|
" rollout_rc=$?",
|
|
"else",
|
|
" rollout_rc=$patch_rc",
|
|
"fi",
|
|
"pods_json=$(kubectl -n \"$namespace\" get pod -l app.kubernetes.io/name=agentrun-mgr -o json 2>/dev/null || printf '{}')",
|
|
"deployment_json=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o json 2>/dev/null || printf '{}')",
|
|
"patch_err=$(cat /tmp/agentrun-restart-patch.err 2>/dev/null || true)",
|
|
"rollout_err=$(cat /tmp/agentrun-rollout-status.err 2>/dev/null || true)",
|
|
"PODS_JSON=\"$pods_json\" DEPLOYMENT_JSON=\"$deployment_json\" PATCH_RC=\"$patch_rc\" ROLLOUT_RC=\"$rollout_rc\" TS=\"$ts\" PATCH_ERR=\"$patch_err\" ROLLOUT_ERR=\"$rollout_err\" node <<'NODE'",
|
|
"const pods = JSON.parse(process.env.PODS_JSON || '{}');",
|
|
"const deployment = JSON.parse(process.env.DEPLOYMENT_JSON || '{}');",
|
|
"const conditions = Array.isArray(deployment.status?.conditions) ? deployment.status.conditions : [];",
|
|
"const items = Array.isArray(pods.items) ? pods.items : [];",
|
|
"const compactError = (value) => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, 500) || null;",
|
|
"console.log(JSON.stringify({",
|
|
" ok: process.env.PATCH_RC === '0' && process.env.ROLLOUT_RC === '0',",
|
|
" restartedAt: process.env.TS,",
|
|
" patch: { exitCode: Number(process.env.PATCH_RC || '1'), error: compactError(process.env.PATCH_ERR) },",
|
|
" rollout: { exitCode: Number(process.env.ROLLOUT_RC || '1'), error: compactError(process.env.ROLLOUT_ERR) },",
|
|
" readyReplicas: deployment.status?.readyReplicas ?? null,",
|
|
" replicas: deployment.status?.replicas ?? null,",
|
|
" conditions: conditions.map((item) => ({ type: item.type, status: item.status, reason: item.reason ?? null })),",
|
|
" pods: items.map((pod) => ({",
|
|
" name: pod.metadata?.name ?? null,",
|
|
" phase: pod.status?.phase ?? null,",
|
|
" ready: (pod.status?.containerStatuses ?? []).map((item) => item.ready),",
|
|
" restarts: (pod.status?.containerStatuses ?? []).map((item) => item.restartCount),",
|
|
" waiting: (pod.status?.containerStatuses ?? []).map((item) => item.state?.waiting?.reason ?? null),",
|
|
" })),",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
"exit \"$rollout_rc\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function applyYamlScript(yaml: string, fieldManager: string, dryRun: boolean): string {
|
|
const encoded = Buffer.from(yaml, "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`manifest_b64=${shQuote(encoded)}`,
|
|
`field_manager=${shQuote(fieldManager)}`,
|
|
`dry_run=${dryRun ? "true" : "false"}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"manifest=\"$tmp_dir/manifest.yaml\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest\"",
|
|
"args=\"--server-side --force-conflicts --field-manager=$field_manager\"",
|
|
"if [ \"$dry_run\" = true ]; then args=\"$args --dry-run=server\"; fi",
|
|
"set +e",
|
|
"kubectl apply $args -f \"$manifest\" > \"$tmp_dir/apply.out\" 2> \"$tmp_dir/apply.err\"",
|
|
"apply_exit=$?",
|
|
"set -e",
|
|
"APPLY_EXIT=\"$apply_exit\" APPLY_OUT=\"$tmp_dir/apply.out\" APPLY_ERR=\"$tmp_dir/apply.err\" MANIFEST=\"$manifest\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const crypto = require('node:crypto');",
|
|
"const out = fs.readFileSync(process.env.APPLY_OUT, 'utf8');",
|
|
"const err = fs.readFileSync(process.env.APPLY_ERR, 'utf8');",
|
|
"const manifest = fs.readFileSync(process.env.MANIFEST, 'utf8');",
|
|
"const resources = out.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean).slice(0, 80);",
|
|
"console.log(JSON.stringify({ ok: process.env.APPLY_EXIT === '0', exitCode: Number(process.env.APPLY_EXIT), resourceCount: resources.length, resources, manifestBytes: Buffer.byteLength(manifest, 'utf8'), manifestDigest: 'sha256:' + crypto.createHash('sha256').update(manifest).digest('hex'), stderrTail: err.slice(-3000), valuesPrinted: false }));",
|
|
"NODE",
|
|
"exit \"$apply_exit\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function manifestObjectRef(object: Record<string, unknown>): Record<string, unknown> {
|
|
const metadata = record(object.metadata);
|
|
return {
|
|
apiVersion: stringOrNull(object.apiVersion),
|
|
kind: stringOrNull(object.kind),
|
|
namespace: stringOrNull(metadata.namespace),
|
|
name: stringOrNull(metadata.name),
|
|
};
|
|
}
|
|
|
|
function cleanupRunnersFactsNodeScript(): string {
|
|
return String.raw`
|
|
function iso(value) {
|
|
if (value instanceof Date) return value.toISOString();
|
|
if (typeof value === "string" && value.length > 0) {
|
|
const ms = Date.parse(value);
|
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const { Pool } = await import("pg");
|
|
const namespace = process.env.RETENTION_NAMESPACE || process.env.AGENTRUN_RUNTIME_NAMESPACE || "";
|
|
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
const sql = [
|
|
"SELECT",
|
|
" j.id, j.run_id, j.command_id, j.attempt_id, j.runner_id, j.namespace, j.job_name, j.created_at, j.updated_at,",
|
|
" r.registered_at AS runner_registered_at, r.heartbeat_at AS runner_heartbeat_at,",
|
|
" rr.status AS run_status, rr.terminal_status AS run_terminal_status, rr.failure_kind AS run_failure_kind, rr.updated_at AS run_updated_at",
|
|
"FROM agentrun_runner_jobs j",
|
|
"LEFT JOIN agentrun_runners r ON r.id = j.runner_id",
|
|
"LEFT JOIN agentrun_runs rr ON rr.id = j.run_id",
|
|
"WHERE j.namespace = $1",
|
|
"ORDER BY COALESCE(r.heartbeat_at, j.updated_at, j.created_at) ASC",
|
|
].join("\n");
|
|
const result = await pool.query(sql, [namespace]);
|
|
await pool.end();
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
itemCount: result.rows.length,
|
|
items: result.rows.map((row) => ({
|
|
id: row.id,
|
|
runId: row.run_id,
|
|
commandId: row.command_id,
|
|
attemptId: row.attempt_id,
|
|
runnerId: row.runner_id,
|
|
namespace: row.namespace,
|
|
jobName: row.job_name,
|
|
runnerJobCreatedAt: iso(row.created_at),
|
|
runnerJobUpdatedAt: iso(row.updated_at),
|
|
runnerRegisteredAt: iso(row.runner_registered_at),
|
|
runnerHeartbeatAt: iso(row.runner_heartbeat_at),
|
|
runStatus: row.run_status || null,
|
|
runTerminalStatus: row.run_terminal_status || null,
|
|
runFailureKind: row.run_failure_kind || null,
|
|
runUpdatedAt: iso(row.run_updated_at),
|
|
valuesPrinted: false,
|
|
})),
|
|
valuesPrinted: false,
|
|
}));
|
|
} catch (error) {
|
|
console.log(JSON.stringify({
|
|
ok: false,
|
|
itemCount: 0,
|
|
items: [],
|
|
failureKind: "manager-runner-facts-query-failed",
|
|
message: error instanceof Error ? error.message : String(error),
|
|
valuesPrinted: false,
|
|
}));
|
|
}
|
|
`;
|
|
}
|
|
|
|
function cleanupRunnersPlanNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const namespace = process.env.NAMESPACE;
|
|
const maxRunners = Number(process.env.MAX_RUNNERS || 0);
|
|
const cleanupOrder = process.env.CLEANUP_ORDER || "";
|
|
const activeHeartbeatMaxAgeMs = Number(process.env.ACTIVE_HEARTBEAT_MAX_AGE_MS || 0);
|
|
const ageBasedCleanupEnabled = process.env.AGE_BASED_CLEANUP_ENABLED === "true";
|
|
const ageBasedMaxAgeHours = process.env.AGE_BASED_MAX_AGE_HOURS ? Number(process.env.AGE_BASED_MAX_AGE_HOURS) : null;
|
|
const forceActive = process.env.FORCE_ACTIVE === "true";
|
|
const matchLabels = JSON.parse(process.env.MATCH_LABELS_JSON || "{}");
|
|
const jobNamePrefixes = JSON.parse(process.env.JOB_NAME_PREFIXES_JSON || "[]");
|
|
const now = Date.now();
|
|
|
|
function readJson(name) {
|
|
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
|
}
|
|
|
|
function dateMs(value) {
|
|
const ms = Date.parse(value || "");
|
|
return Number.isFinite(ms) ? ms : null;
|
|
}
|
|
|
|
function ageMs(value) {
|
|
const ms = dateMs(value);
|
|
return ms === null ? null : Math.max(0, now - ms);
|
|
}
|
|
|
|
function labelsOf(item) {
|
|
return item?.metadata?.labels && typeof item.metadata.labels === "object" ? item.metadata.labels : {};
|
|
}
|
|
|
|
function annotationsOf(item) {
|
|
return item?.metadata?.annotations && typeof item.metadata.annotations === "object" ? item.metadata.annotations : {};
|
|
}
|
|
|
|
function matchesLabels(labels) {
|
|
return Object.entries(matchLabels).every(([key, value]) => labels?.[key] === value);
|
|
}
|
|
|
|
function matchesPrefix(name) {
|
|
return jobNamePrefixes.length === 0 || jobNamePrefixes.some((prefix) => name === prefix || name.startsWith(prefix + "-"));
|
|
}
|
|
|
|
function terminalRunStatus(value) {
|
|
return value === "completed" || value === "failed" || value === "blocked" || value === "cancelled";
|
|
}
|
|
|
|
function jobCondition(job, type) {
|
|
const conditions = Array.isArray(job?.status?.conditions) ? job.status.conditions : [];
|
|
return conditions.find((entry) => entry?.type === type && entry?.status === "True") || null;
|
|
}
|
|
|
|
function isTerminalJob(job) {
|
|
return jobCondition(job, "Complete") !== null || jobCondition(job, "Failed") !== null || Number(job?.status?.succeeded || 0) > 0 || Number(job?.status?.failed || 0) > 0;
|
|
}
|
|
|
|
function isTerminalPod(pod) {
|
|
const phase = pod?.status?.phase || "";
|
|
return phase === "Succeeded" || phase === "Failed";
|
|
}
|
|
|
|
function jobNameForPod(pod) {
|
|
const labels = labelsOf(pod);
|
|
if (typeof labels["job-name"] === "string") return labels["job-name"];
|
|
const owner = (Array.isArray(pod?.metadata?.ownerReferences) ? pod.metadata.ownerReferences : []).find((entry) => entry?.kind === "Job" && typeof entry?.name === "string");
|
|
return owner?.name || null;
|
|
}
|
|
|
|
function preferredLastActiveAt(fact, job) {
|
|
return fact?.runnerHeartbeatAt
|
|
|| fact?.runUpdatedAt
|
|
|| fact?.runnerJobUpdatedAt
|
|
|| job?.status?.completionTime
|
|
|| job?.status?.startTime
|
|
|| job?.metadata?.creationTimestamp
|
|
|| null;
|
|
}
|
|
|
|
function sortOldestLastActive(left, right) {
|
|
const leftMs = dateMs(left.lastActiveAt) ?? dateMs(left.createdAt) ?? Number.MAX_SAFE_INTEGER;
|
|
const rightMs = dateMs(right.lastActiveAt) ?? dateMs(right.createdAt) ?? Number.MAX_SAFE_INTEGER;
|
|
if (leftMs !== rightMs) return leftMs - rightMs;
|
|
return String(left.name).localeCompare(String(right.name));
|
|
}
|
|
|
|
function countNonTerminalPods(pods) {
|
|
return pods.filter((pod) => !isTerminalPod(pod)).length;
|
|
}
|
|
|
|
const jobs = readJson("jobs.json");
|
|
const pods = readJson("pods.json");
|
|
const facts = readJson("runner-facts.json");
|
|
const factItems = Array.isArray(facts.items) ? facts.items : [];
|
|
const factByJob = new Map(factItems.map((item) => [item.jobName, item]));
|
|
const allPods = Array.isArray(pods.items) ? pods.items : [];
|
|
const podsByJob = new Map();
|
|
for (const pod of allPods) {
|
|
const jobName = jobNameForPod(pod);
|
|
if (!jobName) continue;
|
|
const entry = podsByJob.get(jobName) || [];
|
|
entry.push(pod);
|
|
podsByJob.set(jobName, entry);
|
|
}
|
|
|
|
const matchedJobs = (Array.isArray(jobs.items) ? jobs.items : [])
|
|
.map((job) => {
|
|
const name = job?.metadata?.name || "";
|
|
return { job, name, labels: labelsOf(job), annotations: annotationsOf(job) };
|
|
})
|
|
.filter((item) => item.name && matchesLabels(item.labels) && matchesPrefix(item.name));
|
|
|
|
const runnerJobs = matchedJobs.map(({ job, name, labels, annotations }) => {
|
|
const fact = factByJob.get(name) || null;
|
|
const jobPods = podsByJob.get(name) || [];
|
|
const nonTerminalPods = jobPods.filter((pod) => !isTerminalPod(pod));
|
|
const terminatingPods = nonTerminalPods.filter((pod) => typeof pod?.metadata?.deletionTimestamp === "string");
|
|
const terminal = isTerminalJob(job) || terminalRunStatus(fact?.runStatus) || terminalRunStatus(fact?.runTerminalStatus);
|
|
const heartbeatAgeMs = ageMs(fact?.runnerHeartbeatAt);
|
|
const heartbeatFresh = heartbeatAgeMs !== null && heartbeatAgeMs <= activeHeartbeatMaxAgeMs;
|
|
const hasActivePod = Number(job?.status?.active || 0) > 0 || nonTerminalPods.length > 0;
|
|
const lastActiveAt = preferredLastActiveAt(fact, job);
|
|
let inactive = false;
|
|
let protectedActive = false;
|
|
let classification = "unknown";
|
|
if (terminal) {
|
|
inactive = true;
|
|
classification = "terminal";
|
|
} else if (heartbeatFresh) {
|
|
protectedActive = true;
|
|
classification = "active-fresh-heartbeat";
|
|
} else if (heartbeatAgeMs !== null) {
|
|
inactive = true;
|
|
classification = "inactive-stale-heartbeat";
|
|
} else if (terminatingPods.length > 0 && nonTerminalPods.length === terminatingPods.length) {
|
|
inactive = true;
|
|
classification = "inactive-terminating";
|
|
} else if (hasActivePod) {
|
|
protectedActive = true;
|
|
classification = facts.ok === true ? "active-no-heartbeat-row" : "active-unverified-manager-facts";
|
|
} else {
|
|
inactive = true;
|
|
classification = "inactive-no-active-pod";
|
|
}
|
|
return {
|
|
name,
|
|
namespace,
|
|
createdAt: job?.metadata?.creationTimestamp || null,
|
|
lastActiveAt,
|
|
lastActiveAgeMs: ageMs(lastActiveAt),
|
|
lastActiveSource: fact?.runnerHeartbeatAt ? "runner-heartbeat" : fact?.runUpdatedAt ? "run-updated" : fact?.runnerJobUpdatedAt ? "runner-job-updated" : job?.status?.completionTime ? "job-completion" : job?.status?.startTime ? "job-start" : "job-created",
|
|
runId: annotations["agentrun.pikastech.local/run-id"] || fact?.runId || null,
|
|
commandId: annotations["agentrun.pikastech.local/command-id"] || fact?.commandId || null,
|
|
runnerJobId: fact?.id || null,
|
|
runnerId: fact?.runnerId || null,
|
|
runStatus: fact?.runStatus || null,
|
|
runTerminalStatus: fact?.runTerminalStatus || null,
|
|
jobStatus: {
|
|
active: Number(job?.status?.active || 0),
|
|
succeeded: Number(job?.status?.succeeded || 0),
|
|
failed: Number(job?.status?.failed || 0),
|
|
ready: Number(job?.status?.ready || 0),
|
|
terminating: Number(job?.status?.terminating || 0),
|
|
},
|
|
podCount: jobPods.length,
|
|
nonTerminalPodCount: nonTerminalPods.length,
|
|
terminatingPodCount: terminatingPods.length,
|
|
inactive,
|
|
protectedActive,
|
|
classification,
|
|
labels,
|
|
valuesPrinted: false,
|
|
};
|
|
});
|
|
|
|
const runnerJobCount = runnerJobs.length;
|
|
const inactiveCandidates = runnerJobs.filter((item) => item.inactive).sort(sortOldestLastActive);
|
|
const overLimitCount = Math.max(0, runnerJobCount - maxRunners);
|
|
const selectedByName = new Map();
|
|
const selectionReasons = new Map();
|
|
for (const item of inactiveCandidates.slice(0, overLimitCount)) {
|
|
selectedByName.set(item.name, item);
|
|
selectionReasons.set(item.name, "over-max-runners");
|
|
}
|
|
if (ageBasedCleanupEnabled && ageBasedMaxAgeHours !== null) {
|
|
const maxAgeMs = ageBasedMaxAgeHours * 3600 * 1000;
|
|
for (const item of inactiveCandidates) {
|
|
const itemAgeMs = item.lastActiveAgeMs;
|
|
if (itemAgeMs !== null && itemAgeMs >= maxAgeMs) {
|
|
selectedByName.set(item.name, item);
|
|
if (!selectionReasons.has(item.name)) selectionReasons.set(item.name, "age-based-cleanup");
|
|
}
|
|
}
|
|
}
|
|
if (forceActive) {
|
|
for (const item of runnerJobs) {
|
|
selectedByName.set(item.name, item);
|
|
selectionReasons.set(item.name, item.protectedActive ? "force-active-runner-cleanup" : "force-runner-cleanup");
|
|
}
|
|
}
|
|
|
|
const selectionPool = forceActive ? runnerJobs : inactiveCandidates;
|
|
const selected = selectionPool
|
|
.filter((item) => selectedByName.has(item.name))
|
|
.map((item) => ({ ...item, selectionReason: selectionReasons.get(item.name) || "selected" }));
|
|
const matchedPodNames = new Set(runnerJobs.map((item) => item.name));
|
|
const matchedPods = allPods.filter((pod) => {
|
|
const jobName = jobNameForPod(pod);
|
|
return jobName !== null && matchedPodNames.has(jobName);
|
|
});
|
|
const remainingAfterSelection = runnerJobCount - selected.length;
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
planKind: "agentrun-runtime-runner-retention",
|
|
generatedAt: new Date(now).toISOString(),
|
|
namespace,
|
|
criteria: {
|
|
maxRunners,
|
|
cleanupOrder,
|
|
activeHeartbeatMaxAgeMs,
|
|
forceActive,
|
|
selectors: { matchLabels, jobNamePrefixes },
|
|
ageBasedCleanup: { enabled: ageBasedCleanupEnabled, maxAgeHours: ageBasedMaxAgeHours },
|
|
},
|
|
managerFacts: {
|
|
ok: facts.ok === true,
|
|
factCount: factItems.length,
|
|
factsExit: Number(process.env.FACTS_EXIT || 0),
|
|
failureKind: facts.failureKind || null,
|
|
message: facts.ok === true ? null : facts.message || null,
|
|
valuesPrinted: false,
|
|
},
|
|
runnerJobCount,
|
|
nonTerminalRunnerPodCount: countNonTerminalPods(matchedPods),
|
|
overLimitCount,
|
|
inactiveCandidateCount: inactiveCandidates.length,
|
|
protectedActiveRunnerCount: runnerJobs.filter((item) => item.protectedActive).length,
|
|
activeRunRisk: remainingAfterSelection > maxRunners,
|
|
remainingRunnerJobCountAfterSelection: remainingAfterSelection,
|
|
runnerJobs: runnerJobs.sort(sortOldestLastActive),
|
|
candidates: inactiveCandidates,
|
|
selected,
|
|
selectedRunnerJobs: selected.map((item) => item.name),
|
|
selectedRunnerJobCount: selected.length,
|
|
valuesPrinted: false,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupRunnersFinalizeNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
|
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
|
const jobsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "jobs-after.json"), "utf8"));
|
|
const podsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pods-after.json"), "utf8"));
|
|
const matchLabels = plan.criteria?.selectors?.matchLabels || {};
|
|
const jobNamePrefixes = plan.criteria?.selectors?.jobNamePrefixes || [];
|
|
|
|
function labelsOf(item) {
|
|
return item?.metadata?.labels && typeof item.metadata.labels === "object" ? item.metadata.labels : {};
|
|
}
|
|
|
|
function matchesLabels(labels) {
|
|
return Object.entries(matchLabels).every(([key, value]) => labels?.[key] === value);
|
|
}
|
|
|
|
function matchesPrefix(name) {
|
|
return jobNamePrefixes.length === 0 || jobNamePrefixes.some((prefix) => name === prefix || name.startsWith(prefix + "-"));
|
|
}
|
|
|
|
function isTerminalPod(pod) {
|
|
const phase = pod?.status?.phase || "";
|
|
return phase === "Succeeded" || phase === "Failed";
|
|
}
|
|
|
|
function jobNameForPod(pod) {
|
|
const labels = labelsOf(pod);
|
|
if (typeof labels["job-name"] === "string") return labels["job-name"];
|
|
const owner = (Array.isArray(pod?.metadata?.ownerReferences) ? pod.metadata.ownerReferences : []).find((entry) => entry?.kind === "Job" && typeof entry?.name === "string");
|
|
return owner?.name || null;
|
|
}
|
|
|
|
function tail(name) {
|
|
try {
|
|
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
|
return text.length > 3000 ? text.slice(-3000) : text;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
const selected = new Set(Array.isArray(plan.selectedRunnerJobs) ? plan.selectedRunnerJobs : []);
|
|
const remainingJobs = (Array.isArray(jobsAfter.items) ? jobsAfter.items : [])
|
|
.map((job) => ({ name: job?.metadata?.name || "", labels: labelsOf(job) }))
|
|
.filter((item) => item.name && matchesLabels(item.labels) && matchesPrefix(item.name));
|
|
const remainingJobNames = new Set(remainingJobs.map((item) => item.name));
|
|
const matchedPods = (Array.isArray(podsAfter.items) ? podsAfter.items : []).filter((pod) => {
|
|
const jobName = jobNameForPod(pod);
|
|
return jobName !== null && remainingJobNames.has(jobName);
|
|
});
|
|
const remainingSelectedRunnerJobs = Array.from(selected).filter((name) => remainingJobNames.has(name));
|
|
const deletedRunnerJobs = Array.from(selected).filter((name) => !remainingJobNames.has(name));
|
|
console.log(JSON.stringify({
|
|
...plan,
|
|
ok: deleteExit === 0,
|
|
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
|
deletedRunnerJobs,
|
|
deletedRunnerJobCount: deletedRunnerJobs.length,
|
|
remainingSelectedRunnerJobs,
|
|
remainingSelectedRunnerJobCount: remainingSelectedRunnerJobs.length,
|
|
after: {
|
|
runnerJobCount: remainingJobs.length,
|
|
nonTerminalRunnerPodCount: matchedPods.filter((pod) => !isTerminalPod(pod)).length,
|
|
overLimitCount: Math.max(0, remainingJobs.length - Number(plan.criteria?.maxRunners || 0)),
|
|
},
|
|
valuesPrinted: false,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupRunsPlanNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const cp = require("node:child_process");
|
|
const tmp = process.env.TMP_DIR;
|
|
const namespace = process.env.NAMESPACE;
|
|
const pipelineRunPrefix = process.env.PIPELINE_RUN_PREFIX || "";
|
|
const minAgeMinutes = Number(process.env.MIN_AGE_MINUTES || 60);
|
|
const limit = Number(process.env.LIMIT || 20);
|
|
const now = Date.now();
|
|
|
|
function readJson(name) {
|
|
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
|
}
|
|
|
|
function conditionOf(item) {
|
|
const conditions = Array.isArray(item?.status?.conditions) ? item.status.conditions : [];
|
|
return conditions.find((entry) => entry.type === "Succeeded") || conditions[0] || {};
|
|
}
|
|
|
|
function ageMinutes(createdAt) {
|
|
const createdMs = Date.parse(createdAt || "");
|
|
return Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
|
}
|
|
|
|
function localPathOf(pv) {
|
|
return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
|
|
}
|
|
|
|
function duBytes(hostPath) {
|
|
if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
|
try {
|
|
const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
const bytes = Number(String(out).trim().split(/\s+/)[0]);
|
|
return Number.isFinite(bytes) ? bytes : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!Number.isFinite(bytes)) return null;
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let value = bytes;
|
|
let unit = 0;
|
|
while (value >= 1024 && unit < units.length - 1) {
|
|
value /= 1024;
|
|
unit += 1;
|
|
}
|
|
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
|
}
|
|
|
|
const pipelineRuns = readJson("pipelineruns.json");
|
|
const pvcs = readJson("pvcs.json");
|
|
const pvs = readJson("pvs.json");
|
|
const pods = readJson("pods.json");
|
|
const pvByName = new Map((Array.isArray(pvs.items) ? pvs.items : []).map((item) => [item?.metadata?.name, item]));
|
|
const activeClaimPods = new Map();
|
|
for (const pod of Array.isArray(pods.items) ? pods.items : []) {
|
|
const phase = pod?.status?.phase || "";
|
|
if (phase === "Succeeded" || phase === "Failed") continue;
|
|
for (const volume of Array.isArray(pod?.spec?.volumes) ? pod.spec.volumes : []) {
|
|
const claimName = volume?.persistentVolumeClaim?.claimName;
|
|
if (!claimName) continue;
|
|
const entry = activeClaimPods.get(claimName) || [];
|
|
entry.push(pod?.metadata?.name || null);
|
|
activeClaimPods.set(claimName, entry.filter(Boolean));
|
|
}
|
|
}
|
|
|
|
const pvcsByOwner = new Map();
|
|
for (const pvc of Array.isArray(pvcs.items) ? pvcs.items : []) {
|
|
const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
|
|
if (!owner?.name) continue;
|
|
const entry = pvcsByOwner.get(owner.name) || [];
|
|
entry.push(pvc);
|
|
pvcsByOwner.set(owner.name, entry);
|
|
}
|
|
|
|
const allPipelineRuns = (Array.isArray(pipelineRuns.items) ? pipelineRuns.items : [])
|
|
.map((item) => {
|
|
const condition = conditionOf(item);
|
|
return {
|
|
name: item?.metadata?.name || "",
|
|
createdAt: item?.metadata?.creationTimestamp || null,
|
|
ageMinutes: ageMinutes(item?.metadata?.creationTimestamp),
|
|
status: condition.status || null,
|
|
reason: condition.reason || null,
|
|
};
|
|
})
|
|
.filter((item) => pipelineRunPrefix.length === 0 || item.name.startsWith(pipelineRunPrefix + "-"));
|
|
|
|
const protectedActivePipelineRuns = allPipelineRuns
|
|
.filter((item) => item.status !== "True" && item.status !== "False")
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
|
|
const protectedLatestPipelineRun = allPipelineRuns
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0]?.name || null;
|
|
|
|
const candidates = allPipelineRuns
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= minAgeMinutes)
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
|
.slice(0, limit)
|
|
.map((item) => {
|
|
const owned = pvcsByOwner.get(item.name) || [];
|
|
const activeMountPods = owned.flatMap((pvc) => activeClaimPods.get(pvc?.metadata?.name) || []);
|
|
const protectedLatest = item.name === protectedLatestPipelineRun;
|
|
return {
|
|
...item,
|
|
selected: activeMountPods.length === 0 && !protectedLatest,
|
|
selectedReason: protectedLatest ? "protected-latest-pipelinerun" : activeMountPods.length === 0 ? "terminal-and-unmounted" : "owned-pvc-active-mounted",
|
|
ownedPvcCount: owned.length,
|
|
activeMountPods,
|
|
};
|
|
});
|
|
|
|
const selectedPipelineRuns = candidates.filter((item) => item.selected).map((item) => item.name);
|
|
const selectedSet = new Set(selectedPipelineRuns);
|
|
const ownedPvcs = [];
|
|
const protectedOwnedPvcs = [];
|
|
for (const [owner, items] of pvcsByOwner.entries()) {
|
|
if (!selectedSet.has(owner) && !candidates.some((item) => item.name === owner)) continue;
|
|
for (const pvc of items) {
|
|
const volume = pvc?.spec?.volumeName || null;
|
|
const pv = volume ? pvByName.get(volume) : null;
|
|
const hostPath = localPathOf(pv);
|
|
const activeMountPods = activeClaimPods.get(pvc?.metadata?.name) || [];
|
|
const entry = {
|
|
name: pvc?.metadata?.name || null,
|
|
volume,
|
|
phase: pvc?.status?.phase || null,
|
|
ownerKind: "PipelineRun",
|
|
owner,
|
|
storageClass: pv?.spec?.storageClassName || null,
|
|
reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
|
|
hostPath,
|
|
estimatedBytes: duBytes(hostPath),
|
|
activeMountPods,
|
|
};
|
|
if (selectedSet.has(owner)) ownedPvcs.push(entry);
|
|
else protectedOwnedPvcs.push(entry);
|
|
}
|
|
}
|
|
|
|
const estimatedReclaimBytes = ownedPvcs.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
planKind: "agentrun-ci-completed-pipelinerun-workspace-retention",
|
|
generatedAt: new Date().toISOString(),
|
|
namespace,
|
|
criteria: { prefix: pipelineRunPrefix + "-", terminalStatuses: ["True", "False"], minAgeMinutes, limit },
|
|
candidates,
|
|
candidateCount: candidates.length,
|
|
protectedActivePipelineRuns,
|
|
protectedActivePipelineRunCount: protectedActivePipelineRuns.length,
|
|
protectedLatestPipelineRun,
|
|
selectedPipelineRuns,
|
|
selectedPipelineRunCount: selectedPipelineRuns.length,
|
|
ownedPvcs,
|
|
ownedPvcCount: ownedPvcs.length,
|
|
protectedOwnedPvcs,
|
|
protectedOwnedPvcCount: protectedOwnedPvcs.length,
|
|
estimatedReclaimBytes,
|
|
estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupRunsFinalizeNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
|
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
|
const pvcsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvcs-after.json"), "utf8"));
|
|
const selected = new Set(Array.isArray(plan.selectedPipelineRuns) ? plan.selectedPipelineRuns : []);
|
|
const remainingOwnedPvcs = (Array.isArray(pvcsAfter.items) ? pvcsAfter.items : [])
|
|
.map((pvc) => {
|
|
const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
|
|
return {
|
|
name: pvc?.metadata?.name || null,
|
|
volume: pvc?.spec?.volumeName || null,
|
|
phase: pvc?.status?.phase || null,
|
|
ownerKind: owner?.kind || null,
|
|
owner: owner?.name || null,
|
|
};
|
|
})
|
|
.filter((item) => item.owner && selected.has(item.owner));
|
|
|
|
function tail(name) {
|
|
try {
|
|
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
|
return text.length > 3000 ? text.slice(-3000) : text;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
console.log(JSON.stringify({
|
|
...plan,
|
|
ok: deleteExit === 0,
|
|
deletedPipelineRuns: Array.from(selected),
|
|
deletedPipelineRunCount: selected.size,
|
|
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
|
remainingOwnedPvcs,
|
|
remainingOwnedPvcCount: remainingOwnedPvcs.length,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupReleasedPvsPlanNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const cp = require("node:child_process");
|
|
const tmp = process.env.TMP_DIR;
|
|
const namespace = process.env.NAMESPACE;
|
|
const limit = Number(process.env.LIMIT || 20);
|
|
|
|
function readJson(name) {
|
|
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
|
}
|
|
|
|
function localPathOf(pv) {
|
|
return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
|
|
}
|
|
|
|
function duBytes(hostPath) {
|
|
if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
|
try {
|
|
const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
const bytes = Number(String(out).trim().split(/\s+/)[0]);
|
|
return Number.isFinite(bytes) ? bytes : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!Number.isFinite(bytes)) return null;
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let value = bytes;
|
|
let unit = 0;
|
|
while (value >= 1024 && unit < units.length - 1) {
|
|
value /= 1024;
|
|
unit += 1;
|
|
}
|
|
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
|
}
|
|
|
|
const pvs = readJson("pvs.json");
|
|
const candidates = (Array.isArray(pvs.items) ? pvs.items : [])
|
|
.map((pv) => {
|
|
const hostPath = localPathOf(pv);
|
|
return {
|
|
name: pv?.metadata?.name || "",
|
|
createdAt: pv?.metadata?.creationTimestamp || null,
|
|
phase: pv?.status?.phase || null,
|
|
storageClass: pv?.spec?.storageClassName || null,
|
|
reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
|
|
claimNamespace: pv?.spec?.claimRef?.namespace || null,
|
|
claimName: pv?.spec?.claimRef?.name || null,
|
|
capacity: pv?.spec?.capacity?.storage || null,
|
|
hostPath,
|
|
estimatedBytes: duBytes(hostPath),
|
|
};
|
|
})
|
|
.filter((item) => item.phase === "Released")
|
|
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
|
.filter((item) => item.claimNamespace === namespace)
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
|
.slice(0, limit);
|
|
|
|
const selectedPersistentVolumes = candidates.map((item) => item.name);
|
|
const estimatedReclaimBytes = candidates.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
planKind: "agentrun-ci-released-local-path-pv-retention",
|
|
generatedAt: new Date().toISOString(),
|
|
namespace,
|
|
criteria: { phase: "Released", storageClass: "local-path", reclaimPolicy: "Delete", claimNamespace: namespace, limit },
|
|
candidates,
|
|
candidateCount: candidates.length,
|
|
selectedPersistentVolumes,
|
|
selectedPersistentVolumeCount: selectedPersistentVolumes.length,
|
|
estimatedReclaimBytes,
|
|
estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupReleasedPvsFinalizeNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
|
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
|
const pvsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvs-after.json"), "utf8"));
|
|
const selected = new Set(Array.isArray(plan.selectedPersistentVolumes) ? plan.selectedPersistentVolumes : []);
|
|
const remainingPersistentVolumes = (Array.isArray(pvsAfter.items) ? pvsAfter.items : [])
|
|
.filter((pv) => selected.has(pv?.metadata?.name))
|
|
.map((pv) => ({ name: pv?.metadata?.name || null, phase: pv?.status?.phase || null, claimNamespace: pv?.spec?.claimRef?.namespace || null, claimName: pv?.spec?.claimRef?.name || null }));
|
|
|
|
function tail(name) {
|
|
try {
|
|
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
|
return text.length > 3000 ? text.slice(-3000) : text;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
console.log(JSON.stringify({
|
|
...plan,
|
|
ok: deleteExit === 0,
|
|
deletedPersistentVolumes: Array.from(selected),
|
|
deletedPersistentVolumeCount: selected.size,
|
|
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
|
remainingPersistentVolumes,
|
|
remainingPersistentVolumeCount: remainingPersistentVolumes.length,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function refreshYamlLaneScript(spec: AgentRunLaneSpec): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(spec.gitops.argoNamespace)}`,
|
|
`application=${shQuote(spec.gitops.argoApplication)}`,
|
|
"kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/agentrun-refresh-output.txt",
|
|
"kubectl -n \"$namespace\" get application \"$application\" -o json >/tmp/agentrun-refresh-app.json",
|
|
"NAMESPACE=\"$namespace\" APPLICATION=\"$application\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"let app = {};",
|
|
"try { app = JSON.parse(fs.readFileSync('/tmp/agentrun-refresh-app.json', 'utf8')); } catch {}",
|
|
"console.log(JSON.stringify({",
|
|
" ok: true,",
|
|
" namespace: process.env.NAMESPACE,",
|
|
" application: process.env.APPLICATION,",
|
|
" revision: app.status?.sync?.revision ?? null,",
|
|
" syncStatus: app.status?.sync?.status ?? null,",
|
|
" healthStatus: app.status?.health?.status ?? null,",
|
|
" observedAt: new Date().toISOString(),",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
async function gitMirrorStatus(config: UniDeskConfig, options: GitMirrorStatusOptions = { full: false, raw: false, node: null, lane: null }): Promise<Record<string, unknown>> {
|
|
const target = resolveAgentRunLaneTarget(options);
|
|
const spec = target.spec;
|
|
const observation = await readGitMirrorStatus(config, target);
|
|
const summary = observation.summary;
|
|
return {
|
|
ok: observation.ok,
|
|
command: "agentrun git-mirror status",
|
|
mode: "yaml-declared-node-lane",
|
|
configPath: target.configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
namespace: spec.gitMirror.namespace,
|
|
readUrl: spec.gitMirror.readUrl,
|
|
writeUrl: spec.gitMirror.writeUrl,
|
|
summary,
|
|
...(options.raw ? { raw: observation.raw } : {}),
|
|
probe: compactCapture(observation.result, { full: options.full || options.raw, stdoutTailChars: 6000, stderrTailChars: 3000 }),
|
|
disclosure: {
|
|
defaultView: "compact-low-noise",
|
|
full: options.full,
|
|
raw: options.raw,
|
|
rawOmitted: !options.raw,
|
|
probeTailOmitted: !(options.full || options.raw),
|
|
expandWith: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane} --full`,
|
|
rawWith: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane} --raw`,
|
|
},
|
|
next: {
|
|
sync: `bun scripts/cli.ts agentrun git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
flush: summary.pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function readGitMirrorStatus(config: UniDeskConfig, target: { configPath: string; spec: AgentRunLaneSpec }): Promise<Record<string, unknown> & { result: SshCaptureResult; raw: string; summary: Record<string, unknown> }> {
|
|
const spec = target.spec;
|
|
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)]);
|
|
const raw = result.stdout;
|
|
const summary = captureJsonPayload(result);
|
|
return {
|
|
ok: result.exitCode === 0 && summary.ok !== false,
|
|
configPath: target.configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
namespace: spec.gitMirror.namespace,
|
|
readUrl: spec.gitMirror.readUrl,
|
|
writeUrl: spec.gitMirror.writeUrl,
|
|
raw,
|
|
summary,
|
|
result,
|
|
};
|
|
}
|
|
|
|
async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", options: GitMirrorOptions): Promise<Record<string, unknown>> {
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
const previewJobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63);
|
|
const previewManifest = yamlLaneGitMirrorJobManifest(spec, action, previewJobName);
|
|
const command = `agentrun git-mirror ${action}`;
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command,
|
|
mode: "yaml-declared-node-lane",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
dryRun: true,
|
|
namespace: spec.gitMirror.namespace,
|
|
jobName: previewJobName,
|
|
manifest: previewManifest,
|
|
next: { confirm: `bun scripts/cli.ts ${command} --node ${spec.nodeId} --lane ${spec.lane} --confirm` },
|
|
};
|
|
}
|
|
if (!options.wait) {
|
|
const jobName = previewJobName;
|
|
const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
|
|
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
|
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
|
return {
|
|
ok: created.exitCode === 0,
|
|
command,
|
|
mode: created.exitCode === 0 ? "submitted" : "create-failed",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
dryRun: false,
|
|
namespace: spec.gitMirror.namespace,
|
|
jobName,
|
|
result: compactCapture(created),
|
|
status,
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
|
},
|
|
};
|
|
}
|
|
const attempts: Record<string, unknown>[] = [];
|
|
let lastCreated: SshCaptureResult | null = null;
|
|
let lastWait: Record<string, unknown> | null = null;
|
|
for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
|
const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
|
|
const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
|
|
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
|
lastCreated = created;
|
|
if (created.exitCode !== 0) {
|
|
const result = { phase: "create-job", capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }) };
|
|
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
|
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
|
progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure);
|
|
await sleep(agentRunGitMirrorRetryDelayMs(attempt));
|
|
continue;
|
|
}
|
|
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
|
return {
|
|
ok: false,
|
|
command,
|
|
mode: "create-failed",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
dryRun: false,
|
|
namespace: spec.gitMirror.namespace,
|
|
jobName,
|
|
result: compactCapture(created),
|
|
retryableFailure,
|
|
retry: agentRunGitMirrorRetrySummary(attempts),
|
|
status,
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
|
},
|
|
};
|
|
}
|
|
const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds);
|
|
lastWait = wait;
|
|
if (wait.ok === true) {
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, wait });
|
|
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
|
return {
|
|
ok: true,
|
|
command,
|
|
mode: "waited",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
dryRun: false,
|
|
namespace: spec.gitMirror.namespace,
|
|
jobName,
|
|
result: compactCapture(created),
|
|
wait,
|
|
retry: agentRunGitMirrorRetrySummary(attempts),
|
|
status,
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null,
|
|
},
|
|
};
|
|
}
|
|
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, wait);
|
|
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, wait, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
|
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
|
progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure);
|
|
await sleep(agentRunGitMirrorRetryDelayMs(attempt));
|
|
continue;
|
|
}
|
|
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
|
return {
|
|
ok: false,
|
|
command,
|
|
mode: "waited",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
dryRun: false,
|
|
namespace: spec.gitMirror.namespace,
|
|
jobName,
|
|
result: compactCapture(created),
|
|
wait,
|
|
retryableFailure,
|
|
retry: agentRunGitMirrorRetrySummary(attempts),
|
|
status,
|
|
next: {
|
|
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
|
},
|
|
};
|
|
}
|
|
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
|
return {
|
|
ok: false,
|
|
command,
|
|
mode: "retry-exhausted",
|
|
configPath,
|
|
target: agentRunLaneSummary(spec),
|
|
dryRun: false,
|
|
namespace: spec.gitMirror.namespace,
|
|
result: lastCreated === null ? null : compactCapture(lastCreated),
|
|
wait: lastWait,
|
|
retry: agentRunGitMirrorRetrySummary(attempts),
|
|
status,
|
|
next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` },
|
|
};
|
|
}
|
|
|
|
async function waitForGitMirrorJob(config: UniDeskConfig, spec: AgentRunLaneSpec, action: "sync" | "flush", jobName: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
|
const startedAtMs = Date.now();
|
|
let lastProbe: SshCaptureResult | null = null;
|
|
let polls = 0;
|
|
while (Date.now() - startedAtMs <= timeoutSeconds * 1000) {
|
|
polls += 1;
|
|
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
|
|
const summary = captureJsonPayload(lastProbe);
|
|
process.stderr.write(`${JSON.stringify({
|
|
event: `agentrun.git-mirror.${action}.progress`,
|
|
at: new Date().toISOString(),
|
|
stage: "k8s-job",
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
status: summary.succeeded === true ? "succeeded" : summary.failed === true ? "failed" : "running",
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
})}\n`);
|
|
if (summary.succeeded === true) {
|
|
return {
|
|
ok: true,
|
|
action,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
summary,
|
|
probe: compactCapture(lastProbe),
|
|
};
|
|
}
|
|
if (summary.failed === true) {
|
|
return {
|
|
ok: false,
|
|
action,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
degradedReason: "git-mirror-job-failed",
|
|
summary,
|
|
probe: compactCapture(lastProbe),
|
|
};
|
|
}
|
|
await sleep(5_000);
|
|
}
|
|
return {
|
|
ok: false,
|
|
action,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
degradedReason: "git-mirror-job-timeout",
|
|
probe: lastProbe ? compactCapture(lastProbe) : null,
|
|
};
|
|
}
|
|
|
|
function startAsyncAgentRunJob(name: string, command: string[], note: string): Record<string, unknown> {
|
|
const job = startJob(name, command, note);
|
|
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
|
return {
|
|
ok: true,
|
|
mode: "async-job",
|
|
job,
|
|
statusCommand,
|
|
tailCommand: `tail -f ${job.stdoutFile}`,
|
|
next: {
|
|
status: statusCommand,
|
|
tail: `tail -f ${job.stdoutFile}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runAgentRunRestCompatCommand(config: UniDeskConfig | null, group: AgentRunRestCompatGroup, args: string[], canonicalArgs: string[]): Promise<Record<string, unknown>> {
|
|
try {
|
|
return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, parseAgentRunRestTargetOptions(args)), async () => {
|
|
return await runAgentRunRestCommand(config, group, stripAgentRunLaneTargetArgs(args));
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof AgentRunRestError) return error.toPayload(`agentrun ${canonicalArgs.join(" ")}`.trim());
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function runAgentRunRestCommand(config: UniDeskConfig | null, group: AgentRunRestCompatGroup, args: string[]): Promise<Record<string, unknown>> {
|
|
void config;
|
|
const compatGroup = group === "aipods" ? "aipod-specs" : group;
|
|
const [action, id] = args;
|
|
if (compatGroup === "queue") return await runAgentRunQueueRest(action, id, args);
|
|
if (compatGroup === "sessions") return await runAgentRunSessionsRest(action, id, args);
|
|
if (compatGroup === "runs") return await runAgentRunRunsRest(action, id, args);
|
|
if (compatGroup === "commands") return await runAgentRunCommandsRest(action, id, args);
|
|
if (compatGroup === "runner") return await runAgentRunRunnerRest(action, id, args);
|
|
if (compatGroup === "aipod-specs") return await runAgentRunAipodSpecsRest(action, id, args);
|
|
throw new AgentRunRestError("unsupported-version", `unsupported AgentRun REST compatibility group: ${group}`);
|
|
}
|
|
|
|
function parseAgentRunRestTargetOptions(args: string[]): AgentRunRestTargetOptions {
|
|
return {
|
|
node: agentRunOption(args, "node"),
|
|
lane: agentRunOption(args, "lane"),
|
|
};
|
|
}
|
|
|
|
function stripAgentRunLaneTargetArgs(args: string[]): string[] {
|
|
const result: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (arg === "--node" || arg === "--lane") {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue;
|
|
result.push(arg);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
async function runAgentRunQueueRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
|
|
if (action === "list") return await agentRunRestRequest("agentrun queue list", "GET", `/api/v1/queue/tasks${agentRunQuery(args, ["queue", "state", "cursor", "limit", "updated-after"])}`);
|
|
if (action === "commander") return await agentRunRestRequest("agentrun queue commander", "GET", `/api/v1/queue/commander${agentRunQuery(args, ["queue", "reader-id"])}`);
|
|
if (action === "stats") return await agentRunRestRequest("agentrun queue stats", "GET", `/api/v1/queue/stats${agentRunQuery(args, ["queue"])}`);
|
|
if (action === "show" && id) return await agentRunRestRequest("agentrun queue show", "GET", `/api/v1/queue/tasks/${encodeURIComponent(id)}`);
|
|
if (action === "submit") return await submitQueueTaskRest(args);
|
|
if (action === "dispatch" && id) return await mutateQueueTaskRest("queue-dispatch", id, "dispatch", queueDispatchBodyFromArgs(args), args);
|
|
if (action === "read" && id) return await mutateQueueTaskRest("queue-read", id, "read", { readerId: agentRunOption(args, "reader-id") ?? "cli" }, args);
|
|
if (action === "cancel" && id) return await mutateQueueTaskRest("queue-cancel", id, "cancel", cancelBodyFromArgs(args), args);
|
|
if (action === "refresh" && id) return await mutateQueueTaskRest("queue-refresh", id, "refresh", {}, args);
|
|
throw new AgentRunRestError("validation-failed", `unsupported queue command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`);
|
|
}
|
|
|
|
async function runAgentRunSessionsRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
|
|
if (action === "ps") return await agentRunRestRequest("agentrun sessions ps", "GET", `/api/v1/sessions${agentRunQuery(args, ["state", "profile", "backend-profile", "reader-id", "cursor", "limit"])}`);
|
|
if (action === "show" && id) return await agentRunRestRequest("agentrun sessions show", "GET", `/api/v1/sessions/${encodeURIComponent(id)}${agentRunQuery(args, ["reader-id"])}`);
|
|
if ((action === "trace" || action === "output") && id) return await agentRunRestRequest(`agentrun sessions ${action}`, "GET", `/api/v1/sessions/${encodeURIComponent(id)}/${action}${agentRunQuery(args, ["after-seq", "limit", "run-id"])}`);
|
|
if (action === "read" && id) {
|
|
const body = { readerId: agentRunOption(args, "reader-id") ?? "cli" };
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("session-read", `/api/v1/sessions/${encodeURIComponent(id)}/read`, body, `bun scripts/cli.ts agentrun sessions read ${id} --reader-id ${body.readerId}`);
|
|
return await agentRunRestRequest("agentrun sessions read", "POST", `/api/v1/sessions/${encodeURIComponent(id)}/read`, body);
|
|
}
|
|
if (action === "cancel" && id) {
|
|
const body = { action: "cancel", ...cancelBodyFromArgs(args) };
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("session-cancel", `/api/v1/sessions/${encodeURIComponent(id)}/control`, body, `bun scripts/cli.ts agentrun sessions cancel ${id}`);
|
|
return await agentRunRestRequest("agentrun sessions cancel", "POST", `/api/v1/sessions/${encodeURIComponent(id)}/control`, body);
|
|
}
|
|
if (action === "send" && id) return await sessionSendRest(id, args);
|
|
throw new AgentRunRestError("validation-failed", `unsupported sessions command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`);
|
|
}
|
|
|
|
async function runAgentRunRunsRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
|
|
if (action === "create") return await agentRunRestRequest("agentrun runs create", "POST", "/api/v1/runs", await requiredJsonBody(args, "runs create"));
|
|
if (action === "show" && id) return await agentRunRestRequest("agentrun runs show", "GET", `/api/v1/runs/${encodeURIComponent(id)}`);
|
|
if (action === "events" && id) return await agentRunRestRequest("agentrun runs events", "GET", `/api/v1/runs/${encodeURIComponent(id)}/events${agentRunQuery(args, ["after-seq", "limit"])}`);
|
|
if (action === "result" && id) return await agentRunRestRequest("agentrun runs result", "GET", `/api/v1/runs/${encodeURIComponent(id)}/result${agentRunQuery(args, ["command-id", "command"])}`);
|
|
if (action === "cancel" && id) return await agentRunRestRequest("agentrun runs cancel", "POST", `/api/v1/runs/${encodeURIComponent(id)}/cancel`, cancelBodyFromArgs(args));
|
|
throw new AgentRunRestError("validation-failed", `unsupported runs command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`);
|
|
}
|
|
|
|
async function runAgentRunCommandsRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
|
|
if (action === "create" && id) {
|
|
const body = await optionalJsonBody(args);
|
|
if (!body.type) body.type = agentRunOption(args, "type") ?? "turn";
|
|
const idempotencyKey = agentRunOption(args, "idempotency-key");
|
|
if (idempotencyKey) body.idempotencyKey = idempotencyKey;
|
|
return await agentRunRestRequest("agentrun commands create", "POST", `/api/v1/runs/${encodeURIComponent(id)}/commands`, body);
|
|
}
|
|
if (action === "show" && id) {
|
|
const runId = requiredAgentRunOption(args, ["run-id", "run"], "commands show requires --run-id");
|
|
return await agentRunRestRequest("agentrun commands show", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/commands/${encodeURIComponent(id)}`);
|
|
}
|
|
if (action === "result" && id) {
|
|
const runId = requiredAgentRunOption(args, ["run-id", "run"], "commands result requires --run-id");
|
|
return await agentRunRestRequest("agentrun commands result", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/commands/${encodeURIComponent(id)}/result`);
|
|
}
|
|
if (action === "cancel" && id) return await agentRunRestRequest("agentrun commands cancel", "POST", `/api/v1/commands/${encodeURIComponent(id)}/cancel`, cancelBodyFromArgs(args));
|
|
throw new AgentRunRestError("validation-failed", `unsupported commands command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`);
|
|
}
|
|
|
|
async function runAgentRunRunnerRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
|
|
if (action === "jobs") {
|
|
const runId = requiredAgentRunOption(args, ["run-id", "run"], "runner jobs requires --run-id");
|
|
return await agentRunRestRequest("agentrun runner jobs", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs${agentRunQuery(args, ["command-id", "command"])}`);
|
|
}
|
|
if (action === "job-status") {
|
|
const runId = requiredAgentRunOption(args, ["run-id", "run"], "runner job-status requires --run-id");
|
|
const runnerJobId = id ?? agentRunOption(args, "runner-job-id");
|
|
if (!runnerJobId) return await agentRunRestRequest("agentrun runner job-status", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs${agentRunQuery(args, ["command-id", "command"])}`);
|
|
return await agentRunRestRequest("agentrun runner job-status", "GET", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs/${encodeURIComponent(runnerJobId)}`);
|
|
}
|
|
if (action === "job") {
|
|
const runId = requiredAgentRunOption(args, ["run-id", "run"], "runner job requires --run-id");
|
|
const body = queueDispatchBodyFromArgs(args);
|
|
const commandId = requiredAgentRunOption(args, ["command-id", "command"], "runner job requires --command-id");
|
|
body.commandId = commandId;
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("runner-job", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, body, `bun scripts/cli.ts agentrun runner job --run-id ${runId} --command-id ${commandId}`);
|
|
return await agentRunRestRequest("agentrun runner job", "POST", `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, body);
|
|
}
|
|
throw new AgentRunRestError("validation-failed", `unsupported runner command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`);
|
|
}
|
|
|
|
async function runAgentRunAipodSpecsRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
|
|
if (action === "list") return await agentRunRestRequest("agentrun aipod-specs list", "GET", "/api/v1/aipod-specs");
|
|
if (action === "show" && id) return await agentRunRestRequest("agentrun aipod-specs show", "GET", `/api/v1/aipod-specs/${encodeURIComponent(id)}`);
|
|
if (action === "render" && id) return await agentRunRestRequest("agentrun aipod-specs render", "POST", `/api/v1/aipod-specs/${encodeURIComponent(id)}/render`, await aipodRenderInputFromArgs(args, 2));
|
|
if (action === "apply" || action === "set") {
|
|
const yaml = readYamlInputFromArgs(args);
|
|
const pathValue = id ? `/api/v1/aipod-specs/${encodeURIComponent(id)}` : "/api/v1/aipod-specs";
|
|
const method: AgentRunHttpMethod = id ? "PUT" : "POST";
|
|
const body = { yaml };
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("aipod-spec-apply", pathValue, { yamlBytes: Buffer.byteLength(yaml, "utf8") }, `bun scripts/cli.ts agentrun aipod-specs apply${id ? ` ${id}` : ""} --yaml-stdin`, method);
|
|
return await agentRunRestRequest("agentrun aipod-specs apply", method, pathValue, body);
|
|
}
|
|
if ((action === "delete" || action === "rm") && id) return await agentRunRestRequest("agentrun aipod-specs delete", "DELETE", `/api/v1/aipod-specs/${encodeURIComponent(id)}`);
|
|
throw new AgentRunRestError("validation-failed", `unsupported aipod-specs command: ${[action, id].filter(Boolean).join(" ") || "(empty)"}`);
|
|
}
|
|
|
|
async function submitQueueTaskRest(args: string[]): Promise<Record<string, unknown>> {
|
|
const aipod = agentRunOption(args, "aipod") ?? agentRunOption(args, "aipod-spec");
|
|
if (aipod) {
|
|
const rendered = await agentRunRestRequest("agentrun aipod-specs render", "POST", `/api/v1/aipod-specs/${encodeURIComponent(aipod)}/render`, await aipodRenderInputFromArgs(args, 2));
|
|
const body = record(record(innerData(rendered)).queueTask);
|
|
if (Object.keys(body).length === 0) throw new AgentRunRestError("schema-mismatch", `aipod-spec ${aipod} render did not return queueTask`);
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("queue-submit", "/api/v1/queue/tasks", body, queueSubmitConfirmCommand(args, aipod), "POST", { jsonInput: { source: "aipod-spec", aipod, valuesPrinted: false } });
|
|
return await agentRunRestRequest("agentrun queue submit", "POST", "/api/v1/queue/tasks", body);
|
|
}
|
|
const body = await requiredJsonBody(args, "queue submit");
|
|
const idempotencyKey = agentRunOption(args, "idempotency-key");
|
|
if (idempotencyKey) body.idempotencyKey = idempotencyKey;
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan("queue-submit", "/api/v1/queue/tasks", body, queueSubmitConfirmCommand(args), "POST", { jsonInput: jsonInputDisclosureFromArgs(args) });
|
|
return await agentRunRestRequest("agentrun queue submit", "POST", "/api/v1/queue/tasks", body);
|
|
}
|
|
|
|
async function mutateQueueTaskRest(action: string, taskId: string, suffix: string, body: Record<string, unknown>, args: string[]): Promise<Record<string, unknown>> {
|
|
const pathValue = `/api/v1/queue/tasks/${encodeURIComponent(taskId)}/${suffix}`;
|
|
if (agentRunHasFlag(args, "dry-run")) return agentRunDryRunPlan(action, pathValue, body, `bun scripts/cli.ts agentrun queue ${suffix} ${taskId}`);
|
|
return await agentRunRestRequest(`agentrun queue ${suffix}`, "POST", pathValue, body);
|
|
}
|
|
|
|
async function sessionSendRest(sessionId: string, args: string[]): Promise<Record<string, unknown>> {
|
|
const aipod = agentRunOption(args, "aipod") ?? agentRunOption(args, "aipod-spec");
|
|
if (aipod) return await sessionSendWithAipodRest(sessionId, aipod, args);
|
|
const input = await optionalJsonBody(args);
|
|
const prompt = optionalPromptFromArgs(args, 2);
|
|
const sendBody = await sessionSendBodyFromRunInput(sessionId, args, input, sessionSendPayloadFromInput(input, prompt));
|
|
const response = await agentRunRestRequest("agentrun sessions send", "POST", `/api/v1/sessions/${encodeURIComponent(sessionId)}/send`, sendBody);
|
|
return { ok: response.ok !== false, command: "agentrun sessions send", data: innerData(response), bridge: response.bridge };
|
|
}
|
|
|
|
function sessionSendPayloadFromInput(input: Record<string, unknown>, prompt: string | null): Record<string, unknown> {
|
|
const payload = record(input.payload);
|
|
if (Object.keys(payload).length > 0) return payload;
|
|
const text = prompt ?? stringOrNull(input.prompt) ?? stringOrNull(input.message) ?? stringOrNull(input.text);
|
|
if (text === null) throw new AgentRunRestError("validation-failed", "send requires payload or non-empty prompt/message/text; use --prompt-stdin, --prompt-file, --prompt, a trailing prompt, or JSON payload");
|
|
return { prompt: text };
|
|
}
|
|
|
|
async function sessionSendBodyFromRunInput(sessionId: string, args: string[], input: Record<string, unknown>, payload: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
const fullSendBody = isExplicitSessionSendBody(input);
|
|
const runInput = fullSendBody ? record(input.run ?? input.runBase) : input;
|
|
const runBody = await sessionRunBodyFromArgs(sessionId, args, runInput);
|
|
const runnerDefaults = fullSendBody ? record(input.runnerJob) : {};
|
|
const runnerJob = await sessionRunnerJobBody(args, runnerDefaults);
|
|
const body: Record<string, unknown> = fullSendBody ? { ...input } : {};
|
|
body.run = runBody;
|
|
body.payload = record(input.payload);
|
|
if (Object.keys(record(body.payload)).length === 0) body.payload = payload;
|
|
body.createRunnerJob = agentRunHasFlag(args, "no-runner-job") ? false : input.createRunnerJob !== false;
|
|
body.runnerJob = runnerJob;
|
|
body.dryRun = agentRunHasFlag(args, "dry-run");
|
|
const commandIdempotencyKey = agentRunOption(args, "command-idempotency-key") ?? agentRunOption(args, "idempotency-key");
|
|
if (commandIdempotencyKey) body.commandIdempotencyKey = commandIdempotencyKey;
|
|
return body;
|
|
}
|
|
|
|
function isExplicitSessionSendBody(input: Record<string, unknown>): boolean {
|
|
return isRecord(input.run)
|
|
|| isRecord(input.runBase)
|
|
|| isRecord(input.payload)
|
|
|| isRecord(input.runnerJob)
|
|
|| input.createRunnerJob !== undefined
|
|
|| input.commandIdempotencyKey !== undefined;
|
|
}
|
|
|
|
async function sessionRunBodyFromArgs(sessionId: string, args: string[], input: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
const existing = await fetchAgentRunSessionOrNull(sessionId, args);
|
|
const sessionPolicy = readAgentRunClientConfig().client.sessionPolicy;
|
|
const profile = agentRunOption(args, "profile")
|
|
?? agentRunOption(args, "backend-profile")
|
|
?? stringOrNull(input.backendProfile)
|
|
?? stringOrNull(existing?.backendProfile)
|
|
?? sessionPolicy.backendProfile;
|
|
const body: Record<string, unknown> = { ...input };
|
|
body.tenantId = agentRunOption(args, "tenant-id") ?? stringOrNull(body.tenantId) ?? stringOrNull(existing?.tenantId) ?? sessionPolicy.tenantId;
|
|
body.projectId = agentRunOption(args, "project-id") ?? stringOrNull(body.projectId) ?? stringOrNull(existing?.projectId) ?? sessionPolicy.projectId;
|
|
body.providerId = agentRunOption(args, "provider-id") ?? stringOrNull(body.providerId) ?? stringOrNull(existing?.providerId) ?? sessionPolicy.providerId;
|
|
body.backendProfile = profile;
|
|
body.workspaceRef = jsonObjectOption(args, "workspace-json") ?? record(body.workspaceRef);
|
|
if (Object.keys(record(body.workspaceRef)).length === 0) body.workspaceRef = cloneJsonRecord(sessionPolicy.workspaceRef);
|
|
body.executionPolicy = jsonObjectOption(args, "execution-policy-json") ?? record(body.executionPolicy);
|
|
if (Object.keys(record(body.executionPolicy)).length === 0) body.executionPolicy = defaultAgentRunExecutionPolicy(profile, sessionPolicy);
|
|
const inheritedSessionRef = existing === null ? {} : {
|
|
conversationId: existing.conversationId,
|
|
threadId: existing.threadId,
|
|
metadata: existing.metadata,
|
|
};
|
|
const sessionRef = { ...inheritedSessionRef, ...record(body.sessionRef) };
|
|
const metadata = { ...record(inheritedSessionRef.metadata), ...record(sessionRef.metadata) };
|
|
const title = agentRunOption(args, "title");
|
|
if (title) metadata.title = title;
|
|
body.sessionRef = { ...sessionRef, sessionId, metadata };
|
|
return body;
|
|
}
|
|
|
|
async function sessionSendWithAipodRest(sessionId: string, aipod: string, args: string[]): Promise<Record<string, unknown>> {
|
|
const rendered = await agentRunRestRequest("agentrun aipod-specs render", "POST", `/api/v1/aipod-specs/${encodeURIComponent(aipod)}/render`, await aipodRenderInputFromArgs(args, 2, { sessionId }));
|
|
const renderedData = record(innerData(rendered));
|
|
const task = record(renderedData.queueTask);
|
|
if (Object.keys(task).length === 0) throw new AgentRunRestError("schema-mismatch", `aipod-spec ${aipod} render did not return queueTask`);
|
|
const sessionRef = record(task.sessionRef);
|
|
const metadata = record(sessionRef.metadata);
|
|
const title = agentRunOption(args, "title") ?? stringOrNull(task.title);
|
|
if (title) metadata.title = title;
|
|
const sessionPolicy = readAgentRunClientConfig().client.sessionPolicy;
|
|
const backendProfile = stringOrNull(task.backendProfile) ?? sessionPolicy.backendProfile;
|
|
const executionPolicy = record(task.executionPolicy);
|
|
const runBody: Record<string, unknown> = {
|
|
tenantId: stringOrNull(task.tenantId) ?? sessionPolicy.tenantId,
|
|
projectId: stringOrNull(task.projectId) ?? sessionPolicy.projectId,
|
|
providerId: stringOrNull(task.providerId) ?? sessionPolicy.providerId,
|
|
backendProfile,
|
|
workspaceRef: task.workspaceRef ?? cloneJsonRecord(sessionPolicy.workspaceRef),
|
|
sessionRef: { ...sessionRef, sessionId, metadata },
|
|
executionPolicy: Object.keys(executionPolicy).length === 0 ? defaultAgentRunExecutionPolicy(backendProfile, sessionPolicy) : executionPolicy,
|
|
resourceBundleRef: task.resourceBundleRef,
|
|
traceSink: { kind: "aipod-session", aipod, sessionId, valuesPrinted: false },
|
|
};
|
|
const runnerDefaults = record(record(renderedData.dispatchDefaults).runnerJob);
|
|
const sendBody: Record<string, unknown> = {
|
|
run: runBody,
|
|
payload: task.payload,
|
|
createRunnerJob: !agentRunHasFlag(args, "no-runner-job"),
|
|
runnerJob: await sessionRunnerJobBody(args, runnerDefaults),
|
|
dryRun: agentRunHasFlag(args, "dry-run"),
|
|
};
|
|
const commandIdempotencyKey = agentRunOption(args, "command-idempotency-key") ?? agentRunOption(args, "idempotency-key");
|
|
if (commandIdempotencyKey) sendBody.commandIdempotencyKey = commandIdempotencyKey;
|
|
const response = await agentRunRestRequest("agentrun sessions send", "POST", `/api/v1/sessions/${encodeURIComponent(sessionId)}/send`, sendBody);
|
|
const data = record(innerData(response));
|
|
return { ok: response.ok !== false, command: "agentrun sessions send", data: { ...data, aipod, profile: String(task.backendProfile ?? ""), valuesPrinted: false }, bridge: response.bridge };
|
|
}
|
|
|
|
async function fetchAgentRunSessionOrNull(sessionId: string, args: string[]): Promise<Record<string, unknown> | null> {
|
|
try {
|
|
return record(innerData(await agentRunRestRequest("agentrun sessions show", "GET", `/api/v1/sessions/${encodeURIComponent(sessionId)}${agentRunQuery(args, ["reader-id"])}`)));
|
|
} catch (error) {
|
|
if (error instanceof AgentRunRestError && error.httpStatus === 404) return null;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function sessionRunnerJobBody(args: string[], defaults: Record<string, unknown> = {}): Promise<Record<string, unknown>> {
|
|
const runnerBody = { ...defaults, ...(await optionalRunnerJsonBody(args)) };
|
|
copyAgentRunOptions(args, runnerBody, ["image", "namespace", "attempt-id", "runner-id", "source-commit", "service-account-name"]);
|
|
const managerUrl = agentRunOption(args, "runner-manager-url");
|
|
if (managerUrl !== null) runnerBody.managerUrl = managerUrl;
|
|
const runnerIdempotencyKey = agentRunOption(args, "runner-idempotency-key");
|
|
if (runnerIdempotencyKey !== null) runnerBody.idempotencyKey = runnerIdempotencyKey;
|
|
return runnerBody;
|
|
}
|
|
|
|
async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body?: unknown): Promise<Record<string, unknown>> {
|
|
const clientConfig = readAgentRunClientConfig();
|
|
if (activeAgentRunRestTarget !== null) return await agentRunLaneRestRequest(command, method, pathValue, body, clientConfig, activeAgentRunRestTarget);
|
|
const auth = resolveAgentRunAuth(clientConfig);
|
|
const bridgeBase = agentRunRestBridgeMetadata(clientConfig, auth, method, pathValue);
|
|
const startedAt = Date.now();
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), clientConfig.manager.timeoutMs);
|
|
let response: Response;
|
|
try {
|
|
const headers: Record<string, string> = {
|
|
[clientConfig.auth.header]: `${clientConfig.auth.scheme} ${auth.value}`,
|
|
};
|
|
const init: RequestInit = { method, headers, signal: controller.signal };
|
|
if (body !== undefined) {
|
|
headers["content-type"] = "application/json";
|
|
init.body = JSON.stringify(body);
|
|
}
|
|
response = await fetch(new URL(pathValue, clientConfig.manager.baseUrl), init);
|
|
} catch (error) {
|
|
throw new AgentRunRestError("agentrun-unreachable", `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } });
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
const text = await response.text();
|
|
const bridge = { ...bridgeBase, httpStatus: response.status, elapsedMs: Date.now() - startedAt };
|
|
if (text.trim().length === 0) throw new AgentRunRestError("schema-mismatch", `AgentRun server returned an empty response for ${method} ${pathValue}`, { bridge, httpStatus: response.status });
|
|
let envelope: Record<string, unknown>;
|
|
try {
|
|
envelope = record(JSON.parse(text) as unknown);
|
|
} catch {
|
|
throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge, httpStatus: response.status });
|
|
}
|
|
if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
|
|
if (!response.ok) {
|
|
const details = response.status === 404
|
|
? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, null)
|
|
: safeAgentRunEnvelope(envelope);
|
|
throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details });
|
|
}
|
|
if (envelope.ok !== true) {
|
|
const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), response.status);
|
|
throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
|
|
}
|
|
return {
|
|
ok: true,
|
|
command,
|
|
data: envelope.data ?? null,
|
|
bridge,
|
|
};
|
|
}
|
|
|
|
async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise<Record<string, unknown>> {
|
|
const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue);
|
|
const startedAt = Date.now();
|
|
const request = {
|
|
method,
|
|
path: pathValue,
|
|
body: body ?? null,
|
|
baseUrl: target.spec.runtime.internalBaseUrl,
|
|
timeoutMs: clientConfig.manager.timeoutMs,
|
|
authEnv: clientConfig.auth.env,
|
|
header: clientConfig.auth.header,
|
|
scheme: clientConfig.auth.scheme,
|
|
valuesPrinted: false,
|
|
};
|
|
const captureResult = await capture(target.config, target.spec.nodeKubeRoute, ["sh", "--", agentRunLaneRestProxyScript(target.spec, request)]);
|
|
const captureSummary = compactCapture(captureResult, { full: false, stdoutTailChars: 1200, stderrTailChars: 2000 });
|
|
const bridge = { ...bridgeBase, elapsedMs: Date.now() - startedAt, capture: captureSummary };
|
|
if (captureResult.exitCode !== 0) {
|
|
throw new AgentRunRestError("agentrun-unreachable", `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed for ${method} ${pathValue}`, { bridge });
|
|
}
|
|
const proxy = captureJsonPayload(captureResult);
|
|
if (proxy.ok !== true) {
|
|
throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) });
|
|
}
|
|
const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0;
|
|
const text = stringOrNull(proxy.text) ?? "";
|
|
const responseBridge = { ...bridge, httpStatus, proxyElapsedMs: nonNegativeIntegerOrNull(proxy.elapsedMs) };
|
|
if (text.trim().length === 0) throw new AgentRunRestError("schema-mismatch", `AgentRun server returned an empty response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus });
|
|
let envelope: Record<string, unknown>;
|
|
try {
|
|
envelope = record(JSON.parse(text) as unknown);
|
|
} catch {
|
|
throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus });
|
|
}
|
|
if (httpStatus === 401 || httpStatus === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) });
|
|
if (httpStatus < 200 || httpStatus >= 300) {
|
|
const details = httpStatus === 404
|
|
? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, target)
|
|
: safeAgentRunEnvelope(envelope);
|
|
throw new AgentRunRestError(httpStatus === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${httpStatus}`, { bridge: responseBridge, httpStatus, details });
|
|
}
|
|
if (envelope.ok !== true) {
|
|
const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), httpStatus);
|
|
throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) });
|
|
}
|
|
return {
|
|
ok: true,
|
|
command,
|
|
data: envelope.data ?? null,
|
|
bridge: responseBridge,
|
|
};
|
|
}
|
|
|
|
function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record<string, unknown>): string {
|
|
const requestB64 = Buffer.from(JSON.stringify(request), "utf8").toString("base64");
|
|
const evalScript = String.raw`
|
|
const startedAt = Date.now();
|
|
try {
|
|
const input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8"));
|
|
const headers = {};
|
|
const authEnv = String(input.authEnv || "HWLAB_API_KEY");
|
|
const apiKey = process.env[authEnv] || process.env.AGENTRUN_API_KEY || process.env.HWLAB_API_KEY || "";
|
|
if (apiKey && input.header && input.scheme) headers[String(input.header)] = String(input.scheme) + " " + apiKey;
|
|
let body = undefined;
|
|
if (input.body !== null && input.body !== undefined) {
|
|
headers["content-type"] = "application/json";
|
|
body = JSON.stringify(input.body);
|
|
}
|
|
const controller = new AbortController();
|
|
const timeoutMs = Number(input.timeoutMs || 15000);
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
let response;
|
|
try {
|
|
response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), {
|
|
method: String(input.method || "GET"),
|
|
headers,
|
|
body,
|
|
signal: controller.signal,
|
|
});
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
const text = await response.text();
|
|
console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
|
|
} catch (error) {
|
|
console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
|
|
}
|
|
`;
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
`request_b64=${shQuote(requestB64)}`,
|
|
`kubectl -n "$namespace" exec deploy/"$deployment" -- env AGENTRUN_LANE_REST_REQUEST_B64="$request_b64" bun --eval ${shQuote(evalScript)}`,
|
|
].join("\n");
|
|
}
|
|
|
|
function resolveAgentRunRestTarget(config: UniDeskConfig | null, options: AgentRunRestTargetOptions): AgentRunRestTarget | null {
|
|
if (options.node === null && options.lane === null) return null;
|
|
if (config === null) throw new AgentRunRestError("validation-failed", "--node/--lane resource queries require UniDesk config");
|
|
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
|
return { config, configPath, spec };
|
|
}
|
|
|
|
async function withAgentRunRestTarget<T>(target: AgentRunRestTarget | null, action: () => Promise<T>): Promise<T> {
|
|
const previous = activeAgentRunRestTarget;
|
|
activeAgentRunRestTarget = target;
|
|
try {
|
|
return await action();
|
|
} finally {
|
|
activeAgentRunRestTarget = previous;
|
|
}
|
|
}
|
|
|
|
function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult {
|
|
const payload = error.toPayload(command);
|
|
if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false);
|
|
const laneHint = record(record(payload.agentrun).laneAwareLookup);
|
|
const currentEndpoint = record(laneHint.currentEndpoint);
|
|
const nextCommands = Array.isArray(laneHint.nextCommands)
|
|
? laneHint.nextCommands.filter((value): value is string => typeof value === "string" && value.length > 0).slice(0, 4)
|
|
: [];
|
|
const hintLines = Object.keys(laneHint).length === 0
|
|
? []
|
|
: [
|
|
"",
|
|
"Lane lookup hint:",
|
|
` Current baseUrl: ${String(currentEndpoint.baseUrl ?? "(unknown)")}`,
|
|
` Config: ${String(currentEndpoint.configPath ?? "(unknown)")}`,
|
|
` ${String(laneHint.summary ?? "The resource was not found on the currently configured AgentRun manager endpoint.")}`,
|
|
];
|
|
const nextLines = nextCommands.length > 0
|
|
? nextCommands
|
|
: [
|
|
"Check config/agentrun.yaml manager.baseUrl and the AgentRun API route.",
|
|
"Verify HWLAB_API_KEY is present in env or in the configured auth.file without printing the key.",
|
|
];
|
|
return renderedCliResult(false, command, [
|
|
`Error: ${payload.failureKind}`,
|
|
String(payload.message ?? ""),
|
|
...hintLines,
|
|
"",
|
|
"Next:",
|
|
...nextLines.map((line) => ` ${line}`),
|
|
].join("\n"));
|
|
}
|
|
|
|
class AgentRunRestError extends Error {
|
|
readonly failureKind: AgentRunFailureKind;
|
|
readonly bridge: Record<string, unknown> | null;
|
|
readonly httpStatus: number | null;
|
|
readonly details: Record<string, unknown> | null;
|
|
|
|
constructor(failureKind: AgentRunFailureKind, message: string, options: { bridge?: Record<string, unknown>; httpStatus?: number; details?: Record<string, unknown> } = {}) {
|
|
super(message);
|
|
this.name = "AgentRunRestError";
|
|
this.failureKind = failureKind;
|
|
this.bridge = options.bridge ?? null;
|
|
this.httpStatus = options.httpStatus ?? null;
|
|
this.details = options.details ?? null;
|
|
}
|
|
|
|
toPayload(command: string): Record<string, unknown> {
|
|
const transport = stringOrNull(this.bridge?.mode) ?? "direct-http";
|
|
return {
|
|
ok: false,
|
|
command,
|
|
failureKind: this.failureKind,
|
|
message: this.message,
|
|
transport,
|
|
clientRole: "render-only",
|
|
...(this.httpStatus === null ? {} : { httpStatus: this.httpStatus }),
|
|
...(this.bridge === null ? {} : { bridge: this.bridge }),
|
|
...(this.details === null ? {} : { agentrun: this.details }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
}
|
|
|
|
function readAgentRunClientConfig(env: NodeJS.ProcessEnv = process.env): AgentRunClientConfig {
|
|
const configPath = env.AGENTRUN_CLIENT_CONFIG ?? rootPath("config", "agentrun.yaml");
|
|
if (!existsSync(configPath)) throw new AgentRunRestError("validation-failed", `AgentRun client config not found: ${configPath}`);
|
|
try {
|
|
const parsed = parseAgentRunClientConfigYaml(readFileSync(configPath, "utf8"), configPath);
|
|
return { ...parsed, sourcePath: configPath };
|
|
} catch (error) {
|
|
if (error instanceof AgentRunRestError) throw error;
|
|
throw new AgentRunRestError("validation-failed", `AgentRun client config invalid: ${error instanceof Error ? error.message : String(error)}`, {
|
|
bridge: { mode: "direct-http", clientRole: "render-only", configPath },
|
|
});
|
|
}
|
|
}
|
|
|
|
export function parseAgentRunClientConfigYaml(raw: string, sourcePath = "config/agentrun.yaml"): AgentRunClientConfig {
|
|
const input = record(Bun.YAML.parse(raw) as unknown);
|
|
validateAgentRunConfigEnvelope(input, sourcePath);
|
|
const manager = record(input.manager);
|
|
const auth = record(input.auth);
|
|
const client = record(input.client);
|
|
const baseUrl = stringFieldFromRecord(manager, "baseUrl", "manager");
|
|
try {
|
|
const parsed = new URL(baseUrl);
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("invalid protocol");
|
|
} catch {
|
|
throw new Error(`${sourcePath}: manager.baseUrl must be an http(s) URL`);
|
|
}
|
|
const timeoutMs = numberFieldFromRecord(manager, "timeoutMs", "manager", { min: 1, max: 60_000 });
|
|
const header = stringFieldFromRecord(auth, "header", "auth");
|
|
const scheme = stringFieldFromRecord(auth, "scheme", "auth");
|
|
if (header.toLowerCase() !== "authorization") throw new Error(`${sourcePath}: auth.header must be Authorization`);
|
|
if (scheme !== "Bearer") throw new Error(`${sourcePath}: auth.scheme must be Bearer`);
|
|
const role = stringFieldFromRecord(client, "role", "client");
|
|
const transport = stringFieldFromRecord(client, "transport", "client");
|
|
if (role !== "render-only") throw new Error(`${sourcePath}: client.role must be render-only`);
|
|
if (transport !== "direct-http") throw new Error(`${sourcePath}: client.transport must be direct-http`);
|
|
const sessionPolicy = readAgentRunSessionPolicyConfig(client, sourcePath);
|
|
const publicExposure = readAgentRunPublicExposureConfig(input.publicExposure, baseUrl.replace(/\/+$/u, "/"), sourcePath);
|
|
return {
|
|
sourcePath,
|
|
manager: {
|
|
baseUrl: baseUrl.replace(/\/+$/u, "/"),
|
|
timeoutMs,
|
|
},
|
|
auth: {
|
|
env: stringFieldFromRecord(auth, "env", "auth"),
|
|
file: stringFieldFromRecord(auth, "file", "auth"),
|
|
header,
|
|
scheme,
|
|
},
|
|
client: {
|
|
role,
|
|
transport,
|
|
sessionPolicy,
|
|
},
|
|
publicExposure,
|
|
};
|
|
}
|
|
|
|
function readAgentRunSessionPolicyConfig(client: Record<string, unknown>, sourcePath: string): AgentRunSessionPolicyConfig {
|
|
const pathValue = "client.sessionPolicy";
|
|
const sessionPolicy = record(client.sessionPolicy);
|
|
const workspaceRef = record(sessionPolicy.workspaceRef);
|
|
const executionPolicy = record(sessionPolicy.executionPolicy);
|
|
const secretScope = record(executionPolicy.secretScope);
|
|
stringFieldFromRecord(workspaceRef, "kind", `${pathValue}.workspaceRef`);
|
|
stringFieldFromRecord(executionPolicy, "sandbox", `${pathValue}.executionPolicy`);
|
|
stringFieldFromRecord(executionPolicy, "approval", `${pathValue}.executionPolicy`);
|
|
positiveIntegerFieldFromRecord(executionPolicy, "timeoutMs", `${pathValue}.executionPolicy`);
|
|
stringFieldFromRecord(executionPolicy, "network", `${pathValue}.executionPolicy`);
|
|
booleanFieldFromRecord(secretScope, "allowCredentialEcho", `${pathValue}.executionPolicy.secretScope`);
|
|
return {
|
|
sourcePath,
|
|
tenantId: stringFieldFromRecord(sessionPolicy, "tenantId", pathValue),
|
|
projectId: stringFieldFromRecord(sessionPolicy, "projectId", pathValue),
|
|
providerId: stringFieldFromRecord(sessionPolicy, "providerId", pathValue),
|
|
backendProfile: stringFieldFromRecord(sessionPolicy, "backendProfile", pathValue),
|
|
workspaceRef: cloneJsonRecord(workspaceRef),
|
|
executionPolicy: cloneJsonRecord(executionPolicy),
|
|
};
|
|
}
|
|
|
|
function validateAgentRunConfigEnvelope(input: Record<string, unknown>, sourcePath: string): void {
|
|
if (input.version !== 1) throw new Error(`${sourcePath}: version must be 1`);
|
|
if (input.kind !== "AgentRunConfig") throw new Error(`${sourcePath}: kind must be AgentRunConfig`);
|
|
const metadata = record(input.metadata);
|
|
if (stringFieldFromRecord(metadata, "name", "metadata") !== "agentrun") throw new Error(`${sourcePath}: metadata.name must be agentrun`);
|
|
}
|
|
|
|
function readAgentRunPublicExposureConfig(value: unknown, managerBaseUrl: string, sourcePath: string): AgentRunPublicExposure | null {
|
|
if (value === undefined || value === null) return null;
|
|
const exposure = record(value);
|
|
if (exposure.enabled !== true) return null;
|
|
const proxyName = stringFieldFromRecord(exposure, "proxyName", "publicExposure");
|
|
if (!/^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/u.test(proxyName)) throw new Error(`${sourcePath}: publicExposure.proxyName must be a DNS label`);
|
|
const remotePort = numberFieldFromRecord(exposure, "remotePort", "publicExposure", { min: 1, max: 65_535 });
|
|
const publicBaseUrl = normalizedHttpUrl(stringFieldFromRecord(exposure, "publicBaseUrl", "publicExposure"), `${sourcePath}: publicExposure.publicBaseUrl`);
|
|
const masterBaseUrl = normalizedHttpUrl(stringFieldFromRecord(exposure, "masterBaseUrl", "publicExposure"), `${sourcePath}: publicExposure.masterBaseUrl`);
|
|
if (!publicBaseUrl.startsWith("https://")) throw new Error(`${sourcePath}: publicExposure.publicBaseUrl must use https://`);
|
|
if (publicBaseUrl !== managerBaseUrl) throw new Error(`${sourcePath}: manager.baseUrl must match publicExposure.publicBaseUrl`);
|
|
if (!masterBaseUrl.startsWith(`http://127.0.0.1:${remotePort}`)) throw new Error(`${sourcePath}: publicExposure.masterBaseUrl must point to http://127.0.0.1:${remotePort}`);
|
|
const masterFrps = record(exposure.masterFrps);
|
|
const masterCaddy = record(exposure.masterCaddy);
|
|
const caddyEnabled = masterCaddy.enabled === true;
|
|
const domain = stringFieldFromRecord(masterCaddy, "domain", "publicExposure.masterCaddy");
|
|
if (!/^[a-z0-9.-]+$/u.test(domain) || !domain.endsWith(".nip.io")) throw new Error(`${sourcePath}: publicExposure.masterCaddy.domain must be a nip.io hostname`);
|
|
const caddyUpstreamBaseUrl = normalizedHttpUrl(stringFieldFromRecord(masterCaddy, "upstreamBaseUrl", "publicExposure.masterCaddy"), `${sourcePath}: publicExposure.masterCaddy.upstreamBaseUrl`);
|
|
if (caddyUpstreamBaseUrl !== masterBaseUrl) throw new Error(`${sourcePath}: publicExposure.masterCaddy.upstreamBaseUrl must match publicExposure.masterBaseUrl`);
|
|
return {
|
|
enabled: true,
|
|
proxyName,
|
|
remotePort,
|
|
publicBaseUrl,
|
|
masterBaseUrl,
|
|
masterFrps: {
|
|
configPath: absolutePathField(masterFrps, "configPath", "publicExposure.masterFrps", sourcePath),
|
|
containerName: stringFieldFromRecord(masterFrps, "containerName", "publicExposure.masterFrps"),
|
|
},
|
|
masterCaddy: {
|
|
enabled: caddyEnabled,
|
|
domain,
|
|
configPath: absolutePathField(masterCaddy, "configPath", "publicExposure.masterCaddy", sourcePath),
|
|
serviceName: stringFieldFromRecord(masterCaddy, "serviceName", "publicExposure.masterCaddy"),
|
|
upstreamBaseUrl: caddyUpstreamBaseUrl,
|
|
responseHeaderTimeoutSeconds: numberFieldFromRecord(masterCaddy, "responseHeaderTimeoutSeconds", "publicExposure.masterCaddy", { min: 1, max: 600 }),
|
|
},
|
|
};
|
|
}
|
|
|
|
function normalizedHttpUrl(value: string, label: string): string {
|
|
let url: URL;
|
|
try {
|
|
url = new URL(value);
|
|
} catch {
|
|
throw new Error(`${label} must be an http(s) URL`);
|
|
}
|
|
if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`${label} must be an http(s) URL`);
|
|
return value.replace(/\/+$/u, "/");
|
|
}
|
|
|
|
function absolutePathField(obj: Record<string, unknown>, key: string, pathValue: string, sourcePath: string): string {
|
|
const value = stringFieldFromRecord(obj, key, pathValue);
|
|
if (!value.startsWith("/")) throw new Error(`${sourcePath}: ${pathValue}.${key} must be an absolute path`);
|
|
return value;
|
|
}
|
|
|
|
export function resolveAgentRunAuth(config: AgentRunClientConfig, env: NodeJS.ProcessEnv = process.env): AgentRunResolvedAuth {
|
|
const envValue = env[config.auth.env]?.trim();
|
|
if (envValue) return { source: "env", value: envValue };
|
|
if (existsSync(config.auth.file)) {
|
|
const fromFile = readEnvValueFromFile(config.auth.file, config.auth.env);
|
|
if (fromFile !== null) return { source: "file", value: fromFile };
|
|
}
|
|
throw new AgentRunRestError("auth-missing", `${config.auth.env} is required for direct AgentRun REST access`, {
|
|
bridge: {
|
|
mode: "direct-http",
|
|
clientRole: "render-only",
|
|
configPath: config.sourcePath,
|
|
auth: { env: config.auth.env, file: config.auth.file, source: "missing", header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false },
|
|
},
|
|
});
|
|
}
|
|
|
|
function readEnvValueFromFile(pathValue: string, envName: string): string | null {
|
|
const text = readFileSync(pathValue, "utf8");
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const trimmed = line.trim();
|
|
if (trimmed.length === 0 || trimmed.startsWith("#")) continue;
|
|
const normalized = trimmed.startsWith("export ") ? trimmed.slice("export ".length).trim() : trimmed;
|
|
const index = normalized.indexOf("=");
|
|
if (index <= 0) continue;
|
|
if (normalized.slice(0, index).trim() !== envName) continue;
|
|
const raw = normalized.slice(index + 1).trim();
|
|
const value = raw.replace(/^['"]|['"]$/gu, "");
|
|
return value.length > 0 ? value : null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRunResolvedAuth, method: AgentRunHttpMethod, pathValue: string): Record<string, unknown> {
|
|
return {
|
|
mode: "direct-http",
|
|
clientRole: "render-only",
|
|
compatibility: "no-hwlab-runtime-proxy-no-ssh-official-cli",
|
|
configPath: config.sourcePath,
|
|
baseUrl: config.manager.baseUrl,
|
|
request: { method, path: pathValue, timeoutMs: config.manager.timeoutMs },
|
|
auth: { env: config.auth.env, file: config.auth.file, source: auth.source, header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false },
|
|
};
|
|
}
|
|
|
|
function agentRunLaneRestBridgeMetadata(config: AgentRunClientConfig, target: AgentRunRestTarget, method: AgentRunHttpMethod, pathValue: string): Record<string, unknown> {
|
|
return {
|
|
mode: "lane-k8s-service-proxy",
|
|
clientRole: "render-only",
|
|
configPath: config.sourcePath,
|
|
laneConfigPath: target.configPath,
|
|
node: target.spec.nodeId,
|
|
lane: target.spec.lane,
|
|
kubeRoute: target.spec.nodeKubeRoute,
|
|
namespace: target.spec.runtime.namespace,
|
|
managerDeployment: target.spec.runtime.managerDeployment,
|
|
baseUrl: target.spec.runtime.internalBaseUrl,
|
|
request: { method, path: pathValue, timeoutMs: config.manager.timeoutMs },
|
|
auth: { source: "manager-pod-env", env: config.auth.env, fallbackEnv: "AGENTRUN_API_KEY", header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false },
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function addAgentRunNotFoundLookupHint(details: Record<string, unknown>, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string, target: AgentRunRestTarget | null): Record<string, unknown> {
|
|
const resource = agentRunResourceFromPath(pathValue);
|
|
const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath);
|
|
const candidateLanes = laneConfig?.lanes ?? [];
|
|
const nextCommands = candidateLanes
|
|
.filter((lane) => typeof lane.node === "string" && lane.node.length > 0 && typeof lane.lane === "string" && lane.lane.length > 0)
|
|
.slice(0, 4)
|
|
.map((lane) => `bun scripts/cli.ts agentrun control-plane status --node ${lane.node} --lane ${lane.lane}`);
|
|
return {
|
|
...details,
|
|
laneAwareLookup: {
|
|
kind: "agentrun-resource-not-found-on-current-endpoint",
|
|
summary: resource === null
|
|
? "The request returned 404 on the currently configured AgentRun manager endpoint; a resource from another lane will not be visible through this baseUrl."
|
|
: `${resource.kind}/${resource.id} returned 404 on the currently configured AgentRun manager endpoint; if it was created on another lane, inspect that lane before treating the resource as lost.`,
|
|
request: {
|
|
method,
|
|
path: pathValue,
|
|
...(resource === null ? {} : { resource }),
|
|
},
|
|
currentEndpoint: {
|
|
transport: target === null ? "direct-http" : "lane-k8s-service-proxy",
|
|
baseUrl: target === null ? config.manager.baseUrl : target.spec.runtime.internalBaseUrl,
|
|
configPath: config.sourcePath,
|
|
defaultNode: laneConfig?.defaultNode ?? null,
|
|
defaultLane: laneConfig?.defaultLane ?? null,
|
|
selectedNode: target?.spec.nodeId ?? null,
|
|
selectedLane: target?.spec.lane ?? null,
|
|
},
|
|
candidateLanes,
|
|
nextCommands,
|
|
note: target === null
|
|
? "Pass --node <node> --lane <lane> to query a YAML-declared non-default AgentRun lane."
|
|
: "The resource was not found on the selected AgentRun lane; verify the run/session id and the HWLAB runtime lane that created it.",
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function agentRunResourceFromPath(pathValue: string): Record<string, string> | null {
|
|
const run = pathValue.match(/\/runs\/([^/?#]+)/u);
|
|
if (run?.[1] !== undefined) return { kind: "run", id: decodeURIComponent(run[1]) };
|
|
const session = pathValue.match(/\/sessions\/([^/?#]+)/u);
|
|
if (session?.[1] !== undefined) return { kind: "session", id: decodeURIComponent(session[1]) };
|
|
const command = pathValue.match(/\/commands\/([^/?#]+)/u);
|
|
if (command?.[1] !== undefined) return { kind: "command", id: decodeURIComponent(command[1]) };
|
|
const task = pathValue.match(/\/queue\/tasks\/([^/?#]+)/u);
|
|
if (task?.[1] !== undefined) return { kind: "task", id: decodeURIComponent(task[1]) };
|
|
return null;
|
|
}
|
|
|
|
function readAgentRunLaneLookupConfig(configPath: string): { defaultNode: string | null; defaultLane: string | null; lanes: Record<string, unknown>[] } | null {
|
|
try {
|
|
const raw = readFileSync(configPath, "utf8");
|
|
const parsed = record(Bun.YAML.parse(raw) as unknown);
|
|
const controlPlane = record(parsed.controlPlane);
|
|
const defaultTarget = record(controlPlane.default);
|
|
const lanes = record(controlPlane.lanes);
|
|
return {
|
|
defaultNode: stringOrNull(defaultTarget.node),
|
|
defaultLane: stringOrNull(defaultTarget.lane),
|
|
lanes: Object.entries(lanes).map(([laneName, rawLane]) => {
|
|
const lane = record(rawLane);
|
|
const runtime = record(lane.runtime);
|
|
return {
|
|
lane: laneName,
|
|
node: stringOrNull(lane.node) ?? stringOrNull(lane.nodeId),
|
|
version: stringOrNull(lane.version),
|
|
namespace: stringOrNull(runtime.namespace),
|
|
internalBaseUrl: stringOrNull(runtime.internalBaseUrl),
|
|
isDefault: laneName === stringOrNull(defaultTarget.lane),
|
|
};
|
|
}),
|
|
};
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function agentRunDryRunPlan(action: string, pathValue: string, body: Record<string, unknown>, confirmCommand: string, method: AgentRunHttpMethod = "POST", extra: Record<string, unknown> = {}): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
action,
|
|
dryRun: true,
|
|
mutation: false,
|
|
transport: "direct-http",
|
|
clientRole: "render-only",
|
|
request: {
|
|
method,
|
|
path: pathValue,
|
|
bodyKeys: Object.keys(body),
|
|
valuesPrinted: false,
|
|
},
|
|
...extra,
|
|
next: { confirm: confirmCommand },
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function agentRunQuery(args: string[], names: string[]): string {
|
|
const params = new URLSearchParams();
|
|
const keyMap: Record<string, string> = {
|
|
"after-seq": "afterSeq",
|
|
"backend-profile": "backendProfile",
|
|
"command": "commandId",
|
|
"command-id": "commandId",
|
|
"reader-id": "readerId",
|
|
"run-id": "runId",
|
|
"updated-after": "updatedAfter",
|
|
};
|
|
for (const name of names) {
|
|
const value = agentRunOption(args, name);
|
|
if (value !== null) params.set(keyMap[name] ?? name, value);
|
|
}
|
|
const query = params.toString();
|
|
return query.length > 0 ? `?${query}` : "";
|
|
}
|
|
|
|
function agentRunOption(args: string[], name: string): string | null {
|
|
const flag = `--${name}`;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (arg === flag) {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new AgentRunRestError("validation-failed", `${flag} requires a value`);
|
|
return value;
|
|
}
|
|
if (arg.startsWith(`${flag}=`)) return arg.slice(flag.length + 1);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function requiredAgentRunOption(args: string[], names: string[], message: string): string {
|
|
for (const name of names) {
|
|
const value = agentRunOption(args, name);
|
|
if (value !== null) return value;
|
|
}
|
|
throw new AgentRunRestError("validation-failed", message);
|
|
}
|
|
|
|
function agentRunHasFlag(args: string[], name: string): boolean {
|
|
const flag = `--${name}`;
|
|
return args.includes(flag) || args.some((arg) => arg.startsWith(`${flag}=`));
|
|
}
|
|
|
|
async function requiredJsonBody(args: string[], context: string): Promise<Record<string, unknown>> {
|
|
const body = await optionalJsonBody(args);
|
|
if (Object.keys(body).length === 0) throw new AgentRunRestError("validation-failed", `${context} requires --json-stdin or --json-file <path>`);
|
|
return body;
|
|
}
|
|
|
|
async function optionalJsonBody(args: string[]): Promise<Record<string, unknown>> {
|
|
const raw = readJsonTextFromArgs(args, "json");
|
|
if (raw === null) return {};
|
|
return record(JSON.parse(raw) as unknown);
|
|
}
|
|
|
|
async function optionalRunnerJsonBody(args: string[]): Promise<Record<string, unknown>> {
|
|
const raw = readJsonTextFromArgs(args, "runner-json");
|
|
if (raw === null) return {};
|
|
return record(JSON.parse(raw) as unknown);
|
|
}
|
|
|
|
function readJsonTextFromArgs(args: string[], prefix: "json" | "runner-json"): string | null {
|
|
if (agentRunHasFlag(args, `${prefix}-stdin`)) return readFileSync(0, "utf8");
|
|
const file = agentRunOption(args, `${prefix}-file`);
|
|
if (file !== null) return file === "-" ? readFileSync(0, "utf8") : readFileSync(file, "utf8");
|
|
return null;
|
|
}
|
|
|
|
function readYamlInputFromArgs(args: string[]): string {
|
|
if (agentRunHasFlag(args, "yaml-stdin")) return readFileSync(0, "utf8");
|
|
const file = agentRunOption(args, "yaml-file");
|
|
if (file !== null) return file === "-" ? readFileSync(0, "utf8") : readFileSync(file, "utf8");
|
|
throw new AgentRunRestError("validation-failed", "aipod-spec YAML input is required; use --yaml-stdin or --yaml-file <file>");
|
|
}
|
|
|
|
async function aipodRenderInputFromArgs(args: string[], trailingPromptStart: number, overrides: Record<string, unknown> = {}): Promise<Record<string, unknown>> {
|
|
const input = await optionalJsonBody(args);
|
|
const prompt = optionalPromptFromArgs(args, trailingPromptStart);
|
|
if (prompt !== null) input.prompt = prompt;
|
|
copyAgentRunOptions(args, input, ["tenant-id", "project-id", "queue", "lane", "title", "provider-id", "idempotency-key", "session-id"]);
|
|
const profile = agentRunOption(args, "profile") ?? agentRunOption(args, "backend-profile");
|
|
if (profile) input.backendProfile = profile;
|
|
const priority = agentRunOption(args, "priority");
|
|
if (priority) input.priority = Number(priority);
|
|
const workspaceRef = jsonObjectOption(args, "workspace-json");
|
|
if (workspaceRef !== null) input.workspaceRef = workspaceRef;
|
|
return { ...input, ...overrides };
|
|
}
|
|
|
|
function readPromptFromArgs(args: string[], trailingStart: number): string {
|
|
const prompt = optionalPromptFromArgs(args, trailingStart);
|
|
if (prompt === null) throw new AgentRunRestError("validation-failed", "prompt is required; use --prompt-stdin, --prompt-file, --prompt, or a trailing prompt");
|
|
return prompt;
|
|
}
|
|
|
|
function optionalPromptFromArgs(args: string[], trailingStart: number): string | null {
|
|
if (agentRunHasFlag(args, "prompt-stdin") || agentRunHasFlag(args, "stdin")) return readFileSync(0, "utf8");
|
|
const promptFile = agentRunOption(args, "prompt-file");
|
|
if (promptFile !== null) return promptFile === "-" ? readFileSync(0, "utf8") : readFileSync(promptFile, "utf8");
|
|
const prompt = agentRunOption(args, "prompt");
|
|
if (prompt !== null) return prompt;
|
|
const trailing = args.slice(trailingStart).filter((arg) => !arg.startsWith("-"));
|
|
return trailing.length > 0 ? trailing.join(" ") : null;
|
|
}
|
|
|
|
function queueDispatchBodyFromArgs(args: string[]): Record<string, unknown> {
|
|
const body = recordFromMaybeJson(args);
|
|
copyAgentRunOptions(args, body, ["idempotency-key", "image", "namespace", "attempt-id", "runner-id", "source-commit", "service-account-name"]);
|
|
const managerUrl = agentRunOption(args, "runner-manager-url");
|
|
if (managerUrl) body.managerUrl = managerUrl;
|
|
return body;
|
|
}
|
|
|
|
function recordFromMaybeJson(args: string[]): Record<string, unknown> {
|
|
const raw = readJsonTextFromArgs(args, "json");
|
|
if (raw === null) return {};
|
|
return record(JSON.parse(raw) as unknown);
|
|
}
|
|
|
|
function cancelBodyFromArgs(args: string[]): Record<string, unknown> {
|
|
const reason = agentRunOption(args, "reason");
|
|
return reason === null ? {} : { reason };
|
|
}
|
|
|
|
function copyAgentRunOptions(args: string[], target: Record<string, unknown>, flagNames: string[]): void {
|
|
for (const flagName of flagNames) {
|
|
const value = agentRunOption(args, flagName);
|
|
if (value === null) continue;
|
|
target[camelCaseFlag(flagName)] = value;
|
|
}
|
|
}
|
|
|
|
function camelCaseFlag(flagName: string): string {
|
|
return flagName.replace(/-([a-z])/gu, (_, letter: string) => String(letter).toUpperCase());
|
|
}
|
|
|
|
function jsonObjectOption(args: string[], flagName: string): Record<string, unknown> | null {
|
|
const value = agentRunOption(args, flagName);
|
|
if (value === null) return null;
|
|
return record(JSON.parse(value) as unknown);
|
|
}
|
|
|
|
function queueSubmitConfirmCommand(args: string[], aipod?: string): string {
|
|
const dryRunless = args.filter((arg) => arg !== "--dry-run").join(" ");
|
|
return `bun scripts/cli.ts agentrun queue submit${aipod ? ` --aipod ${aipod}` : ""}${dryRunless.length > 0 ? ` ${dryRunless}` : ""}`.trim();
|
|
}
|
|
|
|
function jsonInputDisclosureFromArgs(args: string[]): Record<string, unknown> {
|
|
return {
|
|
source: agentRunHasFlag(args, "json-stdin") ? "stdin" : agentRunOption(args, "json-file") ?? "<task.json>",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function defaultAgentRunExecutionPolicy(profile: string, sessionPolicy: AgentRunSessionPolicyConfig = readAgentRunClientConfig().client.sessionPolicy): Record<string, unknown> {
|
|
const { spec } = resolveAgentRunLaneTarget({});
|
|
const credentials = agentRunProviderCredentialRefs(spec, profile);
|
|
if (credentials.length === 0) {
|
|
throw new AgentRunRestError("validation-failed", `config/agentrun.yaml has no providerCredential Secret binding for backendProfile=${profile} on default lane ${spec.nodeId}/${spec.lane}`);
|
|
}
|
|
const providerCredentials = credentials.map((credential) => {
|
|
if (credential.secretRef.namespace !== spec.runtime.namespace) {
|
|
throw new AgentRunRestError("validation-failed", `providerCredential ${profile} Secret ${credential.secretRef.name} is in namespace ${credential.secretRef.namespace}, expected default lane namespace ${spec.runtime.namespace}`);
|
|
}
|
|
return {
|
|
profile: credential.profile,
|
|
secretRef: {
|
|
name: credential.secretRef.name,
|
|
keys: credential.secretRef.keys,
|
|
},
|
|
};
|
|
});
|
|
const basePolicy = cloneJsonRecord(sessionPolicy.executionPolicy);
|
|
const secretScope = record(basePolicy.secretScope);
|
|
return {
|
|
...basePolicy,
|
|
secretScope: {
|
|
...secretScope,
|
|
providerCredentials,
|
|
},
|
|
};
|
|
}
|
|
|
|
function renderAgentRunSessionPolicyExplanation(): string {
|
|
const config = readAgentRunClientConfig();
|
|
const { spec } = resolveAgentRunLaneTarget({});
|
|
const sessionPolicy = config.client.sessionPolicy;
|
|
const credentials = agentRunProviderCredentialRefs(spec, sessionPolicy.backendProfile);
|
|
const execution = defaultAgentRunExecutionPolicy(sessionPolicy.backendProfile, sessionPolicy);
|
|
const credentialSources = credentials.map((credential) => ({
|
|
profile: credential.profile,
|
|
secretRef: credential.secretRef,
|
|
valuesPrinted: false,
|
|
}));
|
|
return [
|
|
"KIND: session-policy",
|
|
`CONFIG: ${config.sourcePath}`,
|
|
`DEFAULT LANE: ${spec.nodeId}/${spec.lane}`,
|
|
`DEFAULTS: tenantId=${sessionPolicy.tenantId} projectId=${sessionPolicy.projectId} providerId=${sessionPolicy.providerId} backendProfile=${sessionPolicy.backendProfile}`,
|
|
`WORKSPACE: ${JSON.stringify(sessionPolicy.workspaceRef)}`,
|
|
`EXECUTION: ${JSON.stringify(execution)}`,
|
|
`PROVIDER CREDENTIAL SOURCES: ${JSON.stringify(credentialSources)}`,
|
|
"VALUES: secret payloads are not printed",
|
|
].join("\n");
|
|
}
|
|
|
|
function safeAgentRunEnvelope(envelope: Record<string, unknown>): Record<string, unknown> {
|
|
return pickCompact(envelope, ["ok", "failureKind", "message", "code", "details", "valuesPrinted"]);
|
|
}
|
|
|
|
function normalizeAgentRunFailureKind(raw: string | null, httpStatus: number): AgentRunFailureKind {
|
|
if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
|
|
if (httpStatus === 401 || httpStatus === 403) return "auth-failed";
|
|
if (httpStatus === 404) return "not-found";
|
|
return raw === "schema-invalid" ? "validation-failed" : "validation-failed";
|
|
}
|
|
|
|
function stringFieldFromRecord(obj: Record<string, unknown>, key: string, pathValue: string): string {
|
|
const value = obj[key];
|
|
if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${pathValue}.${key} must be a non-empty string`);
|
|
return value.trim();
|
|
}
|
|
|
|
function numberFieldFromRecord(obj: Record<string, unknown>, key: string, pathValue: string, bounds: { min: number; max: number }): number {
|
|
const value = obj[key];
|
|
if (typeof value !== "number" || !Number.isFinite(value) || value < bounds.min || value > bounds.max) throw new Error(`${pathValue}.${key} must be a number between ${bounds.min} and ${bounds.max}`);
|
|
return value;
|
|
}
|
|
|
|
function positiveIntegerFieldFromRecord(obj: Record<string, unknown>, key: string, pathValue: string): number {
|
|
const value = obj[key];
|
|
if (!Number.isInteger(value) || Number(value) <= 0) throw new Error(`${pathValue}.${key} must be a positive integer`);
|
|
return Number(value);
|
|
}
|
|
|
|
function booleanFieldFromRecord(obj: Record<string, unknown>, key: string, pathValue: string): boolean {
|
|
const value = obj[key];
|
|
if (typeof value !== "boolean") throw new Error(`${pathValue}.${key} must be a boolean`);
|
|
return value;
|
|
}
|
|
|
|
function cloneJsonRecord(value: Record<string, unknown>): Record<string, unknown> {
|
|
return JSON.parse(JSON.stringify(value)) as Record<string, unknown>;
|
|
}
|
|
|
|
interface AgentRunClientConfig {
|
|
sourcePath: string;
|
|
manager: {
|
|
baseUrl: string;
|
|
timeoutMs: number;
|
|
};
|
|
auth: {
|
|
env: string;
|
|
file: string;
|
|
header: string;
|
|
scheme: string;
|
|
};
|
|
client: {
|
|
role: string;
|
|
transport: string;
|
|
sessionPolicy: AgentRunSessionPolicyConfig;
|
|
};
|
|
publicExposure: AgentRunPublicExposure | null;
|
|
}
|
|
|
|
interface AgentRunSessionPolicyConfig {
|
|
sourcePath: string;
|
|
tenantId: string;
|
|
projectId: string;
|
|
providerId: string;
|
|
backendProfile: string;
|
|
workspaceRef: Record<string, unknown>;
|
|
executionPolicy: Record<string, unknown>;
|
|
}
|
|
|
|
interface AgentRunPublicExposure {
|
|
enabled: true;
|
|
proxyName: string;
|
|
remotePort: number;
|
|
publicBaseUrl: string;
|
|
masterBaseUrl: string;
|
|
masterFrps: {
|
|
configPath: string;
|
|
containerName: string;
|
|
};
|
|
masterCaddy: {
|
|
enabled: boolean;
|
|
domain: string;
|
|
configPath: string;
|
|
serviceName: string;
|
|
upstreamBaseUrl: string;
|
|
responseHeaderTimeoutSeconds: number;
|
|
};
|
|
}
|
|
|
|
interface AgentRunResolvedAuth {
|
|
source: "env" | "file";
|
|
value: string;
|
|
}
|
|
|
|
type AgentRunHttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
|
|
type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
|
|
|
|
type AgentRunRestCompatGroup = "queue" | "sessions" | "aipod-specs" | "aipods" | "runs" | "commands" | "runner";
|
|
type AgentRunResourceVerb = "get" | "describe" | "events" | "logs" | "result" | "ack" | "cancel" | "dispatch" | "create" | "apply" | "send" | "explain";
|
|
type AgentRunResourceKind = "task" | "run" | "command" | "runnerjob" | "session" | "aipodspec";
|
|
type AgentRunOutputMode = "human" | "wide" | "name" | "json" | "yaml";
|
|
|
|
interface AgentRunResourceRef {
|
|
kind: AgentRunResourceKind;
|
|
name: string;
|
|
}
|
|
|
|
interface AgentRunResourceOptions {
|
|
output: AgentRunOutputMode;
|
|
full: boolean;
|
|
raw: boolean;
|
|
debug: boolean;
|
|
limit: number;
|
|
queue: string | null;
|
|
state: string | null;
|
|
unread: boolean;
|
|
readerId: string;
|
|
taskId: string | null;
|
|
runId: string | null;
|
|
commandId: string | null;
|
|
sessionId: string | null;
|
|
afterSeq: number | null;
|
|
tail: number | null;
|
|
fullText: boolean;
|
|
reason: string | null;
|
|
dryRun: boolean;
|
|
file: string | null;
|
|
aipod: string | null;
|
|
idempotencyKey: string | null;
|
|
promptStdin: boolean;
|
|
node: string | null;
|
|
lane: string | null;
|
|
passthroughArgs: string[];
|
|
}
|
|
|
|
interface AgentRunRestTargetOptions {
|
|
node: string | null;
|
|
lane: string | null;
|
|
}
|
|
|
|
interface AgentRunRestTarget {
|
|
config: UniDeskConfig;
|
|
configPath: string;
|
|
spec: AgentRunLaneSpec;
|
|
}
|
|
|
|
let activeAgentRunRestTarget: AgentRunRestTarget | null = null;
|
|
|
|
type AgentRunBridgeCaptureBackend = "local-backend-core-broker" | "remote-frontend-websocket";
|
|
|
|
interface LocalBackendCoreStatus {
|
|
dockerExecutable: boolean;
|
|
backendCoreContainer: boolean;
|
|
error: string | null;
|
|
}
|
|
|
|
interface AgentRunBridgeCapturePlan {
|
|
backend: AgentRunBridgeCaptureBackend;
|
|
route: string;
|
|
reason: string;
|
|
remoteHost: string | null;
|
|
localBackendCore: LocalBackendCoreStatus;
|
|
}
|
|
|
|
type AgentRunBridgeCaptureResult = SshCaptureResult & { bridgeExecution?: AgentRunBridgeCapturePlan };
|
|
|
|
let localBackendCoreStatusCache: LocalBackendCoreStatus | null = null;
|
|
|
|
async function capture(config: UniDeskConfig, target: string, args: string[]): Promise<AgentRunBridgeCaptureResult> {
|
|
const plan = agentRunBridgeCapturePlan(config, target);
|
|
if (plan.backend === "remote-frontend-websocket" && plan.remoteHost !== null) {
|
|
return await captureRemote(config, plan, target, args);
|
|
}
|
|
const local = attachBridgeExecution(await runSshCommandCapture(config, target, args), plan);
|
|
const fallbackHost = agentRunBridgeRemoteHost(config);
|
|
if (local.exitCode !== 0 && fallbackHost !== null && bridgeExecutionFailureKind(local)?.degradedReason === "capture-backend-unavailable") {
|
|
const fallbackPlan: AgentRunBridgeCapturePlan = {
|
|
...plan,
|
|
backend: "remote-frontend-websocket",
|
|
remoteHost: fallbackHost,
|
|
reason: "local-capture-backend-unavailable",
|
|
};
|
|
return await captureRemote(config, fallbackPlan, target, args);
|
|
}
|
|
return local;
|
|
}
|
|
|
|
async function captureRemote(config: UniDeskConfig, plan: AgentRunBridgeCapturePlan, target: string, args: string[]): Promise<AgentRunBridgeCaptureResult> {
|
|
if (plan.remoteHost === null) return attachBridgeExecution(remoteBridgeCaptureFailure(new Error("remote host is not configured")), plan);
|
|
try {
|
|
return attachBridgeExecution(await runRemoteSshCommandCapture(config, plan.remoteHost, target, args), plan);
|
|
} catch (error) {
|
|
return attachBridgeExecution(remoteBridgeCaptureFailure(error), plan);
|
|
}
|
|
}
|
|
|
|
function remoteBridgeCaptureFailure(error: unknown): SshCaptureResult {
|
|
const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
return {
|
|
exitCode: 255,
|
|
stdout: "",
|
|
stderr: `unidesk remote frontend ssh bridge failed: ${message}\n`,
|
|
};
|
|
}
|
|
|
|
function attachBridgeExecution(result: SshCaptureResult, plan: AgentRunBridgeCapturePlan): AgentRunBridgeCaptureResult {
|
|
return { ...result, bridgeExecution: plan };
|
|
}
|
|
|
|
function agentRunBridgeCapturePlan(config: UniDeskConfig, target: string, env: NodeJS.ProcessEnv = process.env): AgentRunBridgeCapturePlan {
|
|
const localBackendCore = detectLocalBackendCoreStatus();
|
|
const remoteHost = agentRunBridgeRemoteHost(config, env);
|
|
const runnerEnv = isAgentRunRunnerEnvironment(env);
|
|
if (runnerEnv && remoteHost !== null) {
|
|
return { backend: "remote-frontend-websocket", route: target, reason: "runner-environment", remoteHost, localBackendCore };
|
|
}
|
|
if (!localBackendCore.backendCoreContainer && remoteHost !== null) {
|
|
return { backend: "remote-frontend-websocket", route: target, reason: "local-backend-core-unavailable", remoteHost, localBackendCore };
|
|
}
|
|
return { backend: "local-backend-core-broker", route: target, reason: "main-server-local-backend-core", remoteHost: null, localBackendCore };
|
|
}
|
|
|
|
function detectLocalBackendCoreStatus(): LocalBackendCoreStatus {
|
|
if (localBackendCoreStatusCache !== null) return localBackendCoreStatusCache;
|
|
const result = spawnSync("docker", ["ps", "--format", "{{.Names}}"], { encoding: "utf8", timeout: 2000 });
|
|
if (result.error !== undefined) {
|
|
localBackendCoreStatusCache = {
|
|
dockerExecutable: false,
|
|
backendCoreContainer: false,
|
|
error: result.error.message,
|
|
};
|
|
return localBackendCoreStatusCache;
|
|
}
|
|
const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim();
|
|
localBackendCoreStatusCache = {
|
|
dockerExecutable: result.status === 0,
|
|
backendCoreContainer: result.status === 0 && String(result.stdout ?? "").split(/\r?\n/u).includes("unidesk-backend-core"),
|
|
error: result.status === 0 ? null : output || `docker ps exited ${result.status ?? "unknown"}`,
|
|
};
|
|
return localBackendCoreStatusCache;
|
|
}
|
|
|
|
function isAgentRunRunnerEnvironment(env: NodeJS.ProcessEnv): boolean {
|
|
return Boolean(
|
|
env.AGENTRUN_BOOT_MODE
|
|
|| env.AGENTRUN_RUN_ID
|
|
|| env.AGENTRUN_K8S_JOB_NAME
|
|
|| env.CODE_QUEUE_SERVICE_ROLE
|
|
|| env.CODE_QUEUE_INSTANCE_ID
|
|
|| env.KUBERNETES_SERVICE_HOST,
|
|
);
|
|
}
|
|
|
|
function agentRunBridgeRemoteHost(config: UniDeskConfig, env: NodeJS.ProcessEnv = process.env): string | null {
|
|
return normalizeRemoteHostHint(env.UNIDESK_MAIN_SERVER_IP)
|
|
?? normalizeRemoteHostHint(env.UNIDESK_MAIN_SERVER_HOST)
|
|
?? normalizeRemoteHostHint(env.CODE_QUEUE_DEV_CONTAINER_MASTER_HOST)
|
|
?? normalizeRemoteHostHint(config.network.publicHost);
|
|
}
|
|
|
|
function normalizeRemoteHostHint(raw: string | undefined): string | null {
|
|
const value = raw?.trim() ?? "";
|
|
if (value.length === 0 || value === "localhost" || value === "127.0.0.1" || value === "::1") return null;
|
|
return value.replace(/\/+$/u, "");
|
|
}
|
|
|
|
function captureBridgeMetadata(result: SshCaptureResult): Record<string, unknown> | null {
|
|
const bridgeExecution = (result as AgentRunBridgeCaptureResult).bridgeExecution;
|
|
if (bridgeExecution === undefined) return null;
|
|
return {
|
|
backend: bridgeExecution.backend,
|
|
route: bridgeExecution.route,
|
|
reason: bridgeExecution.reason,
|
|
remoteHost: bridgeExecution.remoteHost,
|
|
localBackendCore: bridgeExecution.localBackendCore,
|
|
};
|
|
}
|
|
|
|
function bridgeExecutionFailureKind(result: SshCaptureResult): { degradedReason: string; failureKind: string; recoveryActions: string[] } | null {
|
|
if (result.exitCode === 0) return null;
|
|
const stderr = result.stderr;
|
|
if (/No such container: unidesk-backend-core|failed to start broker|Executable not found.*"docker"|docker: not found|Cannot connect to the Docker daemon/iu.test(stderr)) {
|
|
return {
|
|
degradedReason: "capture-backend-unavailable",
|
|
failureKind: "bridge-execution-environment",
|
|
recoveryActions: [
|
|
"Run the same command from a healthy UniDesk main-server CLI, or provide UNIDESK_MAIN_SERVER_IP/UNIDESK_MAIN_SERVER_HOST so the bridge can use the frontend WebSocket SSH backend.",
|
|
"For AgentRun runner jobs, request the unidesk-ssh tool credential SecretRef instead of embedding secret values in prompts or payloads.",
|
|
"After restoring the bridge backend, retry the same bun scripts/cli.ts agentrun ... command; do not use direct kubectl or GitHub API bypasses.",
|
|
],
|
|
};
|
|
}
|
|
if (/frontend login failed|remote frontend ssh bridge|websocket error|timed out waiting for provider session|route-not-allowed/iu.test(stderr)) {
|
|
return {
|
|
degradedReason: "bridge-execution-environment-unavailable",
|
|
failureKind: "bridge-execution-environment",
|
|
recoveryActions: [
|
|
"Verify the UniDesk frontend/backend-core SSH bridge is reachable from this runner and that UNIDESK_MAIN_SERVER_IP or UNIDESK_MAIN_SERVER_HOST points at the main server.",
|
|
"Verify scoped UNIDESK_SSH_CLIENT_TOKEN route allowlist includes the requested AgentRun route, without printing token values.",
|
|
"Retry with the same bun scripts/cli.ts agentrun ... command after the controlled bridge path is restored.",
|
|
],
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function compactCapture(result: SshCaptureResult, options: { full?: boolean; stdoutTailChars?: number; stderrTailChars?: number } = {}): Record<string, unknown> {
|
|
const stdoutTailChars = options.stdoutTailChars ?? 8000;
|
|
const stderrTailChars = options.stderrTailChars ?? 4000;
|
|
const full = options.full ?? true;
|
|
const payload: Record<string, unknown> = {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
stdoutTailOmitted: !full && result.exitCode === 0,
|
|
stderrTailOmitted: !full && result.exitCode === 0,
|
|
};
|
|
const bridgeExecution = captureBridgeMetadata(result);
|
|
if (bridgeExecution !== null) payload.bridgeExecution = bridgeExecution;
|
|
const failureKind = bridgeExecutionFailureKind(result);
|
|
if (failureKind !== null) {
|
|
payload.degradedReason = failureKind.degradedReason;
|
|
payload.failureKind = failureKind.failureKind;
|
|
payload.recoveryActions = failureKind.recoveryActions;
|
|
}
|
|
if (full || result.exitCode !== 0) {
|
|
payload.stdoutTail = tail(result.stdout, stdoutTailChars);
|
|
payload.stderrTail = tail(result.stderr, stderrTailChars);
|
|
payload.stdoutTruncated = result.stdout.length > stdoutTailChars;
|
|
payload.stderrTruncated = result.stderr.length > stderrTailChars;
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
async function timedStatusStage<T>(stage: string, action: () => Promise<T>): Promise<TimedValue<T>> {
|
|
const startedAtMs = Date.now();
|
|
progressEvent("agentrun.control-plane.status.progress", { stage, status: "started" });
|
|
try {
|
|
const value = await action();
|
|
const exitCode = isCaptureResult(value) ? value.exitCode : typeof record(value).ok === "boolean" && record(value).ok !== true ? 1 : 0;
|
|
progressEvent("agentrun.control-plane.status.progress", {
|
|
stage,
|
|
status: exitCode === 0 ? "succeeded" : "failed",
|
|
exitCode,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
});
|
|
return { value, elapsedMs: Date.now() - startedAtMs };
|
|
} catch (error) {
|
|
progressEvent("agentrun.control-plane.status.progress", {
|
|
stage,
|
|
status: "failed",
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function progressEvent(event: string, payload: Record<string, unknown>): void {
|
|
const stage = displayValue(payload.stage ?? event);
|
|
const status = displayValue(payload.status ?? "-");
|
|
const details = [
|
|
payload.exitCode === undefined ? null : `exit=${displayValue(payload.exitCode)}`,
|
|
payload.elapsedMs === undefined ? null : `elapsed=${displayValue(payload.elapsedMs)}ms`,
|
|
payload.error === undefined ? null : `error=${truncateOneLine(displayValue(payload.error), 160)}`,
|
|
].filter((item): item is string => item !== null);
|
|
process.stderr.write(`status ${stage} ${status}${details.length === 0 ? "" : ` ${details.join(" ")}`}\n`);
|
|
}
|
|
|
|
function isCaptureResult(value: unknown): value is SshCaptureResult {
|
|
return typeof value === "object" && value !== null && "exitCode" in value && typeof (value as { exitCode?: unknown }).exitCode === "number";
|
|
}
|
|
|
|
function isAgentRunPipelineRunName(value: string): boolean {
|
|
return /^agentrun-v[0-9]+-ci-[0-9a-f]{12,40}(?:-[a-z0-9-]+)?$/u.test(value);
|
|
}
|
|
|
|
function captureJsonPayload(result: SshCaptureResult): Record<string, unknown> {
|
|
const trimmed = result.stdout.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(trimmed) as unknown);
|
|
} catch {
|
|
const lastJsonLine = trimmed.split(/\r?\n/u).reverse().find((line) => line.trim().startsWith("{") && line.trim().endsWith("}"));
|
|
if (lastJsonLine === undefined) return {};
|
|
try {
|
|
return record(JSON.parse(lastJsonLine) as unknown);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function nonNegativeIntegerOrNull(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isInteger(value) && value >= 0) return value;
|
|
if (typeof value === "string" && /^\d+$/u.test(value)) return Number(value);
|
|
return null;
|
|
}
|
|
|
|
function isGitSha(value: string): boolean {
|
|
return /^[0-9a-f]{40}$/u.test(value);
|
|
}
|
|
|
|
function tail(text: string, maxChars: number): string {
|
|
return text.length > maxChars ? text.slice(-maxChars) : text;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function shQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, "'\\''")}'`;
|
|
}
|
|
|
|
function unsupported(args: string[]): RenderedCliResult {
|
|
const command = `agentrun ${args.join(" ")}`.trim();
|
|
return renderedCliResult(false, command, [
|
|
`Error: unsupported AgentRun command: ${command}`,
|
|
"",
|
|
"Supported resource commands:",
|
|
" agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send",
|
|
"",
|
|
"Compatibility bridge groups:",
|
|
" agentrun aipod-specs|queue|runs|commands|runner|sessions",
|
|
"",
|
|
"Operations:",
|
|
" agentrun control-plane status|trigger-current|refresh",
|
|
" agentrun git-mirror status|sync|flush",
|
|
].join("\n"));
|
|
}
|