Codex
1724 lines
74 KiB
TypeScript
1724 lines
74 KiB
TypeScript
import { readFileSync } from "node:fs";
|
|
import type { UniDeskConfig } from "./config";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
|
import { startJob } from "./jobs";
|
|
|
|
const g14SourceRoute = "G14:/root/agentrun-v01";
|
|
const g14K3sRoute = "G14:k3s";
|
|
const sourceBranch = "v0.1";
|
|
const runtimeNamespace = "agentrun-v01";
|
|
const ciNamespace = "agentrun-ci";
|
|
const pipelineName = "agentrun-v01-ci-image-publish";
|
|
const argoNamespace = "argocd";
|
|
const argoApplication = "agentrun-g14-v01";
|
|
const gitopsBranch = "v0.1-gitops";
|
|
const gitMirrorNamespace = "devops-infra";
|
|
const gitMirrorReadUrl = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
|
|
const gitMirrorWriteUrl = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/agentrun.git";
|
|
const gitMirrorRepoPath = "/cache/pikasTech/agentrun.git";
|
|
const gitMirrorSyncJobPrefix = "git-mirror-agentrun-sync-manual";
|
|
const gitMirrorFlushJobPrefix = "git-mirror-agentrun-flush-manual";
|
|
const githubSshUrl = "ssh://git@ssh.github.com:443/pikasTech/agentrun.git";
|
|
const mirrorToolsImage = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1";
|
|
|
|
export function agentRunHelp(): unknown {
|
|
return {
|
|
command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs | git-mirror status|sync|flush | queue submit|list|show|stats|commander|read|cancel|dispatch|refresh | sessions ps|show|turn|steer|cancel|output|trace|read",
|
|
output: "json",
|
|
usage: [
|
|
"bun scripts/cli.ts agentrun v01 queue commander --reader-id cli",
|
|
"bun scripts/cli.ts agentrun v01 queue submit --json-file <task.json>",
|
|
"bun scripts/cli.ts agentrun v01 queue dispatch <taskId> --json-file <dispatch.json>",
|
|
"bun scripts/cli.ts agentrun v01 queue cancel <taskId> --reason <text>",
|
|
"bun scripts/cli.ts agentrun v01 sessions trace <sessionId> --after-seq 0 --limit 100",
|
|
"bun scripts/cli.ts agentrun v01 sessions output <sessionId> --after-seq 0 --limit 100",
|
|
"bun scripts/cli.ts agentrun v01 sessions steer <sessionId> --prompt-file <path>",
|
|
"bun scripts/cli.ts agentrun v01 sessions read <sessionId> --reader-id cli",
|
|
"bun scripts/cli.ts agentrun v01 control-plane status",
|
|
"bun scripts/cli.ts agentrun v01 control-plane status --full",
|
|
"bun scripts/cli.ts agentrun v01 control-plane trigger-current --dry-run",
|
|
"bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm",
|
|
"bun scripts/cli.ts agentrun v01 control-plane refresh --dry-run",
|
|
"bun scripts/cli.ts agentrun v01 control-plane refresh --confirm",
|
|
"bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
|
"bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm",
|
|
"bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --dry-run",
|
|
"bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --confirm",
|
|
"bun scripts/cli.ts agentrun v01 git-mirror status",
|
|
"bun scripts/cli.ts agentrun v01 git-mirror status --full",
|
|
"bun scripts/cli.ts agentrun v01 git-mirror sync --confirm",
|
|
"bun scripts/cli.ts agentrun v01 git-mirror flush --confirm",
|
|
],
|
|
description: "Operate AgentRun v0.1 Queue and Sessions through the official G14 /root/agentrun-v01 CLI, plus bounded Tekton/Argo control-plane and devops-infra git mirror actions through UniDesk routes. Queue/session commands are direct AgentRun CLI calls, not a UniDesk Code Queue adapter or double-write path.",
|
|
};
|
|
}
|
|
|
|
export async function runAgentRunCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
|
const [lane, group, action] = args;
|
|
if (lane !== "v01") return unsupported(args);
|
|
if (group === "control-plane") {
|
|
if (action === "status") return await status(config, parseDisclosureOptions(args.slice(3)));
|
|
if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(args.slice(3)));
|
|
if (action === "refresh") return await refresh(config, parseConfirmOptions(args.slice(3)));
|
|
if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(args.slice(3)));
|
|
if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(args.slice(3)));
|
|
}
|
|
if (group === "git-mirror") {
|
|
if (action === "status") return await gitMirrorStatus(config, parseDisclosureOptions(args.slice(3)));
|
|
if (action === "sync" || action === "flush") {
|
|
const options = parseGitMirrorOptions(args.slice(3));
|
|
if (options.confirm && !options.wait) return startAsyncAgentRunJob(`agentrun_v01_git_mirror_${action}`, ["bun", "scripts/cli.ts", "agentrun", "v01", "git-mirror", action, "--confirm", "--wait", "--timeout-seconds", String(options.timeoutSeconds)], `Run AgentRun v0.1 git mirror ${action} on G14`);
|
|
return await runGitMirrorJob(config, action, options);
|
|
}
|
|
}
|
|
if (group === "queue" || group === "sessions") {
|
|
return await runOfficialAgentRunCli(config, group, args.slice(2));
|
|
}
|
|
return unsupported(args);
|
|
}
|
|
|
|
interface TriggerOptions {
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface ConfirmOptions {
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface GitMirrorOptions extends ConfirmOptions {
|
|
timeoutSeconds: number;
|
|
wait: boolean;
|
|
}
|
|
|
|
interface CleanupRunsOptions extends ConfirmOptions {
|
|
minAgeMinutes: number;
|
|
limit: number;
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface CleanupReleasedPvOptions extends ConfirmOptions {
|
|
limit: number;
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface DisclosureOptions {
|
|
full: boolean;
|
|
raw: boolean;
|
|
}
|
|
|
|
interface TimedValue<T> {
|
|
value: T;
|
|
elapsedMs: number;
|
|
}
|
|
|
|
function parseDisclosureOptions(args: string[]): DisclosureOptions {
|
|
for (const arg of args) {
|
|
if (arg !== "--full" && arg !== "--raw") throw new Error(`unsupported status option: ${arg}`);
|
|
}
|
|
const raw = args.includes("--raw");
|
|
return { full: raw || args.includes("--full"), raw };
|
|
}
|
|
|
|
function parseTriggerOptions(args: string[]): TriggerOptions {
|
|
return parseConfirmOptions(args);
|
|
}
|
|
|
|
function parseConfirmOptions(args: string[]): ConfirmOptions {
|
|
if (args.includes("--confirm") && args.includes("--dry-run")) throw new Error("accepts only one of --confirm or --dry-run");
|
|
return {
|
|
confirm: args.includes("--confirm"),
|
|
dryRun: args.includes("--dry-run") || !args.includes("--confirm"),
|
|
};
|
|
}
|
|
|
|
function parseGitMirrorOptions(args: string[]): GitMirrorOptions {
|
|
const base = parseConfirmOptions(args);
|
|
const timeoutIndex = args.indexOf("--timeout-seconds");
|
|
const timeoutSeconds = timeoutIndex >= 0 ? Number(args[timeoutIndex + 1]) : 300;
|
|
if (!Number.isFinite(timeoutSeconds) || timeoutSeconds < 30) throw new Error("--timeout-seconds must be a number >= 30");
|
|
return { ...base, timeoutSeconds, wait: args.includes("--wait") };
|
|
}
|
|
|
|
function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions {
|
|
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds"]));
|
|
const base = parseConfirmOptions(args);
|
|
return {
|
|
...base,
|
|
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
|
limit: positiveIntegerOption(args, "--limit", 20, 500),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600),
|
|
};
|
|
}
|
|
|
|
function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPvOptions {
|
|
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds"]));
|
|
const base = parseConfirmOptions(args);
|
|
return {
|
|
...base,
|
|
limit: positiveIntegerOption(args, "--limit", 20, 500),
|
|
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
|
};
|
|
}
|
|
|
|
function validateOptions(args: string[], booleanOptions: Set<string>, valueOptions: Set<string>): void {
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (booleanOptions.has(arg)) continue;
|
|
if (valueOptions.has(arg)) {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function optionValue(args: string[], name: string): string | undefined {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return undefined;
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const raw = optionValue(args, name);
|
|
if (raw === undefined) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
async function status(config: UniDeskConfig, options: DisclosureOptions): Promise<Record<string, unknown>> {
|
|
const sourceProbe = await timedStatusStage("source", () => capture(config, g14SourceRoute, ["script", "--", [
|
|
"cd /root/agentrun-v01",
|
|
"git fetch origin v0.1 >/dev/null 2>&1 || true",
|
|
"printf 'sourceCommit='",
|
|
"git rev-parse HEAD",
|
|
"printf 'originV01='",
|
|
"git rev-parse origin/v0.1 2>/dev/null || true",
|
|
"printf 'gitopsLatest='",
|
|
`git ls-remote origin ${gitopsBranch} 2>/dev/null | awk '{print $1}' || true`,
|
|
"git status --short --branch",
|
|
].join("\n")]));
|
|
const source = sourceProbe.value;
|
|
const localSourceCommit = matchLine(source.stdout, "sourceCommit=");
|
|
const originSourceCommit = matchLine(source.stdout, "originV01=");
|
|
const sourceCommit = isGitSha(originSourceCommit ?? "") ? originSourceCommit : localSourceCommit;
|
|
const gitopsLatest = matchLine(source.stdout, "gitopsLatest=");
|
|
const pipelineRun = sourceCommit ? pipelineRunName(sourceCommit) : null;
|
|
const [runtimeProbe, mirrorProbe] = await Promise.all([
|
|
timedStatusStage("runtime", () => capture(config, g14K3sRoute, ["script", "--", statusScript(pipelineRun)])),
|
|
timedStatusStage("git-mirror", () => readGitMirrorStatus(config)),
|
|
]);
|
|
const k3s = runtimeProbe.value;
|
|
const mirror = mirrorProbe.value;
|
|
const argo = parseArgoStatus(k3s.stdout);
|
|
const ciSummary = labeledJson(k3s.stdout, "ciSummary");
|
|
const pipelineRunCondition = labeledJson(k3s.stdout, "pipelineRunCondition");
|
|
const managerImage = parseManagerImage(k3s.stdout);
|
|
const mirrorSummary = mirror.summary;
|
|
const runtimeAlignment = {
|
|
localHeadMatchesOrigin: Boolean(localSourceCommit && originSourceCommit && localSourceCommit === originSourceCommit),
|
|
argoRevision: argo.revision,
|
|
argoSyncStatus: argo.syncStatus,
|
|
argoHealthStatus: argo.healthStatus,
|
|
syncedToGitopsLatest: Boolean(gitopsLatest && argo.revision === gitopsLatest),
|
|
managerSourceCommit: managerImage.sourceCommit,
|
|
managerSourceMatchesExpected: Boolean(sourceCommit && managerImage.sourceCommit === sourceCommit),
|
|
};
|
|
const summary = {
|
|
sourceCommit,
|
|
expectedPipelineRun: pipelineRun,
|
|
pipelineRun: {
|
|
status: pipelineRunCondition.status ?? null,
|
|
reason: pipelineRunCondition.reason ?? null,
|
|
completionTime: pipelineRunCondition.completionTime ?? null,
|
|
},
|
|
argo,
|
|
managerImage,
|
|
gitMirror: {
|
|
localV01: mirrorSummary.localV01 ?? null,
|
|
githubV01: mirrorSummary.githubV01 ?? null,
|
|
localGitops: mirrorSummary.localGitops ?? null,
|
|
githubGitops: mirrorSummary.githubGitops ?? null,
|
|
pendingFlush: mirrorSummary.pendingFlush ?? null,
|
|
sourceInSync: mirrorSummary.sourceInSync ?? null,
|
|
gitopsInSync: mirrorSummary.gitopsInSync ?? null,
|
|
githubInSync: mirrorSummary.githubInSync ?? null,
|
|
},
|
|
aligned: pipelineRunCondition.status === "True" &&
|
|
runtimeAlignment.localHeadMatchesOrigin === true &&
|
|
runtimeAlignment.syncedToGitopsLatest === true &&
|
|
runtimeAlignment.managerSourceMatchesExpected === true &&
|
|
mirrorSummary.githubInSync === true &&
|
|
mirrorSummary.pendingFlush === false,
|
|
};
|
|
return {
|
|
ok: source.exitCode === 0 && k3s.exitCode === 0 && mirror.ok === true,
|
|
command: "agentrun v01 control-plane status",
|
|
lane: "v0.1",
|
|
summary,
|
|
sourceCommit,
|
|
sourceCommitSource: sourceCommit === originSourceCommit ? "origin/v0.1" : "local-head",
|
|
localSourceCommit,
|
|
originSourceCommit,
|
|
gitopsLatest,
|
|
expectedPipelineRun: pipelineRun,
|
|
timings: {
|
|
sourceMs: sourceProbe.elapsedMs,
|
|
runtimeMs: runtimeProbe.elapsedMs,
|
|
gitMirrorMs: mirrorProbe.elapsedMs,
|
|
totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs),
|
|
},
|
|
source: compactCapture(source, { full: options.full || options.raw, stdoutTailChars: 3000, stderrTailChars: 2000 }),
|
|
runtime: compactCapture(k3s, { full: options.full || options.raw, stdoutTailChars: 8000, stderrTailChars: 4000 }),
|
|
pipelineRunCondition,
|
|
ciSummary,
|
|
gitMirror: {
|
|
ok: mirror.ok,
|
|
readUrl: mirror.readUrl,
|
|
writeUrl: mirror.writeUrl,
|
|
summary: mirror.summary,
|
|
probe: compactCapture(mirror.result, { full: options.full || options.raw, stdoutTailChars: 6000, stderrTailChars: 3000 }),
|
|
...(options.raw ? { raw: mirror.raw } : {}),
|
|
},
|
|
runtimeAlignment,
|
|
disclosure: {
|
|
defaultView: "compact-low-noise",
|
|
full: options.full,
|
|
raw: options.raw,
|
|
stdoutTailOmitted: !(options.full || options.raw),
|
|
rawGitMirrorOmitted: !options.raw,
|
|
expandWith: "bun scripts/cli.ts agentrun v01 control-plane status --full",
|
|
rawWith: "bun scripts/cli.ts agentrun v01 control-plane status --raw",
|
|
},
|
|
next: {
|
|
triggerCurrent: "bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm",
|
|
refresh: "bun scripts/cli.ts agentrun v01 control-plane refresh --confirm",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): Promise<Record<string, unknown>> {
|
|
const source = await capture(config, g14SourceRoute, ["script", "--", [
|
|
"set -u",
|
|
"cd /root/agentrun-v01",
|
|
"fetch_status=succeeded",
|
|
"fetch_output=$(git fetch origin v0.1 2>&1)",
|
|
"fetch_code=$?",
|
|
"if [ \"$fetch_code\" -ne 0 ]; then fetch_status=failed; fi",
|
|
"printf 'fetchStatus=%s\\n' \"$fetch_status\"",
|
|
"printf 'fetchExitCode=%s\\n' \"$fetch_code\"",
|
|
"printf 'fetchOutput=%s\\n' \"$fetch_output\" | tail -n 8",
|
|
"origin_ref=$(git rev-parse refs/remotes/origin/v0.1 2>/dev/null || true)",
|
|
"merge_status=skipped",
|
|
"merge_code=0",
|
|
"if [ -n \"$origin_ref\" ]; then",
|
|
" merge_output=$(git merge --ff-only refs/remotes/origin/v0.1 2>&1)",
|
|
" merge_code=$?",
|
|
" if [ \"$merge_code\" -eq 0 ]; then merge_status=succeeded; else merge_status=failed; fi",
|
|
" printf 'mergeStatus=%s\\n' \"$merge_status\"",
|
|
" printf 'mergeExitCode=%s\\n' \"$merge_code\"",
|
|
" printf 'mergeOutput=%s\\n' \"$merge_output\" | tail -n 8",
|
|
" if [ \"$merge_code\" -ne 0 ]; then exit \"$merge_code\"; fi",
|
|
"else",
|
|
" printf 'mergeStatus=skipped-origin-missing\\n'",
|
|
" printf 'originRefMissing=true\\n'",
|
|
"fi",
|
|
"printf 'sourceCommit='",
|
|
"git rev-parse HEAD",
|
|
"printf 'originV01='",
|
|
"git rev-parse refs/remotes/origin/v0.1 2>/dev/null || true",
|
|
"git status --short --branch",
|
|
].join("\n")]);
|
|
const sourceCommit = matchLine(source.stdout, "sourceCommit=");
|
|
const pipelineRun = sourceCommit ? pipelineRunName(sourceCommit) : null;
|
|
if (source.exitCode !== 0 || !sourceCommit || !isGitSha(sourceCommit) || !pipelineRun) {
|
|
return { ok: false, command: "agentrun v01 control-plane trigger-current", degradedReason: "source-head-unresolved", source: compactCapture(source) };
|
|
}
|
|
const plan = {
|
|
lane: "v0.1",
|
|
sourceBranch,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
namespace: ciNamespace,
|
|
pipeline: pipelineName,
|
|
runtimeNamespace,
|
|
gitMirror: {
|
|
readUrl: gitMirrorReadUrl,
|
|
writeUrl: gitMirrorWriteUrl,
|
|
},
|
|
};
|
|
const mirrorBefore = await readGitMirrorStatus(config);
|
|
const mirrorRequirement = gitMirrorSyncRequirement(sourceCommit, mirrorBefore.raw);
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun v01 control-plane trigger-current",
|
|
dryRun: true,
|
|
plan,
|
|
gitMirrorPreSync: {
|
|
required: mirrorRequirement.required,
|
|
reason: mirrorRequirement.reason,
|
|
before: mirrorBefore.summary,
|
|
},
|
|
next: { confirm: "bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm" },
|
|
};
|
|
}
|
|
let gitMirrorPreSync: Record<string, unknown> = {
|
|
required: mirrorRequirement.required,
|
|
reason: mirrorRequirement.reason,
|
|
before: mirrorBefore.summary,
|
|
};
|
|
if (mirrorRequirement.required) {
|
|
const synced = await runGitMirrorJob(config, "sync", { confirm: true, dryRun: false, timeoutSeconds: 300, wait: true });
|
|
const after = await readGitMirrorStatus(config);
|
|
const afterRequirement = gitMirrorSyncRequirement(sourceCommit, after.raw);
|
|
gitMirrorPreSync = { ...gitMirrorPreSync, sync: synced, after: after.summary, ok: synced.ok === true && afterRequirement.required === false };
|
|
if (synced.ok !== true || afterRequirement.required !== false) {
|
|
return {
|
|
ok: false,
|
|
command: "agentrun v01 control-plane trigger-current",
|
|
dryRun: false,
|
|
degradedReason: "git-mirror-local-v01-not-current-after-sync",
|
|
plan,
|
|
gitMirrorPreSync,
|
|
};
|
|
}
|
|
}
|
|
const created = await capture(config, g14K3sRoute, ["script", "--", triggerScript(sourceCommit, pipelineRun)]);
|
|
return {
|
|
ok: created.exitCode === 0,
|
|
command: "agentrun v01 control-plane trigger-current",
|
|
dryRun: false,
|
|
plan,
|
|
gitMirrorPreSync,
|
|
created: compactCapture(created),
|
|
next: {
|
|
status: "bun scripts/cli.ts agentrun v01 control-plane status",
|
|
logs: `trans G14:k3s logs -n ${ciNamespace} -l tekton.dev/pipelineRun=${pipelineRun} --tail 120`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function refresh(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const source = await capture(config, g14SourceRoute, ["script", "--", [
|
|
"cd /root/agentrun-v01",
|
|
"printf 'gitopsLatest='",
|
|
`git ls-remote origin ${gitopsBranch} 2>/dev/null | awk '{print $1}' || true`,
|
|
].join("\n")]);
|
|
const gitopsLatest = matchLine(source.stdout, "gitopsLatest=");
|
|
const plan = {
|
|
lane: "v0.1",
|
|
argoNamespace,
|
|
argoApplication,
|
|
gitopsBranch,
|
|
gitopsLatest,
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command: "agentrun v01 control-plane refresh",
|
|
dryRun: true,
|
|
plan,
|
|
next: { confirm: "bun scripts/cli.ts agentrun v01 control-plane refresh --confirm" },
|
|
};
|
|
}
|
|
const refreshed = await capture(config, g14K3sRoute, ["script", "--", refreshScript()]);
|
|
return {
|
|
ok: source.exitCode === 0 && refreshed.exitCode === 0,
|
|
command: "agentrun v01 control-plane refresh",
|
|
dryRun: false,
|
|
plan,
|
|
source: compactCapture(source),
|
|
refreshed: compactCapture(refreshed),
|
|
next: { status: "bun scripts/cli.ts agentrun v01 control-plane status" },
|
|
};
|
|
}
|
|
|
|
async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise<Record<string, unknown>> {
|
|
const result = await capture(config, g14K3sRoute, ["script", "--", cleanupRunsScript(options)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun v01 control-plane cleanup-runs",
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: ciNamespace,
|
|
minAgeMinutes: options.minAgeMinutes,
|
|
limit: options.limit,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
status: "bun scripts/cli.ts agentrun v01 control-plane status",
|
|
releasedPvs: `bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit ${options.limit} --dry-run`,
|
|
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise<Record<string, unknown>> {
|
|
const result = await capture(config, g14K3sRoute, ["script", "--", cleanupReleasedPvsScript(options)]);
|
|
const payload = captureJsonPayload(result);
|
|
const ok = result.exitCode === 0 && payload.ok !== false;
|
|
const base = {
|
|
...payload,
|
|
ok,
|
|
command: "agentrun v01 control-plane cleanup-released-pvs",
|
|
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
|
namespace: ciNamespace,
|
|
limit: options.limit,
|
|
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
|
};
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
...base,
|
|
dryRun: true,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit ${options.limit} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
dryRun: false,
|
|
mutation: true,
|
|
followUp: {
|
|
cleanupRuns: `bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit ${options.limit} --dry-run`,
|
|
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
|
},
|
|
};
|
|
}
|
|
|
|
function cleanupRunsScript(options: CleanupRunsOptions): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(ciNamespace)}`,
|
|
`min_age_minutes=${String(options.minAgeMinutes)}`,
|
|
`limit=${String(options.limit)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"",
|
|
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
|
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
|
|
"NAMESPACE=\"$namespace\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupRunsPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-names.txt\" ]; then",
|
|
" xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupRunsFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions): string {
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(ciNamespace)}`,
|
|
`limit=${String(options.limit)}`,
|
|
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
|
"NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
|
cleanupReleasedPvsPlanNodeScript(),
|
|
"NODE",
|
|
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
|
" cat \"$tmp_dir/plan.json\"",
|
|
" exit 0",
|
|
"fi",
|
|
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"",
|
|
"delete_exit=0",
|
|
"if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then",
|
|
" xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
|
"else",
|
|
" : > \"$tmp_dir/delete.out\"",
|
|
" : > \"$tmp_dir/delete.err\"",
|
|
"fi",
|
|
"kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"",
|
|
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
|
cleanupReleasedPvsFinalizeNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function cleanupRunsPlanNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const cp = require("node:child_process");
|
|
const tmp = process.env.TMP_DIR;
|
|
const namespace = process.env.NAMESPACE;
|
|
const minAgeMinutes = Number(process.env.MIN_AGE_MINUTES || 60);
|
|
const limit = Number(process.env.LIMIT || 20);
|
|
const now = Date.now();
|
|
|
|
function readJson(name) {
|
|
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
|
}
|
|
|
|
function conditionOf(item) {
|
|
const conditions = Array.isArray(item?.status?.conditions) ? item.status.conditions : [];
|
|
return conditions.find((entry) => entry.type === "Succeeded") || conditions[0] || {};
|
|
}
|
|
|
|
function ageMinutes(createdAt) {
|
|
const createdMs = Date.parse(createdAt || "");
|
|
return Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
|
}
|
|
|
|
function localPathOf(pv) {
|
|
return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
|
|
}
|
|
|
|
function duBytes(hostPath) {
|
|
if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
|
try {
|
|
const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
const bytes = Number(String(out).trim().split(/\s+/)[0]);
|
|
return Number.isFinite(bytes) ? bytes : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!Number.isFinite(bytes)) return null;
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let value = bytes;
|
|
let unit = 0;
|
|
while (value >= 1024 && unit < units.length - 1) {
|
|
value /= 1024;
|
|
unit += 1;
|
|
}
|
|
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
|
}
|
|
|
|
const pipelineRuns = readJson("pipelineruns.json");
|
|
const pvcs = readJson("pvcs.json");
|
|
const pvs = readJson("pvs.json");
|
|
const pods = readJson("pods.json");
|
|
const pvByName = new Map((Array.isArray(pvs.items) ? pvs.items : []).map((item) => [item?.metadata?.name, item]));
|
|
const activeClaimPods = new Map();
|
|
for (const pod of Array.isArray(pods.items) ? pods.items : []) {
|
|
const phase = pod?.status?.phase || "";
|
|
if (phase === "Succeeded" || phase === "Failed") continue;
|
|
for (const volume of Array.isArray(pod?.spec?.volumes) ? pod.spec.volumes : []) {
|
|
const claimName = volume?.persistentVolumeClaim?.claimName;
|
|
if (!claimName) continue;
|
|
const entry = activeClaimPods.get(claimName) || [];
|
|
entry.push(pod?.metadata?.name || null);
|
|
activeClaimPods.set(claimName, entry.filter(Boolean));
|
|
}
|
|
}
|
|
|
|
const pvcsByOwner = new Map();
|
|
for (const pvc of Array.isArray(pvcs.items) ? pvcs.items : []) {
|
|
const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
|
|
if (!owner?.name) continue;
|
|
const entry = pvcsByOwner.get(owner.name) || [];
|
|
entry.push(pvc);
|
|
pvcsByOwner.set(owner.name, entry);
|
|
}
|
|
|
|
const allPipelineRuns = (Array.isArray(pipelineRuns.items) ? pipelineRuns.items : [])
|
|
.map((item) => {
|
|
const condition = conditionOf(item);
|
|
return {
|
|
name: item?.metadata?.name || "",
|
|
createdAt: item?.metadata?.creationTimestamp || null,
|
|
ageMinutes: ageMinutes(item?.metadata?.creationTimestamp),
|
|
status: condition.status || null,
|
|
reason: condition.reason || null,
|
|
};
|
|
})
|
|
.filter((item) => item.name.startsWith("agentrun-v01-ci-"));
|
|
|
|
const protectedActivePipelineRuns = allPipelineRuns
|
|
.filter((item) => item.status !== "True" && item.status !== "False")
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
|
|
const protectedLatestPipelineRun = allPipelineRuns
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0]?.name || null;
|
|
|
|
const candidates = allPipelineRuns
|
|
.filter((item) => item.status === "True" || item.status === "False")
|
|
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= minAgeMinutes)
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
|
.slice(0, limit)
|
|
.map((item) => {
|
|
const owned = pvcsByOwner.get(item.name) || [];
|
|
const activeMountPods = owned.flatMap((pvc) => activeClaimPods.get(pvc?.metadata?.name) || []);
|
|
const protectedLatest = item.name === protectedLatestPipelineRun;
|
|
return {
|
|
...item,
|
|
selected: activeMountPods.length === 0 && !protectedLatest,
|
|
selectedReason: protectedLatest ? "protected-latest-pipelinerun" : activeMountPods.length === 0 ? "terminal-and-unmounted" : "owned-pvc-active-mounted",
|
|
ownedPvcCount: owned.length,
|
|
activeMountPods,
|
|
};
|
|
});
|
|
|
|
const selectedPipelineRuns = candidates.filter((item) => item.selected).map((item) => item.name);
|
|
const selectedSet = new Set(selectedPipelineRuns);
|
|
const ownedPvcs = [];
|
|
const protectedOwnedPvcs = [];
|
|
for (const [owner, items] of pvcsByOwner.entries()) {
|
|
if (!selectedSet.has(owner) && !candidates.some((item) => item.name === owner)) continue;
|
|
for (const pvc of items) {
|
|
const volume = pvc?.spec?.volumeName || null;
|
|
const pv = volume ? pvByName.get(volume) : null;
|
|
const hostPath = localPathOf(pv);
|
|
const activeMountPods = activeClaimPods.get(pvc?.metadata?.name) || [];
|
|
const entry = {
|
|
name: pvc?.metadata?.name || null,
|
|
volume,
|
|
phase: pvc?.status?.phase || null,
|
|
ownerKind: "PipelineRun",
|
|
owner,
|
|
storageClass: pv?.spec?.storageClassName || null,
|
|
reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
|
|
hostPath,
|
|
estimatedBytes: duBytes(hostPath),
|
|
activeMountPods,
|
|
};
|
|
if (selectedSet.has(owner)) ownedPvcs.push(entry);
|
|
else protectedOwnedPvcs.push(entry);
|
|
}
|
|
}
|
|
|
|
const estimatedReclaimBytes = ownedPvcs.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
planKind: "agentrun-ci-completed-pipelinerun-workspace-retention",
|
|
generatedAt: new Date().toISOString(),
|
|
namespace,
|
|
criteria: { prefix: "agentrun-v01-ci-", terminalStatuses: ["True", "False"], minAgeMinutes, limit },
|
|
candidates,
|
|
candidateCount: candidates.length,
|
|
protectedActivePipelineRuns,
|
|
protectedActivePipelineRunCount: protectedActivePipelineRuns.length,
|
|
protectedLatestPipelineRun,
|
|
selectedPipelineRuns,
|
|
selectedPipelineRunCount: selectedPipelineRuns.length,
|
|
ownedPvcs,
|
|
ownedPvcCount: ownedPvcs.length,
|
|
protectedOwnedPvcs,
|
|
protectedOwnedPvcCount: protectedOwnedPvcs.length,
|
|
estimatedReclaimBytes,
|
|
estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupRunsFinalizeNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
|
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
|
const pvcsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvcs-after.json"), "utf8"));
|
|
const selected = new Set(Array.isArray(plan.selectedPipelineRuns) ? plan.selectedPipelineRuns : []);
|
|
const remainingOwnedPvcs = (Array.isArray(pvcsAfter.items) ? pvcsAfter.items : [])
|
|
.map((pvc) => {
|
|
const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
|
|
return {
|
|
name: pvc?.metadata?.name || null,
|
|
volume: pvc?.spec?.volumeName || null,
|
|
phase: pvc?.status?.phase || null,
|
|
ownerKind: owner?.kind || null,
|
|
owner: owner?.name || null,
|
|
};
|
|
})
|
|
.filter((item) => item.owner && selected.has(item.owner));
|
|
|
|
function tail(name) {
|
|
try {
|
|
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
|
return text.length > 3000 ? text.slice(-3000) : text;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
console.log(JSON.stringify({
|
|
...plan,
|
|
ok: deleteExit === 0,
|
|
deletedPipelineRuns: Array.from(selected),
|
|
deletedPipelineRunCount: selected.size,
|
|
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
|
remainingOwnedPvcs,
|
|
remainingOwnedPvcCount: remainingOwnedPvcs.length,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupReleasedPvsPlanNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const cp = require("node:child_process");
|
|
const tmp = process.env.TMP_DIR;
|
|
const namespace = process.env.NAMESPACE;
|
|
const limit = Number(process.env.LIMIT || 20);
|
|
|
|
function readJson(name) {
|
|
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
|
}
|
|
|
|
function localPathOf(pv) {
|
|
return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
|
|
}
|
|
|
|
function duBytes(hostPath) {
|
|
if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
|
try {
|
|
const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
const bytes = Number(String(out).trim().split(/\s+/)[0]);
|
|
return Number.isFinite(bytes) ? bytes : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!Number.isFinite(bytes)) return null;
|
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
let value = bytes;
|
|
let unit = 0;
|
|
while (value >= 1024 && unit < units.length - 1) {
|
|
value /= 1024;
|
|
unit += 1;
|
|
}
|
|
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
|
}
|
|
|
|
const pvs = readJson("pvs.json");
|
|
const candidates = (Array.isArray(pvs.items) ? pvs.items : [])
|
|
.map((pv) => {
|
|
const hostPath = localPathOf(pv);
|
|
return {
|
|
name: pv?.metadata?.name || "",
|
|
createdAt: pv?.metadata?.creationTimestamp || null,
|
|
phase: pv?.status?.phase || null,
|
|
storageClass: pv?.spec?.storageClassName || null,
|
|
reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
|
|
claimNamespace: pv?.spec?.claimRef?.namespace || null,
|
|
claimName: pv?.spec?.claimRef?.name || null,
|
|
capacity: pv?.spec?.capacity?.storage || null,
|
|
hostPath,
|
|
estimatedBytes: duBytes(hostPath),
|
|
};
|
|
})
|
|
.filter((item) => item.phase === "Released")
|
|
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
|
.filter((item) => item.claimNamespace === namespace)
|
|
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
|
.slice(0, limit);
|
|
|
|
const selectedPersistentVolumes = candidates.map((item) => item.name);
|
|
const estimatedReclaimBytes = candidates.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
planKind: "agentrun-ci-released-local-path-pv-retention",
|
|
generatedAt: new Date().toISOString(),
|
|
namespace,
|
|
criteria: { phase: "Released", storageClass: "local-path", reclaimPolicy: "Delete", claimNamespace: namespace, limit },
|
|
candidates,
|
|
candidateCount: candidates.length,
|
|
selectedPersistentVolumes,
|
|
selectedPersistentVolumeCount: selectedPersistentVolumes.length,
|
|
estimatedReclaimBytes,
|
|
estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function cleanupReleasedPvsFinalizeNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
|
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
|
const pvsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvs-after.json"), "utf8"));
|
|
const selected = new Set(Array.isArray(plan.selectedPersistentVolumes) ? plan.selectedPersistentVolumes : []);
|
|
const remainingPersistentVolumes = (Array.isArray(pvsAfter.items) ? pvsAfter.items : [])
|
|
.filter((pv) => selected.has(pv?.metadata?.name))
|
|
.map((pv) => ({ name: pv?.metadata?.name || null, phase: pv?.status?.phase || null, claimNamespace: pv?.spec?.claimRef?.namespace || null, claimName: pv?.spec?.claimRef?.name || null }));
|
|
|
|
function tail(name) {
|
|
try {
|
|
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
|
return text.length > 3000 ? text.slice(-3000) : text;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
console.log(JSON.stringify({
|
|
...plan,
|
|
ok: deleteExit === 0,
|
|
deletedPersistentVolumes: Array.from(selected),
|
|
deletedPersistentVolumeCount: selected.size,
|
|
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
|
remainingPersistentVolumes,
|
|
remainingPersistentVolumeCount: remainingPersistentVolumes.length,
|
|
}));
|
|
`;
|
|
}
|
|
|
|
function statusScript(pipelineRun: string | null): string {
|
|
const pr = pipelineRun ?? "";
|
|
return [
|
|
"set -eu",
|
|
"printf 'pipelineRun\\tstatus\\treason\\tstart\\tcompletion\\n'",
|
|
pr.length > 0
|
|
? `kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pr)} -o 'jsonpath={.metadata.name}{\"\\t\"}{.status.conditions[0].status}{\"\\t\"}{.status.conditions[0].reason}{\"\\t\"}{.status.startTime}{\"\\t\"}{.status.completionTime}{\"\\n\"}' 2>/dev/null || true`
|
|
: "true",
|
|
"printf 'pipelineRunCondition='",
|
|
pr.length > 0
|
|
? [
|
|
`if kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pr)} -o json >/tmp/agentrun-v01-pipelinerun.json 2>/dev/null; then`,
|
|
"node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const pr = JSON.parse(fs.readFileSync('/tmp/agentrun-v01-pipelinerun.json', 'utf8'));",
|
|
"const condition = pr?.status?.conditions?.[0] || {};",
|
|
"console.log(JSON.stringify({",
|
|
" name: pr?.metadata?.name || null,",
|
|
" status: condition.status || null,",
|
|
" reason: condition.reason || null,",
|
|
" message: condition.message || null,",
|
|
" startTime: pr?.status?.startTime || null,",
|
|
" completionTime: pr?.status?.completionTime || null",
|
|
"}));",
|
|
"NODE",
|
|
"else printf '{}\\n'; fi",
|
|
].join("\n")
|
|
: "printf '{}\\n'",
|
|
"printf 'taskRuns\\n'",
|
|
pr.length > 0
|
|
? `kubectl -n ${ciNamespace} get taskrun -l tekton.dev/pipelineRun=${shQuote(pr)} -o 'custom-columns=NAME:.metadata.name,STATUS:.status.conditions[0].status,REASON:.status.conditions[0].reason,START:.status.startTime,COMPLETION:.status.completionTime' --no-headers 2>/dev/null || true`
|
|
: "true",
|
|
"printf 'taskRunPods\\n'",
|
|
pr.length > 0
|
|
? `kubectl -n ${ciNamespace} get pod -l tekton.dev/pipelineRun=${shQuote(pr)} -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,AGE:.metadata.creationTimestamp' --no-headers 2>/dev/null || true`
|
|
: "true",
|
|
"printf 'recentPipelineRuns\\n'",
|
|
`kubectl -n ${ciNamespace} get pipelinerun --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,STATUS:.status.conditions[0].status,REASON:.status.conditions[0].reason,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null | tail -n 5 || true`,
|
|
"printf 'ciSummary='",
|
|
pr.length > 0
|
|
? [
|
|
`if kubectl -n ${ciNamespace} get taskrun -l tekton.dev/pipelineRun=${shQuote(pr)} -o json >/tmp/agentrun-v01-taskruns.json 2>/dev/null; then`,
|
|
"node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const data = JSON.parse(fs.readFileSync('/tmp/agentrun-v01-taskruns.json', 'utf8'));",
|
|
"const items = Array.isArray(data.items) ? data.items : [];",
|
|
"function task(item) { return item?.metadata?.labels?.['tekton.dev/pipelineTask'] || item?.metadata?.name || null; }",
|
|
"function result(item, name) { return (item?.status?.results || item?.status?.taskResults || []).find((entry) => entry.name === name)?.value ?? null; }",
|
|
"function seconds(start, end) { const s = Date.parse(start || ''); const e = Date.parse(end || ''); return Number.isFinite(s) && Number.isFinite(e) ? Math.round((e - s) / 1000) : null; }",
|
|
"const taskRuns = items.map((item) => ({ name: item?.metadata?.name || null, task: task(item), status: item?.status?.conditions?.[0]?.status || null, reason: item?.status?.conditions?.[0]?.reason || null, seconds: seconds(item?.status?.startTime, item?.status?.completionTime) }));",
|
|
"const plan = items.find((item) => task(item) === 'plan-artifacts');",
|
|
"const image = items.find((item) => task(item) === 'image-publish');",
|
|
"const imageSeconds = taskRuns.find((item) => item.task === 'image-publish')?.seconds ?? null;",
|
|
"console.log(JSON.stringify({",
|
|
" planArtifacts: { summary: plan ? result(plan, 'summary') : null, envIdentity: plan ? result(plan, 'env-identity') : null, buildCount: plan ? Number(result(plan, 'build-count') || 0) : null, reuseCount: plan ? Number(result(plan, 'reuse-count') || 0) : null },",
|
|
" envImage: { status: image ? result(image, 'status') : null, envIdentity: image ? result(image, 'env-identity') : null, image: image ? result(image, 'image') : null, digest: image ? result(image, 'digest') : null, repositoryDigest: image ? result(image, 'repository-digest') : null },",
|
|
" taskRuns,",
|
|
" performance: { imagePublishSeconds: imageSeconds, buildWarning: typeof imageSeconds === 'number' && imageSeconds > 120, criticalWarning: typeof imageSeconds === 'number' && imageSeconds > 180 }",
|
|
"}));",
|
|
"NODE",
|
|
"else printf '{}\\n'; fi",
|
|
].join("\n")
|
|
: "printf '{}\\n'",
|
|
"printf 'argo\\n'",
|
|
`kubectl -n ${argoNamespace} get application ${argoApplication} -o 'jsonpath={.status.sync.revision}{"\\t"}{.status.sync.status}{"\\t"}{.status.health.status}{"\\n"}' 2>/dev/null || true`,
|
|
"printf 'workloads\\n'",
|
|
`kubectl -n ${runtimeNamespace} get deploy,sts -o wide 2>/dev/null || true`,
|
|
"printf 'managerImage\\n'",
|
|
`kubectl -n ${runtimeNamespace} get deploy agentrun-mgr -o 'jsonpath={.spec.template.spec.containers[0].image}{"\\t"}{.spec.template.spec.containers[0].env[?(@.name=="AGENTRUN_SOURCE_COMMIT")].value}{"\\n"}' 2>/dev/null || true`,
|
|
"printf 'managerPods\\n'",
|
|
`kubectl -n ${runtimeNamespace} get pod -l app.kubernetes.io/name=agentrun-mgr -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null || true`,
|
|
"printf 'recentRunnerPods\\n'",
|
|
`kubectl -n ${runtimeNamespace} get pod -l app.kubernetes.io/component=runner --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null | tail -n 8 || true`,
|
|
].join("\n");
|
|
}
|
|
|
|
function refreshScript(): string {
|
|
return [
|
|
"set -eu",
|
|
`kubectl -n ${argoNamespace} annotate application ${argoApplication} argocd.argoproj.io/refresh=hard --overwrite`,
|
|
`kubectl -n ${argoNamespace} get application ${argoApplication} -o 'jsonpath={.status.sync.revision}{"\\t"}{.status.sync.status}{"\\t"}{.status.health.status}{"\\n"}' 2>/dev/null || true`,
|
|
].join("\n");
|
|
}
|
|
|
|
function triggerScript(sourceCommit: string, pipelineRun: string): string {
|
|
return [
|
|
"set -eu",
|
|
"cd /root/agentrun-v01",
|
|
`if kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pipelineRun)} >/dev/null 2>&1; then`,
|
|
` existing_status="$(kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pipelineRun)} -o 'jsonpath={.status.conditions[0].status}:{.status.conditions[0].reason}' 2>/dev/null || true)"`,
|
|
" case \"$existing_status\" in",
|
|
" False:*)",
|
|
" printf 'deleteExisting=%s\\n' \"$existing_status\"",
|
|
` kubectl -n ${ciNamespace} delete pipelinerun ${shQuote(pipelineRun)} --wait=true --timeout=20s`,
|
|
" ;;",
|
|
" *)",
|
|
" printf 'refuseExisting=%s\\n' \"$existing_status\"",
|
|
" printf 'reason=existing-pipelinerun-active-or-succeeded\\n'",
|
|
" exit 20",
|
|
" ;;",
|
|
" esac",
|
|
"fi",
|
|
"kubectl apply -f deploy/templates/tekton/rbac.yaml",
|
|
"kubectl apply -f deploy/templates/tekton/pipeline.yaml",
|
|
"kubectl apply -f deploy/templates/argocd/project.yaml",
|
|
"kubectl apply -f deploy/templates/argocd/application-v01.yaml",
|
|
"cat <<'YAML' | kubectl create -f -",
|
|
"apiVersion: tekton.dev/v1",
|
|
"kind: PipelineRun",
|
|
"metadata:",
|
|
` name: ${pipelineRun}`,
|
|
` namespace: ${ciNamespace}`,
|
|
"spec:",
|
|
" pipelineRef:",
|
|
` name: ${pipelineName}`,
|
|
" taskRunTemplate:",
|
|
" serviceAccountName: agentrun-v01-tekton-runner",
|
|
" podTemplate:",
|
|
" hostNetwork: true",
|
|
" dnsPolicy: ClusterFirstWithHostNet",
|
|
" securityContext:",
|
|
" fsGroup: 1000",
|
|
" params:",
|
|
" - name: revision",
|
|
` value: ${sourceCommit}`,
|
|
" - name: git-read-url",
|
|
` value: ${gitMirrorReadUrl}`,
|
|
" - name: git-write-url",
|
|
` value: ${gitMirrorWriteUrl}`,
|
|
" - name: tools-image",
|
|
` value: ${mirrorToolsImage}`,
|
|
" workspaces:",
|
|
" - name: source",
|
|
" volumeClaimTemplate:",
|
|
" spec:",
|
|
" accessModes: [\"ReadWriteOnce\"]",
|
|
" resources:",
|
|
" requests:",
|
|
" storage: 5Gi",
|
|
" - name: git-ssh",
|
|
" secret:",
|
|
" secretName: agentrun-git-ssh",
|
|
"YAML",
|
|
`printf 'created=${pipelineRun}\\n'`,
|
|
].join("\n");
|
|
}
|
|
|
|
async function gitMirrorStatus(config: UniDeskConfig, options: DisclosureOptions = { full: false, raw: false }): Promise<Record<string, unknown>> {
|
|
const observation = await readGitMirrorStatus(config);
|
|
const summary = observation.summary;
|
|
return {
|
|
ok: observation.ok,
|
|
command: "agentrun v01 git-mirror status",
|
|
namespace: gitMirrorNamespace,
|
|
readUrl: gitMirrorReadUrl,
|
|
writeUrl: gitMirrorWriteUrl,
|
|
summary,
|
|
...(options.raw ? { raw: observation.raw } : {}),
|
|
probe: compactCapture(observation.result, { full: options.full || options.raw, stdoutTailChars: 6000, stderrTailChars: 3000 }),
|
|
disclosure: {
|
|
defaultView: "compact-low-noise",
|
|
full: options.full,
|
|
raw: options.raw,
|
|
rawOmitted: !options.raw,
|
|
probeTailOmitted: !(options.full || options.raw),
|
|
expandWith: "bun scripts/cli.ts agentrun v01 git-mirror status --full",
|
|
rawWith: "bun scripts/cli.ts agentrun v01 git-mirror status --raw",
|
|
},
|
|
next: {
|
|
sync: "bun scripts/cli.ts agentrun v01 git-mirror sync --confirm",
|
|
flush: summary.pendingFlush === true ? "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm" : null,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function readGitMirrorStatus(config: UniDeskConfig): Promise<Record<string, unknown> & { result: SshCaptureResult; raw: string; summary: Record<string, unknown> }> {
|
|
const script = [
|
|
"set +e",
|
|
"printf 'resources\\n'",
|
|
`kubectl -n ${gitMirrorNamespace} get deploy,svc,pvc,cm -l app.kubernetes.io/name=git-mirror -o name 2>/dev/null || true`,
|
|
"printf 'jobs\\n'",
|
|
`kubectl -n ${gitMirrorNamespace} get job -l app.kubernetes.io/name=git-mirror --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime' --no-headers 2>/dev/null | tail -n 8 || true`,
|
|
"printf 'cache\\n'",
|
|
`kubectl exec -n ${gitMirrorNamespace} deploy/git-mirror-http -- sh -lc ${shQuote(gitMirrorCacheProbeScript())}`,
|
|
].join("\n");
|
|
const result = await capture(config, g14K3sRoute, ["script", "--", script]);
|
|
const raw = result.stdout;
|
|
const summary = gitMirrorStatusSummary(raw);
|
|
return {
|
|
ok: result.exitCode === 0 && summary.localV01 !== null,
|
|
namespace: gitMirrorNamespace,
|
|
readUrl: gitMirrorReadUrl,
|
|
writeUrl: gitMirrorWriteUrl,
|
|
raw,
|
|
summary,
|
|
result,
|
|
};
|
|
}
|
|
|
|
async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", options: GitMirrorOptions): Promise<Record<string, unknown>> {
|
|
const jobName = `${action === "sync" ? gitMirrorSyncJobPrefix : gitMirrorFlushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63);
|
|
const manifest = gitMirrorJobManifest(action, jobName);
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const command = `agentrun v01 git-mirror ${action}`;
|
|
if (options.dryRun || !options.confirm) {
|
|
return {
|
|
ok: true,
|
|
command,
|
|
dryRun: true,
|
|
namespace: gitMirrorNamespace,
|
|
jobName,
|
|
manifest,
|
|
next: { confirm: `bun scripts/cli.ts ${command} --confirm` },
|
|
};
|
|
}
|
|
const created = await createGitMirrorJob(config, jobName, manifestB64);
|
|
if (created.exitCode !== 0 || !options.wait) {
|
|
const status = await gitMirrorStatus(config);
|
|
return {
|
|
ok: created.exitCode === 0,
|
|
command,
|
|
dryRun: false,
|
|
mode: options.wait ? "create-failed" : "submitted",
|
|
namespace: gitMirrorNamespace,
|
|
jobName,
|
|
result: compactCapture(created),
|
|
status,
|
|
next: {
|
|
status: "bun scripts/cli.ts agentrun v01 git-mirror status",
|
|
wait: `bun scripts/cli.ts agentrun v01 git-mirror ${action} --confirm --wait`,
|
|
},
|
|
};
|
|
}
|
|
const wait = await waitForGitMirrorJob(config, action, jobName, options.timeoutSeconds);
|
|
const status = await gitMirrorStatus(config);
|
|
return {
|
|
ok: created.exitCode === 0 && wait.ok === true,
|
|
command,
|
|
dryRun: false,
|
|
mode: "waited",
|
|
namespace: gitMirrorNamespace,
|
|
jobName,
|
|
result: compactCapture(created),
|
|
wait,
|
|
status,
|
|
next: {
|
|
status: "bun scripts/cli.ts agentrun v01 git-mirror status",
|
|
flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm" : null,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function createGitMirrorJob(config: UniDeskConfig, jobName: string, manifestB64: string): Promise<SshCaptureResult> {
|
|
const script = [
|
|
"set -eu",
|
|
`job=${shQuote(jobName)}`,
|
|
`manifest_b64=${shQuote(manifestB64)}`,
|
|
"manifest_path=\"/tmp/$job.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
`kubectl delete job -n ${gitMirrorNamespace} "$job" --ignore-not-found=true >/dev/null`,
|
|
"kubectl create -f \"$manifest_path\"",
|
|
`kubectl get job -n ${gitMirrorNamespace} "$job" -o 'jsonpath=created={.metadata.name}{\"\\n\"}'`,
|
|
].join("\n");
|
|
return await capture(config, g14K3sRoute, ["script", "--", script]);
|
|
}
|
|
|
|
async function waitForGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", jobName: string, timeoutSeconds: number): Promise<Record<string, unknown>> {
|
|
const startedAtMs = Date.now();
|
|
let lastProbe: SshCaptureResult | null = null;
|
|
let polls = 0;
|
|
while (Date.now() - startedAtMs <= timeoutSeconds * 1000) {
|
|
polls += 1;
|
|
lastProbe = await capture(config, g14K3sRoute, ["script", "--", gitMirrorJobProbeScript(jobName)]);
|
|
const summary = gitMirrorJobProbeSummary(lastProbe.stdout);
|
|
process.stderr.write(`${JSON.stringify({
|
|
event: `agentrun.git-mirror.${action}.progress`,
|
|
at: new Date().toISOString(),
|
|
stage: "k8s-job",
|
|
status: summary.succeeded ? "succeeded" : summary.failed ? "failed" : "running",
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
})}\n`);
|
|
if (summary.succeeded) {
|
|
return {
|
|
ok: true,
|
|
action,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
summary,
|
|
probe: compactCapture(lastProbe),
|
|
};
|
|
}
|
|
if (summary.failed) {
|
|
return {
|
|
ok: false,
|
|
action,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
degradedReason: "git-mirror-job-failed",
|
|
summary,
|
|
probe: compactCapture(lastProbe),
|
|
};
|
|
}
|
|
await sleep(5_000);
|
|
}
|
|
return {
|
|
ok: false,
|
|
action,
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
degradedReason: "git-mirror-job-timeout",
|
|
probe: lastProbe ? compactCapture(lastProbe) : null,
|
|
};
|
|
}
|
|
|
|
function gitMirrorJobProbeScript(jobName: string): string {
|
|
return [
|
|
"set +e",
|
|
`job=${shQuote(jobName)}`,
|
|
"printf 'jobStatus='",
|
|
`kubectl get job -n ${gitMirrorNamespace} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed} active={.status.active} start={.status.startTime} completion={.status.completionTime}' 2>/dev/null || true`,
|
|
"printf '\\n'",
|
|
"printf 'pods\\n'",
|
|
`kubectl get pod -n ${gitMirrorNamespace} -l job-name="$job" -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,START:.status.startTime' --no-headers 2>/dev/null || true`,
|
|
"printf 'logs\\n'",
|
|
`kubectl logs -n ${gitMirrorNamespace} "job/$job" --tail=120 2>/dev/null || true`,
|
|
].join("\n");
|
|
}
|
|
|
|
function gitMirrorJobProbeSummary(stdout: string): Record<string, unknown> {
|
|
const line = matchLine(stdout, "jobStatus=") ?? "";
|
|
const succeeded = /(?:^|\s)succeeded=1(?:\s|$)/u.test(line);
|
|
const failedRaw = firstMatch(line, /(?:^|\s)failed=([0-9]+)/u);
|
|
const failed = failedRaw !== null && Number(failedRaw) > 0;
|
|
const activeRaw = firstMatch(line, /(?:^|\s)active=([0-9]+)/u);
|
|
return {
|
|
statusLine: line,
|
|
succeeded,
|
|
failed,
|
|
active: activeRaw === null ? null : Number(activeRaw),
|
|
};
|
|
}
|
|
|
|
function gitMirrorJobManifest(action: "sync" | "flush", name: string): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name,
|
|
namespace: gitMirrorNamespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "devops-infra",
|
|
"app.kubernetes.io/component": action === "sync" ? "sync-controller" : "flush-controller",
|
|
"agentrun.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 300,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "devops-infra",
|
|
"app.kubernetes.io/component": action === "sync" ? "sync-controller" : "flush-controller",
|
|
"agentrun.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
volumes: [
|
|
{ name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } },
|
|
{ name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } },
|
|
],
|
|
containers: [{
|
|
name: action,
|
|
image: mirrorToolsImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", action === "sync" ? gitMirrorSyncShellScript() : gitMirrorFlushShellScript()],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function gitMirrorSyncShellScript(): string {
|
|
return [
|
|
"set -eu",
|
|
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"mkdir -p /cache/pikasTech",
|
|
"export GIT_SSH_COMMAND='ssh -i /git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/agentrun-known-hosts'",
|
|
`repo=${shQuote(gitMirrorRepoPath)}`,
|
|
`remote=${shQuote(githubSshUrl)}`,
|
|
"if [ ! -d \"$repo\" ]; then git clone --mirror \"$remote\" \"$repo\"; else git --git-dir=\"$repo\" remote set-url origin \"$remote\"; fi",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
|
"git --git-dir=\"$repo\" config http.receivepack true",
|
|
"git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1:refs/heads/v0.1 +refs/heads/v0.1:refs/mirror-stage/heads/v0.1",
|
|
"if git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1-gitops:refs/mirror-stage/heads/v0.1-gitops; then",
|
|
" if ! git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' >/dev/null 2>&1; then",
|
|
" git --git-dir=\"$repo\" update-ref refs/heads/v0.1-gitops \"$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}')\"",
|
|
" fi",
|
|
"fi",
|
|
"local_v01=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1^{commit}')",
|
|
"github_v01=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1^{commit}')",
|
|
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
|
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
|
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
|
|
"json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }",
|
|
"cat > /cache/agentrun.last-sync.json <<EOF",
|
|
"{\"event\":\"git-mirror-sync\",\"repo\":\"pikasTech/agentrun\",\"status\":\"succeeded\",\"startedAt\":\"$started_at\",\"syncedAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"localV01\":\"$local_v01\",\"githubV01\":\"$github_v01\",\"localGitops\":$(json_ref \"$local_gitops\"),\"githubGitops\":$(json_ref \"$github_gitops\"),\"pendingFlush\":$pending}",
|
|
"EOF",
|
|
"cat /cache/agentrun.last-sync.json",
|
|
].join("\n");
|
|
}
|
|
|
|
function gitMirrorFlushShellScript(): string {
|
|
return [
|
|
"set -eu",
|
|
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"export GIT_SSH_COMMAND='ssh -i /git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/tmp/agentrun-known-hosts'",
|
|
`repo=${shQuote(gitMirrorRepoPath)}`,
|
|
`remote=${shQuote(githubSshUrl)}`,
|
|
"test -d \"$repo\"",
|
|
"git --git-dir=\"$repo\" remote set-url origin \"$remote\"",
|
|
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
|
"if [ -n \"$local_gitops\" ]; then",
|
|
" git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin refs/heads/v0.1-gitops:refs/heads/v0.1-gitops",
|
|
" git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1-gitops:refs/mirror-stage/heads/v0.1-gitops",
|
|
"fi",
|
|
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}' 2>/dev/null || true)",
|
|
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
|
|
"json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }",
|
|
"cat > /cache/agentrun.last-flush.json <<EOF",
|
|
"{\"event\":\"git-mirror-flush\",\"repo\":\"pikasTech/agentrun\",\"status\":\"succeeded\",\"startedAt\":\"$started_at\",\"flushedAt\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"localGitops\":$(json_ref \"$local_gitops\"),\"githubGitops\":$(json_ref \"$github_gitops\"),\"pendingFlush\":$pending}",
|
|
"EOF",
|
|
"cat /cache/agentrun.last-flush.json",
|
|
].join("\n");
|
|
}
|
|
|
|
function gitMirrorCacheProbeScript(): string {
|
|
return [
|
|
"printf 'lastSync='; cat /cache/agentrun.last-sync.json 2>/dev/null || true; printf '\\n'",
|
|
"printf 'lastFlush='; cat /cache/agentrun.last-flush.json 2>/dev/null || true; printf '\\n'",
|
|
`repo=${shQuote(gitMirrorRepoPath)}`,
|
|
"rev() { git --git-dir=\"$repo\" rev-parse --verify \"$1^{commit}\" 2>/dev/null || true; }",
|
|
"local_v01=$(rev refs/heads/v0.1)",
|
|
"github_v01=$(rev refs/mirror-stage/heads/v0.1)",
|
|
"local_gitops=$(rev refs/heads/v0.1-gitops)",
|
|
"github_gitops=$(rev refs/mirror-stage/heads/v0.1-gitops)",
|
|
"exact=false",
|
|
"exact_commit=\"\"",
|
|
"if [ -n \"$local_v01\" ]; then",
|
|
" tmp=$(mktemp -d)",
|
|
" if git init -q \"$tmp\" && git -C \"$tmp\" remote add origin http://127.0.0.1:8080/pikasTech/agentrun.git && git -C \"$tmp\" fetch --depth=1 origin \"$local_v01\" >/tmp/agentrun-exact-fetch.log 2>&1; then",
|
|
" exact_commit=$(git -C \"$tmp\" rev-parse FETCH_HEAD 2>/dev/null || true)",
|
|
" if [ \"$exact_commit\" = \"$local_v01\" ]; then exact=true; fi",
|
|
" fi",
|
|
" rm -rf \"$tmp\"",
|
|
"fi",
|
|
"node -e 'const [localV01,githubV01,localGitops,githubGitops,exact,exactCommit]=process.argv.slice(1); const n=v=>v&&v.length>0?v:null; const pending=Boolean(n(localGitops)&&(!n(githubGitops)||localGitops!==githubGitops)); console.log(\"refs=\"+JSON.stringify({refs:{localV01:n(localV01),githubV01:n(githubV01),localGitops:n(localGitops),githubGitops:n(githubGitops)},pendingFlush:pending,exactFetch:{localV01:exact===\"true\",commit:n(exactCommit)}}));' \"$local_v01\" \"$github_v01\" \"$local_gitops\" \"$github_gitops\" \"$exact\" \"$exact_commit\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function gitMirrorStatusSummary(raw: string): Record<string, unknown> {
|
|
const refs = labeledJson(raw, "refs");
|
|
const lastSync = labeledJson(raw, "lastSync");
|
|
const lastFlush = labeledJson(raw, "lastFlush");
|
|
const refsRecord = record(record(refs).refs);
|
|
const localV01 = stringOrNull(refsRecord.localV01);
|
|
const githubV01 = stringOrNull(refsRecord.githubV01);
|
|
const localGitops = stringOrNull(refsRecord.localGitops);
|
|
const githubGitops = stringOrNull(refsRecord.githubGitops);
|
|
const pendingFlush = typeof record(refs).pendingFlush === "boolean" ? Boolean(record(refs).pendingFlush) : null;
|
|
return {
|
|
localV01,
|
|
githubV01,
|
|
localGitops,
|
|
githubGitops,
|
|
pendingFlush,
|
|
sourceInSync: Boolean(localV01 && githubV01 && localV01 === githubV01),
|
|
gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops),
|
|
githubInSync: Boolean(localV01 && githubV01 && localV01 === githubV01 && (!localGitops || localGitops === githubGitops)),
|
|
flushNeeded: pendingFlush === true,
|
|
flushCommand: pendingFlush === true ? "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm" : null,
|
|
exactFetch: record(refs).exactFetch ?? null,
|
|
lastSync,
|
|
lastFlush,
|
|
};
|
|
}
|
|
|
|
function gitMirrorSyncRequirement(sourceCommit: string, rawStatus: string): { required: boolean; reason: string; localV01: string | null } {
|
|
const summary = gitMirrorStatusSummary(rawStatus);
|
|
const localV01 = stringOrNull(summary.localV01);
|
|
return {
|
|
required: localV01 !== sourceCommit,
|
|
reason: localV01 === sourceCommit ? "local-v01-current" : localV01 === null ? "local-v01-unresolved" : "local-v01-stale",
|
|
localV01,
|
|
};
|
|
}
|
|
|
|
function startAsyncAgentRunJob(name: string, command: string[], note: string): Record<string, unknown> {
|
|
const job = startJob(name, command, note);
|
|
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
|
return {
|
|
ok: true,
|
|
mode: "async-job",
|
|
job,
|
|
statusCommand,
|
|
tailCommand: `tail -f ${job.stdoutFile}`,
|
|
next: {
|
|
status: statusCommand,
|
|
tail: `tail -f ${job.stdoutFile}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
interface AgentRunCliMaterializedFile {
|
|
flag: string;
|
|
source: string;
|
|
remotePath: string;
|
|
bytes: number;
|
|
base64: string;
|
|
}
|
|
|
|
interface PreparedAgentRunCliArgs {
|
|
args: string[];
|
|
materializedFiles: AgentRunCliMaterializedFile[];
|
|
}
|
|
|
|
async function runOfficialAgentRunCli(config: UniDeskConfig, group: "queue" | "sessions", args: string[]): Promise<Record<string, unknown>> {
|
|
const prepared = prepareOfficialAgentRunCliArgs([group, ...args]);
|
|
const command = `agentrun v01 ${prepared.args.join(" ")}`.trim();
|
|
const bridge = agentRunQueueBridgeMetadata(prepared.materializedFiles);
|
|
const script = officialAgentRunCliScript(prepared);
|
|
const result = await capture(config, g14SourceRoute, ["script", "--", script]);
|
|
const payload = captureJsonPayload(result);
|
|
if (result.exitCode === 0 && Object.keys(payload).length > 0) {
|
|
return {
|
|
...payload,
|
|
command,
|
|
bridge,
|
|
};
|
|
}
|
|
if (result.exitCode === 0) {
|
|
return {
|
|
ok: true,
|
|
command,
|
|
bridge,
|
|
stdout: result.stdout.trim(),
|
|
remote: compactCapture(result, { full: true, stdoutTailChars: 4000, stderrTailChars: 2000 }),
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
command,
|
|
degradedReason: "agentrun-cli-bridge-failed",
|
|
bridge,
|
|
agentrun: payload,
|
|
remote: compactCapture(result, { full: true, stdoutTailChars: 8000, stderrTailChars: 4000 }),
|
|
};
|
|
}
|
|
|
|
function prepareOfficialAgentRunCliArgs(args: string[]): PreparedAgentRunCliArgs {
|
|
const fileFlags = new Set(["--json-file", "--prompt-file", "--runner-json-file"]);
|
|
const remoteTmpDir = `/tmp/unidesk-agentrun-cli-${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2)}`;
|
|
const materializedFiles: AgentRunCliMaterializedFile[] = [];
|
|
const prepared: string[] = [];
|
|
let stdinBuffer: Buffer | null = null;
|
|
|
|
const stdin = (): Buffer => {
|
|
if (stdinBuffer === null) stdinBuffer = readFileSync(0);
|
|
return stdinBuffer;
|
|
};
|
|
|
|
const materialize = (flag: string, source: string, buffer: Buffer): string => {
|
|
const remotePath = `${remoteTmpDir}/input-${materializedFiles.length}`;
|
|
materializedFiles.push({
|
|
flag,
|
|
source,
|
|
remotePath,
|
|
bytes: buffer.length,
|
|
base64: buffer.toString("base64"),
|
|
});
|
|
return remotePath;
|
|
};
|
|
|
|
for (let i = 0; i < args.length; i += 1) {
|
|
const arg = args[i] ?? "";
|
|
const equalsFlag = Array.from(fileFlags).find((flag) => arg.startsWith(`${flag}=`));
|
|
if (equalsFlag !== undefined) {
|
|
const source = arg.slice(equalsFlag.length + 1);
|
|
if (source.length === 0) throw new Error(`${equalsFlag} requires a path`);
|
|
const buffer = source === "-" ? stdin() : readFileSync(source);
|
|
prepared.push(equalsFlag, materialize(equalsFlag, source === "-" ? "stdin" : source, buffer));
|
|
continue;
|
|
}
|
|
if (fileFlags.has(arg)) {
|
|
const source = args[i + 1];
|
|
if (source === undefined || source.length === 0) throw new Error(`${arg} requires a path`);
|
|
const buffer = source === "-" ? stdin() : readFileSync(source);
|
|
prepared.push(arg, materialize(arg, source === "-" ? "stdin" : source, buffer));
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if (arg === "--prompt-stdin" || arg === "--stdin") {
|
|
prepared.push("--prompt-file", materialize("--prompt-stdin", "stdin", stdin()));
|
|
continue;
|
|
}
|
|
prepared.push(arg);
|
|
}
|
|
|
|
return { args: prepared, materializedFiles };
|
|
}
|
|
|
|
function officialAgentRunCliScript(prepared: PreparedAgentRunCliArgs): string {
|
|
const setup = prepared.materializedFiles.length === 0
|
|
? []
|
|
: [
|
|
`tmp_dir=${shQuote(parentDir(prepared.materializedFiles[0]?.remotePath ?? "/tmp/unidesk-agentrun-cli"))}`,
|
|
"mkdir -p \"$tmp_dir\"",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
...prepared.materializedFiles.map((file) => `printf %s ${shQuote(file.base64)} | base64 -d > ${shQuote(file.remotePath)}`),
|
|
];
|
|
return [
|
|
"set -euo pipefail",
|
|
...setup,
|
|
"cd /root/agentrun-v01",
|
|
`./scripts/agentrun --manager-url auto ${prepared.args.map(shQuote).join(" ")}`,
|
|
].join("\n");
|
|
}
|
|
|
|
function parentDir(pathValue: string): string {
|
|
const index = pathValue.lastIndexOf("/");
|
|
return index > 0 ? pathValue.slice(0, index) : ".";
|
|
}
|
|
|
|
function agentRunQueueBridgeMetadata(materializedFiles: AgentRunCliMaterializedFile[]): Record<string, unknown> {
|
|
return {
|
|
route: g14SourceRoute,
|
|
sourceWorktree: "/root/agentrun-v01",
|
|
sourceBranch,
|
|
managerUrl: "auto",
|
|
officialCli: "./scripts/agentrun",
|
|
mode: "direct-official-cli",
|
|
compatibility: "no-code-queue-adapter-no-double-write",
|
|
fileFlagsMaterialized: materializedFiles.map((file) => ({
|
|
flag: file.flag,
|
|
source: file.source,
|
|
remotePath: file.remotePath,
|
|
bytes: file.bytes,
|
|
})),
|
|
};
|
|
}
|
|
|
|
async function capture(config: UniDeskConfig, target: string, args: string[]): Promise<SshCaptureResult> {
|
|
return await runSshCommandCapture(config, target, args);
|
|
}
|
|
|
|
function compactCapture(result: SshCaptureResult, options: { full?: boolean; stdoutTailChars?: number; stderrTailChars?: number } = {}): Record<string, unknown> {
|
|
const stdoutTailChars = options.stdoutTailChars ?? 8000;
|
|
const stderrTailChars = options.stderrTailChars ?? 4000;
|
|
const full = options.full ?? true;
|
|
const payload: Record<string, unknown> = {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
stdoutTailOmitted: !full && result.exitCode === 0,
|
|
stderrTailOmitted: !full && result.exitCode === 0,
|
|
};
|
|
if (full || result.exitCode !== 0) {
|
|
payload.stdoutTail = tail(result.stdout, stdoutTailChars);
|
|
payload.stderrTail = tail(result.stderr, stderrTailChars);
|
|
payload.stdoutTruncated = result.stdout.length > stdoutTailChars;
|
|
payload.stderrTruncated = result.stderr.length > stderrTailChars;
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
async function timedStatusStage<T>(stage: string, action: () => Promise<T>): Promise<TimedValue<T>> {
|
|
const startedAtMs = Date.now();
|
|
progressEvent("agentrun.control-plane.status.progress", { stage, status: "started" });
|
|
try {
|
|
const value = await action();
|
|
const exitCode = isCaptureResult(value) ? value.exitCode : typeof record(value).ok === "boolean" && record(value).ok !== true ? 1 : 0;
|
|
progressEvent("agentrun.control-plane.status.progress", {
|
|
stage,
|
|
status: exitCode === 0 ? "succeeded" : "failed",
|
|
exitCode,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
});
|
|
return { value, elapsedMs: Date.now() - startedAtMs };
|
|
} catch (error) {
|
|
progressEvent("agentrun.control-plane.status.progress", {
|
|
stage,
|
|
status: "failed",
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function progressEvent(event: string, payload: Record<string, unknown>): void {
|
|
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...payload })}\n`);
|
|
}
|
|
|
|
function isCaptureResult(value: unknown): value is SshCaptureResult {
|
|
return typeof value === "object" && value !== null && "exitCode" in value && typeof (value as { exitCode?: unknown }).exitCode === "number";
|
|
}
|
|
|
|
function pipelineRunName(sourceCommit: string): string {
|
|
return `agentrun-v01-ci-${sourceCommit.slice(0, 12)}`;
|
|
}
|
|
|
|
function matchLine(text: string, prefix: string): string | null {
|
|
const line = text.split(/\r?\n/u).find((item) => item.startsWith(prefix));
|
|
return line ? line.slice(prefix.length).trim() || null : null;
|
|
}
|
|
|
|
function firstMatch(text: string, pattern: RegExp): string | null {
|
|
return pattern.exec(text)?.[1] ?? null;
|
|
}
|
|
|
|
function labeledJson(text: string, label: string): Record<string, unknown> {
|
|
const prefix = `${label}=`;
|
|
const line = text.split(/\r?\n/u).find((item) => item.startsWith(prefix));
|
|
if (!line) return {};
|
|
try {
|
|
return record(JSON.parse(line.slice(prefix.length)));
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function captureJsonPayload(result: SshCaptureResult): Record<string, unknown> {
|
|
const trimmed = result.stdout.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(trimmed) as unknown);
|
|
} catch {
|
|
const lastJsonLine = trimmed.split(/\r?\n/u).reverse().find((line) => line.trim().startsWith("{") && line.trim().endsWith("}"));
|
|
if (lastJsonLine === undefined) return {};
|
|
try {
|
|
return record(JSON.parse(lastJsonLine) as unknown);
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function parseArgoStatus(text: string): { revision: string | null; syncStatus: string | null; healthStatus: string | null } {
|
|
const lines = text.split(/\r?\n/u);
|
|
const index = lines.findIndex((line) => line.trim() === "argo");
|
|
if (index < 0) return { revision: null, syncStatus: null, healthStatus: null };
|
|
const [revision, syncStatus, healthStatus] = (lines[index + 1] ?? "").split("\t");
|
|
return {
|
|
revision: revision?.trim() || null,
|
|
syncStatus: syncStatus?.trim() || null,
|
|
healthStatus: healthStatus?.trim() || null,
|
|
};
|
|
}
|
|
|
|
function parseManagerImage(text: string): { image: string | null; sourceCommit: string | null } {
|
|
const lines = text.split(/\r?\n/u);
|
|
const index = lines.findIndex((line) => line.trim() === "managerImage");
|
|
if (index < 0) return { image: null, sourceCommit: null };
|
|
const [image, sourceCommit] = (lines[index + 1] ?? "").split("\t");
|
|
return {
|
|
image: image?.trim() || null,
|
|
sourceCommit: sourceCommit?.trim() || null,
|
|
};
|
|
}
|
|
|
|
function isGitSha(value: string): boolean {
|
|
return /^[0-9a-f]{40}$/u.test(value);
|
|
}
|
|
|
|
function tail(text: string, maxChars: number): string {
|
|
return text.length > maxChars ? text.slice(-maxChars) : text;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function shQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, "'\\''")}'`;
|
|
}
|
|
|
|
function unsupported(args: string[]): Record<string, unknown> {
|
|
return {
|
|
ok: false,
|
|
command: `agentrun ${args.join(" ")}`.trim(),
|
|
degradedReason: "unsupported-command",
|
|
message: "supported commands: agentrun v01 queue submit|list|show|stats|commander|read|cancel|dispatch|refresh; agentrun v01 sessions ps|show|turn|steer|cancel|output|trace|read; agentrun v01 control-plane status|trigger-current|refresh; agentrun v01 git-mirror status|sync|flush",
|
|
};
|
|
}
|