diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 67660dad..6b237ae1 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -127,7 +127,7 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str if (action === "restart") return await restartYamlLane(config, parseLaneConfirmOptions(actionArgs)); if (action === "expose") return await exposeAgentRun(config, parseConfirmOptions(actionArgs)); if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(actionArgs)); - if (action === "refresh") return await refresh(config, parseConfirmOptions(actionArgs)); + if (action === "refresh") return await refresh(config, parseRefreshOptions(actionArgs)); if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(actionArgs)); if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs)); } @@ -1673,6 +1673,11 @@ interface LaneConfirmOptions extends ConfirmOptions { lane: string | null; } +interface RefreshOptions extends ConfirmOptions { + node: string | null; + lane: string | null; +} + interface GitMirrorOptions extends ConfirmOptions { timeoutSeconds: number; wait: boolean; @@ -1869,6 +1874,32 @@ function parseLaneConfirmOptions(args: string[]): LaneConfirmOptions { return { ...base, node, lane }; } +function parseRefreshOptions(args: string[]): RefreshOptions { + const base = parseConfirmOptions(args); + let node: string | null = null; + let lane: string | null = null; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--confirm" || arg === "--dry-run") continue; + if (arg === "--node") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); + node = value; + index += 1; + continue; + } + if (arg === "--lane") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); + lane = value; + index += 1; + continue; + } + throw new Error(`unsupported refresh option: ${arg}`); + } + return { ...base, node, lane }; +} + function parseConfirmOptions(args: string[]): ConfirmOptions { if (args.includes("--confirm") && args.includes("--dry-run")) throw new Error("accepts only one of --confirm or --dry-run"); return { @@ -2181,20 +2212,26 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["script", "--", yamlLaneSourceStatusScript(spec)])); const sourcePayload = captureJsonPayload(sourceProbe.value); const sourceCommit = options.sourceCommit - ?? (options.pipelineRun !== null ? null : stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead)); - const pipelineRun = options.pipelineRun ?? (sourceCommit ? agentRunPipelineRunName(spec, sourceCommit) : null); + ?? stringOrNull(sourcePayload.remoteBranchCommit) + ?? stringOrNull(sourcePayload.localHead); + const pipelineRunName = options.pipelineRun ?? (sourceCommit ? agentRunPipelineRunName(spec, sourceCommit) : null); const [runtimeProbe, mirrorProbe] = await Promise.all([ - timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneRuntimeStatusScript(spec, pipelineRun)])), + timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])), timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneGitMirrorStatusScript(spec)])), ]); const runtimePayload = captureJsonPayload(runtimeProbe.value); const mirrorPayload = captureJsonPayload(mirrorProbe.value); const pipeline = record(runtimePayload.pipeline); + const pipelineRunStatus = record(runtimePayload.pipelineRun); const argo = record(runtimePayload.argo); const manager = record(runtimePayload.manager); const database = record(runtimePayload.database); const secrets = record(runtimePayload.secrets); const localPostgres = record(runtimePayload.localPostgres); + const expectedGitopsRevision = stringOrNull(mirrorPayload.gitopsCommit); + const managerSourceMatchesExpected = Boolean(sourceCommit && manager.sourceCommit === sourceCommit); + const argoSyncedToGitops = Boolean(expectedGitopsRevision && argo.revision === expectedGitopsRevision); + const pipelineSucceeded = pipelineRunStatus.status === "True"; const blockers = [ ...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]), ...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]), @@ -2204,10 +2241,15 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar ...(mirrorPayload.cachePvcExists === true ? [] : ["git-mirror-cache-pvc-missing"]), ...(runtimePayload.ciNamespaceExists === true ? [] : ["ci-namespace-missing"]), ...(pipeline.exists === true ? [] : ["pipeline-missing"]), + ...(pipelineRunName !== null && pipelineRunStatus.exists !== true ? ["pipelinerun-missing"] : []), + ...(pipelineRunStatus.exists !== true || pipelineRunStatus.status === undefined || pipelineRunStatus.status === null || pipelineRunStatus.status === "True" ? [] : ["pipelinerun-not-succeeded"]), ...(runtimePayload.serviceAccountExists === true ? [] : ["ci-serviceaccount-missing"]), ...(argo.exists === true ? [] : ["argo-application-missing"]), + ...(mirrorPayload.readReady === true && expectedGitopsRevision === null ? ["gitops-revision-unresolved"] : []), + ...(argo.exists !== true || expectedGitopsRevision === null || argoSyncedToGitops ? [] : ["argo-revision-stale"]), ...(runtimePayload.runtimeNamespaceExists === true ? [] : ["runtime-namespace-missing"]), ...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]), + ...(manager.deploymentExists !== true || sourceCommit === null || managerSourceMatchesExpected ? [] : ["manager-source-stale"]), ...(manager.serviceExists === true ? [] : ["manager-service-missing"]), ...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []), ...(secrets.ready === true ? [] : ["lane-secret-missing"]), @@ -2220,10 +2262,18 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar configPath: target.configPath, target: agentRunLaneSummary(spec), summary: { - aligned: blockers.length === 0, + aligned: blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected, blockers, sourceCommit, - expectedPipelineRun: pipelineRun, + expectedPipelineRun: pipelineRunName, + expectedGitopsRevision, + runtimeAlignment: { + pipelineSucceeded, + argoRevision: stringOrNull(argo.revision), + argoSyncedToGitops, + managerSourceCommit: stringOrNull(manager.sourceCommit), + managerSourceMatchesExpected, + }, source: { workspaceExists: sourcePayload.workspaceExists ?? false, workspaceClean: sourcePayload.workspaceClean ?? null, @@ -2241,7 +2291,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar namespaceExists: runtimePayload.ciNamespaceExists ?? false, serviceAccountExists: runtimePayload.serviceAccountExists ?? false, pipeline, - pipelineRun: record(runtimePayload.pipelineRun), + pipelineRun: pipelineRunStatus, }, argo, runtime: { @@ -2877,7 +2927,8 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen }; } -async function refresh(config: UniDeskConfig, options: ConfirmOptions): Promise> { +async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise> { + if (options.node !== null || options.lane !== null) return await refreshYamlLane(config, options); const source = await capture(config, g14SourceRoute, ["script", "--", [ "cd /root/agentrun-v01", "printf 'gitopsLatest='", @@ -2912,6 +2963,53 @@ async function refresh(config: UniDeskConfig, options: ConfirmOptions): Promise< }; } +async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise> { + const { configPath, spec } = resolveAgentRunLaneTarget(options); + const plan = { + node: spec.nodeId, + lane: spec.lane, + version: spec.version, + argoNamespace: spec.gitops.argoNamespace, + argoApplication: spec.gitops.argoApplication, + gitopsBranch: spec.gitops.branch, + mutation: "argocd-hard-refresh", + valuesPrinted: false, + }; + if (options.dryRun || !options.confirm) { + return { + ok: true, + command: "agentrun control-plane refresh", + mode: "dry-run", + mutation: false, + configPath, + target: agentRunLaneSummary(spec), + plan, + next: { + confirm: `bun scripts/cli.ts agentrun control-plane refresh --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; + } + const refreshed = await capture(config, spec.nodeKubeRoute, ["script", "--", refreshYamlLaneScript(spec)]); + const payload = captureJsonPayload(refreshed); + return { + ok: refreshed.exitCode === 0 && payload.ok !== false, + command: "agentrun control-plane refresh", + mode: "confirmed-yaml-lane", + mutation: true, + configPath, + target: agentRunLaneSummary(spec), + plan, + result: payload, + refreshed: compactCapture(refreshed, { full: refreshed.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), + next: { + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; +} + async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise> { const result = await capture(config, g14K3sRoute, ["script", "--", cleanupRunsScript(options)]); const payload = captureJsonPayload(result); @@ -3727,9 +3825,13 @@ function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", `namespace=${shQuote(spec.gitMirror.namespace)}`, + `read_deployment=${shQuote(spec.gitMirror.readDeployment)}`, `read_service=${shQuote(spec.gitMirror.readService)}`, `write_service=${shQuote(spec.gitMirror.writeService)}`, `cache_pvc=${shQuote(spec.gitMirror.cachePvc)}`, + `repository=${shQuote(spec.source.repository)}`, + `source_branch=${shQuote(spec.source.branch)}`, + `gitops_branch=${shQuote(spec.gitops.branch)}`, `repositories_json=${shQuote(JSON.stringify(spec.gitMirror.repositories))}`, "kubectl -n \"$namespace\" get svc \"$read_service\" -o json >/tmp/agentrun-gitmirror-read.json 2>/dev/null", "read_exit=$?", @@ -3738,10 +3840,17 @@ function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { "kubectl -n \"$namespace\" get pvc \"$cache_pvc\" -o json >/tmp/agentrun-gitmirror-cache.json 2>/dev/null", "cache_exit=$?", "kubectl -n \"$namespace\" get deploy,svc,pvc -o name > /tmp/agentrun-gitmirror-names.txt 2>/dev/null", - "NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" REPOSITORIES_JSON=\"$repositories_json\" node <<'NODE'", + "repo_path=\"/cache/${repository}.git\"", + "rm -f /tmp/agentrun-gitmirror-refs.txt", + "if [ \"$read_exit\" -eq 0 ]; then", + " kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; gitops_branch=\"$3\"; printf \"sourceCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$source_branch\" 2>/dev/null || true; printf \"gitopsCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$gitops_branch\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" \"$gitops_branch\" > /tmp/agentrun-gitmirror-refs.txt 2>/dev/null", + "fi", + "NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" REPOSITORY=\"$repository\" SOURCE_BRANCH=\"$source_branch\" GITOPS_BRANCH=\"$gitops_branch\" REPOSITORIES_JSON=\"$repositories_json\" node <<'NODE'", "const fs = require('node:fs');", "const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');", "let names = ''; try { names = fs.readFileSync('/tmp/agentrun-gitmirror-names.txt', 'utf8'); } catch {}", + "let refs = ''; try { refs = fs.readFileSync('/tmp/agentrun-gitmirror-refs.txt', 'utf8'); } catch {}", + "const refValue = (key) => refs.split(/\\r?\\n/).find((line) => line.startsWith(`${key}=`))?.slice(key.length + 1).trim() || null;", "const readReady = process.env.READ_EXIT === '0';", "const writeReady = process.env.WRITE_EXIT === '0';", "const cachePvcExists = process.env.CACHE_EXIT === '0';", @@ -3752,6 +3861,11 @@ function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { " writeReady,", " cachePvcExists,", " resources: names.split(/\\r?\\n/).filter(Boolean).slice(0, 40),", + " repository: process.env.REPOSITORY,", + " sourceBranch: process.env.SOURCE_BRANCH,", + " gitopsBranch: process.env.GITOPS_BRANCH,", + " sourceCommit: refValue('sourceCommit'),", + " gitopsCommit: refValue('gitopsCommit'),", " repositories,", " valuesPrinted: false", "}));", @@ -4412,6 +4526,31 @@ function refreshScript(): string { ].join("\n"); } +function refreshYamlLaneScript(spec: AgentRunLaneSpec): string { + return [ + "set -eu", + `namespace=${shQuote(spec.gitops.argoNamespace)}`, + `application=${shQuote(spec.gitops.argoApplication)}`, + "kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/agentrun-refresh-output.txt", + "kubectl -n \"$namespace\" get application \"$application\" -o json >/tmp/agentrun-refresh-app.json", + "NAMESPACE=\"$namespace\" APPLICATION=\"$application\" node <<'NODE'", + "const fs = require('node:fs');", + "let app = {};", + "try { app = JSON.parse(fs.readFileSync('/tmp/agentrun-refresh-app.json', 'utf8')); } catch {}", + "console.log(JSON.stringify({", + " ok: true,", + " namespace: process.env.NAMESPACE,", + " application: process.env.APPLICATION,", + " revision: app.status?.sync?.revision ?? null,", + " syncStatus: app.status?.sync?.status ?? null,", + " healthStatus: app.status?.health?.status ?? null,", + " observedAt: new Date().toISOString(),", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); +} + function triggerScript(sourceCommit: string, pipelineRun: string): string { return [ "set -eu",