diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 4fda85c9..35cf1c5b 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -26,37 +26,6 @@ import { type AgentRunArtifactService, } from "./agentrun-manifests"; -const g14SourceRoute = "G14:/root/agentrun-v01"; -const g14K3sRoute = "G14:k3s"; -const sourceBranch = "v0.1"; -const runtimeNamespace = "agentrun-v01"; -const ciNamespace = "agentrun-ci"; -const pipelineName = "agentrun-v01-ci-image-publish"; -const argoNamespace = "argocd"; -const argoApplication = "agentrun-g14-v01"; -const gitopsBranch = "v0.1-gitops"; -const gitMirrorNamespace = "devops-infra"; -const gitMirrorReadBaseUrl = "http://git-mirror-http.devops-infra.svc.cluster.local"; -const gitMirrorWriteBaseUrl = "http://git-mirror-write.devops-infra.svc.cluster.local"; -const gitMirrorReadUrl = `${gitMirrorReadBaseUrl}/pikasTech/agentrun.git`; -const gitMirrorWriteUrl = `${gitMirrorWriteBaseUrl}/pikasTech/agentrun.git`; -const gitMirrorSyncJobPrefix = "git-mirror-agentrun-sync-manual"; -const gitMirrorFlushJobPrefix = "git-mirror-agentrun-flush-manual"; -const mirrorToolsImage = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1"; - -type GitMirrorRepositorySpec = { - key: string; - repository: string; - sourceBranch: string; - gitopsBranch?: string; -}; - -const gitMirrorRepositories: readonly GitMirrorRepositorySpec[] = [ - { key: "agentrun", repository: "pikasTech/agentrun", sourceBranch, gitopsBranch }, - { key: "unidesk", repository: "pikasTech/unidesk", sourceBranch: "master" }, - { key: "agent_skills", repository: "pikasTech/agent_skills", sourceBranch: "master" }, -]; - export function agentRunHelp(): unknown { return { command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|explain", @@ -87,7 +56,7 @@ export function agentRunHelp(): unknown { "bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane status", "bun scripts/cli.ts agentrun control-plane status --full", - "bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", + "bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-vNN-ci-", "bun scripts/cli.ts agentrun control-plane status --source-commit ", "bun scripts/cli.ts agentrun control-plane expose --dry-run", "bun scripts/cli.ts agentrun control-plane expose --confirm", @@ -133,10 +102,17 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs)); } if (group === "git-mirror") { - if (action === "status") return await gitMirrorStatus(config, parseDisclosureOptions(actionArgs)); + if (action === "status") return await gitMirrorStatus(config, parseGitMirrorStatusOptions(actionArgs)); if (action === "sync" || action === "flush") { const options = parseGitMirrorOptions(actionArgs); - if (options.confirm && !options.wait) return startAsyncAgentRunJob(`agentrun_v01_git_mirror_${action}`, ["bun", "scripts/cli.ts", "agentrun", "git-mirror", action, "--confirm", "--wait", "--timeout-seconds", String(options.timeoutSeconds)], `Run AgentRun v0.1 git mirror ${action} on G14`); + const { spec } = resolveAgentRunLaneTarget(options); + if (options.confirm && !options.wait) { + return startAsyncAgentRunJob( + `agentrun_${spec.lane}_git_mirror_${action}`, + ["bun", "scripts/cli.ts", "agentrun", "git-mirror", action, "--node", spec.nodeId, "--lane", spec.lane, "--confirm", "--wait", "--timeout-seconds", String(options.timeoutSeconds)], + `Run AgentRun ${spec.version} git mirror ${action} on ${spec.nodeId}`, + ); + } return await runGitMirrorJob(config, action, options); } } @@ -258,7 +234,7 @@ function agentRunHelpText(args: string[]): string { " bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane status", - " bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", + " bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-vNN-ci-", " bun scripts/cli.ts agentrun control-plane expose --dry-run", " bun scripts/cli.ts agentrun control-plane trigger-current --dry-run", " bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run", @@ -1679,18 +1655,29 @@ interface RefreshOptions extends ConfirmOptions { lane: string | null; } +interface GitMirrorStatusOptions extends DisclosureOptions { + node: string | null; + lane: string | null; +} + interface GitMirrorOptions extends ConfirmOptions { + node: string | null; + lane: string | null; timeoutSeconds: number; wait: boolean; } interface CleanupRunsOptions extends ConfirmOptions { + node: string | null; + lane: string | null; minAgeMinutes: number; limit: number; timeoutSeconds: number; } interface CleanupReleasedPvOptions extends ConfirmOptions { + node: string | null; + lane: string | null; limit: number; timeoutSeconds: number; } @@ -1728,6 +1715,17 @@ function parseDisclosureOptions(args: string[]): DisclosureOptions { return { full: raw || args.includes("--full"), raw }; } +function parseGitMirrorStatusOptions(args: string[]): GitMirrorStatusOptions { + validateOptions(args, new Set(["--full", "--raw"]), new Set(["--node", "--lane"])); + const raw = args.includes("--raw"); + return { + full: raw || args.includes("--full"), + raw, + node: optionValue(args, "--node") ?? null, + lane: optionValue(args, "--lane") ?? null, + }; +} + function parseStatusOptions(args: string[]): StatusOptions { let node: string | null = null; let lane: string | null = null; @@ -1910,18 +1908,27 @@ function parseConfirmOptions(args: string[]): ConfirmOptions { } function parseGitMirrorOptions(args: string[]): GitMirrorOptions { + validateOptions(args, new Set(["--confirm", "--dry-run", "--wait"]), new Set(["--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); const timeoutIndex = args.indexOf("--timeout-seconds"); const timeoutSeconds = timeoutIndex >= 0 ? Number(args[timeoutIndex + 1]) : 300; if (!Number.isFinite(timeoutSeconds) || timeoutSeconds < 30) throw new Error("--timeout-seconds must be a number >= 30"); - return { ...base, timeoutSeconds, wait: args.includes("--wait") }; + return { + ...base, + node: optionValue(args, "--node") ?? null, + lane: optionValue(args, "--lane") ?? null, + timeoutSeconds, + wait: args.includes("--wait"), + }; } function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions { - validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds"])); + validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); return { ...base, + node: optionValue(args, "--node") ?? null, + lane: optionValue(args, "--lane") ?? null, minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(args, "--limit", 20, 500), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600), @@ -1929,10 +1936,12 @@ function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions { } function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPvOptions { - validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds"])); + validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); return { ...base, + node: optionValue(args, "--node") ?? null, + lane: optionValue(args, "--lane") ?? null, limit: positiveIntegerOption(args, "--limit", 20, 500), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), }; @@ -2055,157 +2064,7 @@ async function controlPlaneApply(config: UniDeskConfig, options: LaneConfirmOpti } async function status(config: UniDeskConfig, options: StatusOptions): Promise> { - if (options.node !== null || options.lane !== null) { - return await statusYamlLane(config, options, resolveAgentRunLaneTarget(options)); - } - const sourceProbe = await timedStatusStage("source", () => capture(config, g14SourceRoute, ["script", "--", [ - "cd /root/agentrun-v01", - "git fetch origin v0.1 >/dev/null 2>&1 || true", - "printf 'sourceCommit='", - "git rev-parse HEAD", - "printf 'originV01='", - "git rev-parse origin/v0.1 2>/dev/null || true", - "printf 'gitopsLatest='", - `git ls-remote origin ${gitopsBranch} 2>/dev/null | awk '{print $1}' || true`, - "git status --short --branch", - ].join("\n")])); - const source = sourceProbe.value; - const localSourceCommit = matchLine(source.stdout, "sourceCommit="); - const originSourceCommit = matchLine(source.stdout, "originV01="); - const latestSourceCommit = isGitSha(originSourceCommit ?? "") ? originSourceCommit : localSourceCommit; - const gitopsLatest = matchLine(source.stdout, "gitopsLatest="); - let sourceCommit = options.sourceCommit ?? (options.pipelineRun !== null ? null : latestSourceCommit); - let sourceCommitSource = options.sourceCommit !== null - ? "option" - : options.pipelineRun !== null - ? "pipeline-run-param" - : sourceCommit === originSourceCommit - ? "origin/v0.1" - : "local-head"; - const pipelineRun = options.pipelineRun ?? (sourceCommit ? pipelineRunName(sourceCommit) : null); - const [runtimeProbe, mirrorProbe] = await Promise.all([ - timedStatusStage("runtime", () => capture(config, g14K3sRoute, ["script", "--", statusScript(pipelineRun)])), - timedStatusStage("git-mirror", () => readGitMirrorStatus(config)), - ]); - const k3s = runtimeProbe.value; - const mirror = mirrorProbe.value; - const argo = parseArgoStatus(k3s.stdout); - const ciSummary = labeledJson(k3s.stdout, "ciSummary"); - const pipelineRunCondition = labeledJson(k3s.stdout, "pipelineRunCondition"); - const managerImage = parseManagerImage(k3s.stdout); - const mirrorSummary = mirror.summary; - const pipelineRunSourceCommit = stringOrNull(pipelineRunCondition.sourceCommit); - if (options.pipelineRun !== null && sourceCommit === null && pipelineRunSourceCommit !== null && isGitSha(pipelineRunSourceCommit)) { - sourceCommit = pipelineRunSourceCommit; - sourceCommitSource = "pipeline-run-param"; - } - const target = { - mode: options.targetMode, - sourceCommit, - sourceCommitSource, - pipelineRun, - pipelineRunSourceCommit, - latestSourceCommit, - latestPipelineRun: latestSourceCommit ? pipelineRunName(latestSourceCommit) : null, - isLatestSource: Boolean(sourceCommit && latestSourceCommit && sourceCommit === latestSourceCommit), - sourceMatchesPipelineRun: sourceCommit === null || pipelineRunSourceCommit === null ? null : sourceCommit === pipelineRunSourceCommit, - }; - const runtimeAlignment = { - localHeadMatchesOrigin: Boolean(localSourceCommit && originSourceCommit && localSourceCommit === originSourceCommit), - argoRevision: argo.revision, - argoSyncStatus: argo.syncStatus, - argoHealthStatus: argo.healthStatus, - syncedToGitopsLatest: Boolean(gitopsLatest && argo.revision === gitopsLatest), - managerSourceCommit: managerImage.sourceCommit, - managerSourceMatchesExpected: Boolean(sourceCommit && managerImage.sourceCommit === sourceCommit), - }; - const targetValidation = buildAgentRunTargetValidation({ - pipelineRun, - pipelineRunCondition, - sourceCommit, - targetIsLatestSource: target.isLatestSource, - managerSourceMatchesExpected: runtimeAlignment.managerSourceMatchesExpected, - }); - const summary = { - target, - sourceCommit, - expectedPipelineRun: pipelineRun, - pipelineRun: { - status: pipelineRunCondition.status ?? null, - reason: pipelineRunCondition.reason ?? null, - completionTime: pipelineRunCondition.completionTime ?? null, - }, - targetValidation, - argo, - managerImage, - gitMirror: { - localV01: mirrorSummary.localV01 ?? null, - githubV01: mirrorSummary.githubV01 ?? null, - localGitops: mirrorSummary.localGitops ?? null, - githubGitops: mirrorSummary.githubGitops ?? null, - pendingFlush: mirrorSummary.pendingFlush ?? null, - sourceInSync: mirrorSummary.sourceInSync ?? null, - gitopsInSync: mirrorSummary.gitopsInSync ?? null, - githubInSync: mirrorSummary.githubInSync ?? null, - }, - aligned: pipelineRunCondition.status === "True" && - runtimeAlignment.localHeadMatchesOrigin === true && - runtimeAlignment.syncedToGitopsLatest === true && - runtimeAlignment.managerSourceMatchesExpected === true && - mirrorSummary.githubInSync === true && - mirrorSummary.pendingFlush === false, - }; - return { - ok: source.exitCode === 0 && k3s.exitCode === 0 && mirror.ok === true, - command: "agentrun control-plane status", - lane: "v0.1", - summary, - target, - sourceCommit, - sourceCommitSource, - latestSourceCommit, - localSourceCommit, - originSourceCommit, - gitopsLatest, - expectedPipelineRun: pipelineRun, - timings: { - sourceMs: sourceProbe.elapsedMs, - runtimeMs: runtimeProbe.elapsedMs, - gitMirrorMs: mirrorProbe.elapsedMs, - totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs), - }, - source: compactCapture(source, { full: options.full || options.raw, stdoutTailChars: 3000, stderrTailChars: 2000 }), - runtime: compactCapture(k3s, { full: options.full || options.raw, stdoutTailChars: 8000, stderrTailChars: 4000 }), - pipelineRunCondition, - ciSummary, - gitMirror: { - ok: mirror.ok, - readUrl: mirror.readUrl, - writeUrl: mirror.writeUrl, - summary: mirror.summary, - probe: compactCapture(mirror.result, { full: options.full || options.raw, stdoutTailChars: 6000, stderrTailChars: 3000 }), - ...(options.raw ? { raw: mirror.raw } : {}), - }, - runtimeAlignment, - targetValidation, - disclosure: { - defaultView: "compact-low-noise", - full: options.full, - raw: options.raw, - stdoutTailOmitted: !(options.full || options.raw), - rawGitMirrorOmitted: !options.raw, - expandWith: `bun scripts/cli.ts agentrun control-plane status${statusTargetArg(options, target)} --full`, - rawWith: `bun scripts/cli.ts agentrun control-plane status${statusTargetArg(options, target)} --raw`, - }, - next: { - statusByPipelineRun: pipelineRun ? `bun scripts/cli.ts agentrun control-plane status --pipeline-run ${pipelineRun} --full` : null, - statusBySourceCommit: sourceCommit ? `bun scripts/cli.ts agentrun control-plane status --source-commit ${sourceCommit} --full` : null, - taskRuns: pipelineRun ? `trans G14:k3s kubectl get taskrun -n ${ciNamespace} -l tekton.dev/pipelineRun=${pipelineRun} -o wide` : null, - logs: pipelineRun ? `trans G14:k3s logs -n ${ciNamespace} -l tekton.dev/pipelineRun=${pipelineRun} --tail 120` : null, - triggerCurrent: "bun scripts/cli.ts agentrun control-plane trigger-current --confirm", - refresh: "bun scripts/cli.ts agentrun control-plane refresh --confirm", - }, - }; + return await statusYamlLane(config, options, resolveAgentRunLaneTarget(options)); } async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { @@ -2604,109 +2463,7 @@ function frpsAllowPortExists(toml: string, port: number): boolean { } async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): Promise> { - if (options.node !== null || options.lane !== null) { - const target = resolveAgentRunLaneTarget(options); - return await triggerCurrentYamlLane(config, options, target); - } - const source = await capture(config, g14SourceRoute, ["script", "--", [ - "set -u", - "cd /root/agentrun-v01", - "fetch_status=succeeded", - "fetch_output=$(git fetch origin v0.1 2>&1)", - "fetch_code=$?", - "if [ \"$fetch_code\" -ne 0 ]; then fetch_status=failed; fi", - "printf 'fetchStatus=%s\\n' \"$fetch_status\"", - "printf 'fetchExitCode=%s\\n' \"$fetch_code\"", - "printf 'fetchOutput=%s\\n' \"$fetch_output\" | tail -n 8", - "origin_ref=$(git rev-parse refs/remotes/origin/v0.1 2>/dev/null || true)", - "merge_status=skipped", - "merge_code=0", - "if [ -n \"$origin_ref\" ]; then", - " merge_output=$(git merge --ff-only refs/remotes/origin/v0.1 2>&1)", - " merge_code=$?", - " if [ \"$merge_code\" -eq 0 ]; then merge_status=succeeded; else merge_status=failed; fi", - " printf 'mergeStatus=%s\\n' \"$merge_status\"", - " printf 'mergeExitCode=%s\\n' \"$merge_code\"", - " printf 'mergeOutput=%s\\n' \"$merge_output\" | tail -n 8", - " if [ \"$merge_code\" -ne 0 ]; then exit \"$merge_code\"; fi", - "else", - " printf 'mergeStatus=skipped-origin-missing\\n'", - " printf 'originRefMissing=true\\n'", - "fi", - "printf 'sourceCommit='", - "git rev-parse HEAD", - "printf 'originV01='", - "git rev-parse refs/remotes/origin/v0.1 2>/dev/null || true", - "git status --short --branch", - ].join("\n")]); - const sourceCommit = matchLine(source.stdout, "sourceCommit="); - const pipelineRun = sourceCommit ? pipelineRunName(sourceCommit) : null; - if (source.exitCode !== 0 || !sourceCommit || !isGitSha(sourceCommit) || !pipelineRun) { - return { ok: false, command: "agentrun control-plane trigger-current", degradedReason: "source-head-unresolved", source: compactCapture(source) }; - } - const plan = { - lane: "v0.1", - sourceBranch, - sourceCommit, - pipelineRun, - namespace: ciNamespace, - pipeline: pipelineName, - runtimeNamespace, - gitMirror: { - readUrl: gitMirrorReadUrl, - writeUrl: gitMirrorWriteUrl, - }, - }; - const mirrorBefore = await readGitMirrorStatus(config); - const mirrorRequirement = gitMirrorSyncRequirement(sourceCommit, mirrorBefore.raw); - if (options.dryRun || !options.confirm) { - return { - ok: true, - command: "agentrun control-plane trigger-current", - dryRun: true, - plan, - gitMirrorPreSync: { - required: mirrorRequirement.required, - reason: mirrorRequirement.reason, - before: mirrorBefore.summary, - }, - next: { confirm: "bun scripts/cli.ts agentrun control-plane trigger-current --confirm" }, - }; - } - let gitMirrorPreSync: Record = { - required: mirrorRequirement.required, - reason: mirrorRequirement.reason, - before: mirrorBefore.summary, - }; - if (mirrorRequirement.required) { - const synced = await runGitMirrorJob(config, "sync", { confirm: true, dryRun: false, timeoutSeconds: 300, wait: true }); - const after = await readGitMirrorStatus(config); - const afterRequirement = gitMirrorSyncRequirement(sourceCommit, after.raw); - gitMirrorPreSync = { ...gitMirrorPreSync, sync: synced, after: after.summary, ok: synced.ok === true && afterRequirement.required === false }; - if (synced.ok !== true || afterRequirement.required !== false) { - return { - ok: false, - command: "agentrun control-plane trigger-current", - dryRun: false, - degradedReason: "git-mirror-local-v01-not-current-after-sync", - plan, - gitMirrorPreSync, - }; - } - } - const created = await capture(config, g14K3sRoute, ["script", "--", triggerScript(sourceCommit, pipelineRun)]); - return { - ok: created.exitCode === 0, - command: "agentrun control-plane trigger-current", - dryRun: false, - plan, - gitMirrorPreSync, - created: compactCapture(created), - next: { - status: "bun scripts/cli.ts agentrun control-plane status", - logs: `trans G14:k3s logs -n ${ciNamespace} -l tekton.dev/pipelineRun=${pipelineRun} --tail 120`, - }, - }; + return await triggerCurrentYamlLane(config, options, resolveAgentRunLaneTarget(options)); } async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { @@ -2929,39 +2686,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen } async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise> { - if (options.node !== null || options.lane !== null) return await refreshYamlLane(config, options); - const source = await capture(config, g14SourceRoute, ["script", "--", [ - "cd /root/agentrun-v01", - "printf 'gitopsLatest='", - `git ls-remote origin ${gitopsBranch} 2>/dev/null | awk '{print $1}' || true`, - ].join("\n")]); - const gitopsLatest = matchLine(source.stdout, "gitopsLatest="); - const plan = { - lane: "v0.1", - argoNamespace, - argoApplication, - gitopsBranch, - gitopsLatest, - }; - if (options.dryRun || !options.confirm) { - return { - ok: true, - command: "agentrun control-plane refresh", - dryRun: true, - plan, - next: { confirm: "bun scripts/cli.ts agentrun control-plane refresh --confirm" }, - }; - } - const refreshed = await capture(config, g14K3sRoute, ["script", "--", refreshScript()]); - return { - ok: source.exitCode === 0 && refreshed.exitCode === 0, - command: "agentrun control-plane refresh", - dryRun: false, - plan, - source: compactCapture(source), - refreshed: compactCapture(refreshed), - next: { status: "bun scripts/cli.ts agentrun control-plane status" }, - }; + return await refreshYamlLane(config, options); } async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): Promise> { @@ -3012,15 +2737,18 @@ async function refreshYamlLane(config: UniDeskConfig, options: RefreshOptions): } async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise> { - const result = await capture(config, g14K3sRoute, ["script", "--", cleanupRunsScript(options)]); + const { configPath, spec } = resolveAgentRunLaneTarget(options); + const result = await capture(config, spec.nodeKubeRoute, ["script", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-runs", + configPath, + target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", - namespace: ciNamespace, + namespace: spec.ci.namespace, minAgeMinutes: options.minAgeMinutes, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), @@ -3031,7 +2759,7 @@ async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): dryRun: true, mutation: false, next: { - confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`, + confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`, }, }; } @@ -3040,23 +2768,26 @@ async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): dryRun: false, mutation: true, followUp: { - status: "bun scripts/cli.ts agentrun control-plane status", - releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit ${options.limit} --dry-run`, - diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'", + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + releasedPvs: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`, + diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise> { - const result = await capture(config, g14K3sRoute, ["script", "--", cleanupReleasedPvsScript(options)]); + const { configPath, spec } = resolveAgentRunLaneTarget(options); + const result = await capture(config, spec.nodeKubeRoute, ["script", "--", cleanupReleasedPvsScript(options, spec.ci.namespace)]); const payload = captureJsonPayload(result); const ok = result.exitCode === 0 && payload.ok !== false; const base = { ...payload, ok, command: "agentrun control-plane cleanup-released-pvs", + configPath, + target: agentRunLaneSummary(spec), mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", - namespace: ciNamespace, + namespace: spec.ci.namespace, limit: options.limit, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), }; @@ -3066,7 +2797,7 @@ async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupRelease dryRun: true, mutation: false, next: { - confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit ${options.limit} --confirm`, + confirm: `bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm`, }, }; } @@ -3075,16 +2806,17 @@ async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupRelease dryRun: false, mutation: true, followUp: { - cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit ${options.limit} --dry-run`, - diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'", + cleanupRuns: `bun scripts/cli.ts agentrun control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 30 --limit ${options.limit} --dry-run`, + diskPressure: `trans ${spec.nodeKubeRoute} kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\";\"}{end}{\"\\n\"}{end}'`, }, }; } -function cleanupRunsScript(options: CleanupRunsOptions): string { +function cleanupRunsScript(options: CleanupRunsOptions, namespace: string, pipelineRunPrefix: string): string { return [ "set -eu", - `namespace=${shQuote(ciNamespace)}`, + `namespace=${shQuote(namespace)}`, + `pipeline_run_prefix=${shQuote(pipelineRunPrefix)}`, `min_age_minutes=${String(options.minAgeMinutes)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, @@ -3094,7 +2826,7 @@ function cleanupRunsScript(options: CleanupRunsOptions): string { "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"", "kubectl get pv -o json > \"$tmp_dir/pvs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", - "NAMESPACE=\"$namespace\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", + "NAMESPACE=\"$namespace\" PIPELINE_RUN_PREFIX=\"$pipeline_run_prefix\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunsPlanNodeScript(), "NODE", "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", @@ -3116,10 +2848,10 @@ function cleanupRunsScript(options: CleanupRunsOptions): string { ].join("\n"); } -function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions): string { +function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions, namespace: string): string { return [ "set -eu", - `namespace=${shQuote(ciNamespace)}`, + `namespace=${shQuote(namespace)}`, `limit=${String(options.limit)}`, `timeout_seconds=${String(options.timeoutSeconds)}`, "tmp_dir=$(mktemp -d)", @@ -3704,7 +3436,8 @@ function yamlLaneGitMirrorCacheVolume(spec: AgentRunLaneSpec): Record { +function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync" | "flush", name: string): Record { + const proxyHostNeedsHostNetwork = spec.gitMirror.githubProxy.host === "127.0.0.1" || spec.gitMirror.githubProxy.host === "localhost"; return { apiVersion: "batch/v1", kind: "Job", @@ -3735,6 +3468,7 @@ function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync", na }, spec: { restartPolicy: "Never", + ...(proxyHostNeedsHostNetwork ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}), volumes: [ yamlLaneGitMirrorCacheVolume(spec), { name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName, defaultMode: 0o400 } }, @@ -3743,7 +3477,7 @@ function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync", na name: action, image: spec.gitMirror.toolsImage, imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-ec", yamlLaneGitMirrorSyncShell(spec)], + command: ["/bin/sh", "-ec", action === "sync" ? yamlLaneGitMirrorSyncShell(spec) : yamlLaneGitMirrorFlushShell(spec)], volumeMounts: [ { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, @@ -3755,9 +3489,8 @@ function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync", na }; } -function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string { +function yamlLaneGitMirrorSshSetupShellLines(spec: AgentRunLaneSpec): string[] { return [ - "set -eu", "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", @@ -3792,6 +3525,13 @@ function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string { "NODE_PROXY", "chmod 0700 /tmp/agentrun-github-proxy-connect.cjs", `export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/agentrun-github-proxy-connect.cjs ${spec.gitMirror.githubProxy.host} ${spec.gitMirror.githubProxy.port} %h %p'`)}`, + ]; +} + +function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string { + return [ + "set -eu", + ...yamlLaneGitMirrorSshSetupShellLines(spec), `repository=${shQuote(spec.source.repository)}`, `source_branch=${shQuote(spec.source.branch)}`, `gitops_branch=${shQuote(spec.gitops.branch)}`, @@ -3822,6 +3562,29 @@ function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string { ].join("\n"); } +function yamlLaneGitMirrorFlushShell(spec: AgentRunLaneSpec): string { + return [ + "set -eu", + ...yamlLaneGitMirrorSshSetupShellLines(spec), + `repository=${shQuote(spec.source.repository)}`, + `gitops_branch=${shQuote(spec.gitops.branch)}`, + "repo=\"/cache/${repository}.git\"", + "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", + "test -d \"$repo\"", + "git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", + "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "if [ -n \"$local_gitops\" ]; then", + " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", + " git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", + "fi", + "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", + "LOCAL_GITOPS=\"$local_gitops\" GITHUB_GITOPS=\"$github_gitops\" PENDING=\"$pending\" node <<'NODE'", + "console.log(JSON.stringify({ ok: process.env.PENDING !== 'true', localGitops: process.env.LOCAL_GITOPS || null, githubGitops: process.env.GITHUB_GITOPS || null, pendingFlush: process.env.PENDING === 'true', valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", @@ -4119,6 +3882,7 @@ const path = require("node:path"); const cp = require("node:child_process"); const tmp = process.env.TMP_DIR; const namespace = process.env.NAMESPACE; +const pipelineRunPrefix = process.env.PIPELINE_RUN_PREFIX || ""; const minAgeMinutes = Number(process.env.MIN_AGE_MINUTES || 60); const limit = Number(process.env.LIMIT || 20); const now = Date.now(); @@ -4202,7 +3966,7 @@ const allPipelineRuns = (Array.isArray(pipelineRuns.items) ? pipelineRuns.items reason: condition.reason || null, }; }) - .filter((item) => item.name.startsWith("agentrun-v01-ci-")); + .filter((item) => pipelineRunPrefix.length === 0 || item.name.startsWith(pipelineRunPrefix + "-")); const protectedActivePipelineRuns = allPipelineRuns .filter((item) => item.status !== "True" && item.status !== "False") @@ -4263,7 +4027,7 @@ console.log(JSON.stringify({ planKind: "agentrun-ci-completed-pipelinerun-workspace-retention", generatedAt: new Date().toISOString(), namespace, - criteria: { prefix: "agentrun-v01-ci-", terminalStatuses: ["True", "False"], minAgeMinutes, limit }, + criteria: { prefix: pipelineRunPrefix + "-", terminalStatuses: ["True", "False"], minAgeMinutes, limit }, candidates, candidateCount: candidates.length, protectedActivePipelineRuns, @@ -4439,94 +4203,6 @@ console.log(JSON.stringify({ `; } -function statusScript(pipelineRun: string | null): string { - const pr = pipelineRun ?? ""; - return [ - "set -eu", - "printf 'pipelineRun\\tstatus\\treason\\tstart\\tcompletion\\n'", - pr.length > 0 - ? `kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pr)} -o 'jsonpath={.metadata.name}{\"\\t\"}{.status.conditions[0].status}{\"\\t\"}{.status.conditions[0].reason}{\"\\t\"}{.status.startTime}{\"\\t\"}{.status.completionTime}{\"\\n\"}' 2>/dev/null || true` - : "true", - "printf 'pipelineRunCondition='", - pr.length > 0 - ? [ - `if kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pr)} -o json >/tmp/agentrun-v01-pipelinerun.json 2>/dev/null; then`, - "node <<'NODE' || printf '{}\\n'", - "const fs = require('node:fs');", - "const pr = JSON.parse(fs.readFileSync('/tmp/agentrun-v01-pipelinerun.json', 'utf8'));", - "const condition = pr?.status?.conditions?.[0] || {};", - "const params = Array.isArray(pr?.spec?.params) ? pr.spec.params : [];", - "const sourceCommit = params.find((entry) => entry?.name === 'revision')?.value || null;", - "console.log(JSON.stringify({", - " name: pr?.metadata?.name || null,", - " sourceCommit,", - " createdAt: pr?.metadata?.creationTimestamp || null,", - " status: condition.status || null,", - " reason: condition.reason || null,", - " message: condition.message || null,", - " startTime: pr?.status?.startTime || null,", - " completionTime: pr?.status?.completionTime || null", - "}));", - "NODE", - "else printf '{}\\n'; fi", - ].join("\n") - : "printf '{}\\n'", - "printf 'taskRuns\\n'", - pr.length > 0 - ? `kubectl -n ${ciNamespace} get taskrun -l tekton.dev/pipelineRun=${shQuote(pr)} -o 'custom-columns=NAME:.metadata.name,STATUS:.status.conditions[0].status,REASON:.status.conditions[0].reason,START:.status.startTime,COMPLETION:.status.completionTime' --no-headers 2>/dev/null || true` - : "true", - "printf 'taskRunPods\\n'", - pr.length > 0 - ? `kubectl -n ${ciNamespace} get pod -l tekton.dev/pipelineRun=${shQuote(pr)} -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,AGE:.metadata.creationTimestamp' --no-headers 2>/dev/null || true` - : "true", - "printf 'recentPipelineRuns\\n'", - `kubectl -n ${ciNamespace} get pipelinerun --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,STATUS:.status.conditions[0].status,REASON:.status.conditions[0].reason,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null | tail -n 5 || true`, - "printf 'ciSummary='", - pr.length > 0 - ? [ - `if kubectl -n ${ciNamespace} get taskrun -l tekton.dev/pipelineRun=${shQuote(pr)} -o json >/tmp/agentrun-v01-taskruns.json 2>/dev/null; then`, - "node <<'NODE' || printf '{}\\n'", - "const fs = require('node:fs');", - "const data = JSON.parse(fs.readFileSync('/tmp/agentrun-v01-taskruns.json', 'utf8'));", - "const items = Array.isArray(data.items) ? data.items : [];", - "function task(item) { return item?.metadata?.labels?.['tekton.dev/pipelineTask'] || item?.metadata?.name || null; }", - "function result(item, name) { return (item?.status?.results || item?.status?.taskResults || []).find((entry) => entry.name === name)?.value ?? null; }", - "function seconds(start, end) { const s = Date.parse(start || ''); const e = Date.parse(end || ''); return Number.isFinite(s) && Number.isFinite(e) ? Math.round((e - s) / 1000) : null; }", - "const taskRuns = items.map((item) => ({ name: item?.metadata?.name || null, task: task(item), status: item?.status?.conditions?.[0]?.status || null, reason: item?.status?.conditions?.[0]?.reason || null, seconds: seconds(item?.status?.startTime, item?.status?.completionTime) }));", - "const plan = items.find((item) => task(item) === 'plan-artifacts');", - "const image = items.find((item) => task(item) === 'image-publish');", - "const imageSeconds = taskRuns.find((item) => item.task === 'image-publish')?.seconds ?? null;", - "console.log(JSON.stringify({", - " planArtifacts: { summary: plan ? result(plan, 'summary') : null, envIdentity: plan ? result(plan, 'env-identity') : null, buildCount: plan ? Number(result(plan, 'build-count') || 0) : null, reuseCount: plan ? Number(result(plan, 'reuse-count') || 0) : null },", - " envImage: { status: image ? result(image, 'status') : null, envIdentity: image ? result(image, 'env-identity') : null, image: image ? result(image, 'image') : null, digest: image ? result(image, 'digest') : null, repositoryDigest: image ? result(image, 'repository-digest') : null },", - " taskRuns,", - " performance: { imagePublishSeconds: imageSeconds, buildWarning: typeof imageSeconds === 'number' && imageSeconds > 120, criticalWarning: typeof imageSeconds === 'number' && imageSeconds > 180 }", - "}));", - "NODE", - "else printf '{}\\n'; fi", - ].join("\n") - : "printf '{}\\n'", - "printf 'argo\\n'", - `kubectl -n ${argoNamespace} get application ${argoApplication} -o 'jsonpath={.status.sync.revision}{"\\t"}{.status.sync.status}{"\\t"}{.status.health.status}{"\\n"}' 2>/dev/null || true`, - "printf 'workloads\\n'", - `kubectl -n ${runtimeNamespace} get deploy,sts -o wide 2>/dev/null || true`, - "printf 'managerImage\\n'", - `kubectl -n ${runtimeNamespace} get deploy agentrun-mgr -o 'jsonpath={.spec.template.spec.containers[0].image}{"\\t"}{.spec.template.spec.containers[0].env[?(@.name=="AGENTRUN_SOURCE_COMMIT")].value}{"\\n"}' 2>/dev/null || true`, - "printf 'managerPods\\n'", - `kubectl -n ${runtimeNamespace} get pod -l app.kubernetes.io/name=agentrun-mgr -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null || true`, - "printf 'recentRunnerPods\\n'", - `kubectl -n ${runtimeNamespace} get pod -l app.kubernetes.io/component=runner --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,CREATED:.metadata.creationTimestamp' --no-headers 2>/dev/null | tail -n 8 || true`, - ].join("\n"); -} - -function refreshScript(): string { - return [ - "set -eu", - `kubectl -n ${argoNamespace} annotate application ${argoApplication} argocd.argoproj.io/refresh=hard --overwrite`, - `kubectl -n ${argoNamespace} get application ${argoApplication} -o 'jsonpath={.status.sync.revision}{"\\t"}{.status.sync.status}{"\\t"}{.status.health.status}{"\\n"}' 2>/dev/null || true`, - ].join("\n"); -} - function refreshYamlLaneScript(spec: AgentRunLaneSpec): string { return [ "set -eu", @@ -4552,78 +4228,20 @@ function refreshYamlLaneScript(spec: AgentRunLaneSpec): string { ].join("\n"); } -function triggerScript(sourceCommit: string, pipelineRun: string): string { - return [ - "set -eu", - "cd /root/agentrun-v01", - `if kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pipelineRun)} >/dev/null 2>&1; then`, - ` existing_status="$(kubectl -n ${ciNamespace} get pipelinerun ${shQuote(pipelineRun)} -o 'jsonpath={.status.conditions[0].status}:{.status.conditions[0].reason}' 2>/dev/null || true)"`, - " case \"$existing_status\" in", - " False:*)", - " printf 'deleteExisting=%s\\n' \"$existing_status\"", - ` kubectl -n ${ciNamespace} delete pipelinerun ${shQuote(pipelineRun)} --wait=true --timeout=20s`, - " ;;", - " *)", - " printf 'refuseExisting=%s\\n' \"$existing_status\"", - " printf 'reason=existing-pipelinerun-active-or-succeeded\\n'", - " exit 20", - " ;;", - " esac", - "fi", - "kubectl apply -f deploy/templates/tekton/rbac.yaml", - "kubectl apply -f deploy/templates/tekton/pipeline.yaml", - "kubectl apply -f deploy/templates/argocd/project.yaml", - "kubectl apply -f deploy/templates/argocd/application-v01.yaml", - "cat <<'YAML' | kubectl create -f -", - "apiVersion: tekton.dev/v1", - "kind: PipelineRun", - "metadata:", - ` name: ${pipelineRun}`, - ` namespace: ${ciNamespace}`, - "spec:", - " pipelineRef:", - ` name: ${pipelineName}`, - " taskRunTemplate:", - " serviceAccountName: agentrun-v01-tekton-runner", - " podTemplate:", - " hostNetwork: true", - " dnsPolicy: ClusterFirstWithHostNet", - " securityContext:", - " fsGroup: 1000", - " params:", - " - name: revision", - ` value: ${sourceCommit}`, - " - name: git-read-url", - ` value: ${gitMirrorReadUrl}`, - " - name: git-write-url", - ` value: ${gitMirrorWriteUrl}`, - " - name: tools-image", - ` value: ${mirrorToolsImage}`, - " workspaces:", - " - name: source", - " volumeClaimTemplate:", - " spec:", - " accessModes: [\"ReadWriteOnce\"]", - " resources:", - " requests:", - " storage: 5Gi", - " - name: git-ssh", - " secret:", - " secretName: agentrun-git-ssh", - "YAML", - `printf 'created=${pipelineRun}\\n'`, - ].join("\n"); -} - -async function gitMirrorStatus(config: UniDeskConfig, options: DisclosureOptions = { full: false, raw: false }): Promise> { - const observation = await readGitMirrorStatus(config); +async function gitMirrorStatus(config: UniDeskConfig, options: GitMirrorStatusOptions = { full: false, raw: false, node: null, lane: null }): Promise> { + const target = resolveAgentRunLaneTarget(options); + const spec = target.spec; + const observation = await readGitMirrorStatus(config, target); const summary = observation.summary; return { ok: observation.ok, command: "agentrun git-mirror status", - namespace: gitMirrorNamespace, - readUrl: gitMirrorReadUrl, - writeUrl: gitMirrorWriteUrl, + mode: "yaml-declared-node-lane", + configPath: target.configPath, + target: agentRunLaneSummary(spec), + namespace: spec.gitMirror.namespace, + readUrl: spec.gitMirror.readUrl, + writeUrl: spec.gitMirror.writeUrl, summary, ...(options.raw ? { raw: observation.raw } : {}), probe: compactCapture(observation.result, { full: options.full || options.raw, stdoutTailChars: 6000, stderrTailChars: 3000 }), @@ -4633,34 +4251,28 @@ async function gitMirrorStatus(config: UniDeskConfig, options: DisclosureOptions raw: options.raw, rawOmitted: !options.raw, probeTailOmitted: !(options.full || options.raw), - expandWith: "bun scripts/cli.ts agentrun git-mirror status --full", - rawWith: "bun scripts/cli.ts agentrun git-mirror status --raw", + expandWith: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane} --full`, + rawWith: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane} --raw`, }, next: { - sync: "bun scripts/cli.ts agentrun git-mirror sync --confirm", - flush: summary.pendingFlush === true ? "bun scripts/cli.ts agentrun git-mirror flush --confirm" : null, + sync: `bun scripts/cli.ts agentrun git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + flush: summary.pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, }, }; } -async function readGitMirrorStatus(config: UniDeskConfig): Promise & { result: SshCaptureResult; raw: string; summary: Record }> { - const script = [ - "set +e", - "printf 'resources\\n'", - `kubectl -n ${gitMirrorNamespace} get deploy,svc,pvc,cm -l app.kubernetes.io/name=git-mirror -o name 2>/dev/null || true`, - "printf 'jobs\\n'", - `kubectl -n ${gitMirrorNamespace} get job -l app.kubernetes.io/name=git-mirror --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,SUCCEEDED:.status.succeeded,FAILED:.status.failed,START:.status.startTime,COMPLETION:.status.completionTime' --no-headers 2>/dev/null | tail -n 8 || true`, - "printf 'cache\\n'", - `kubectl exec -n ${gitMirrorNamespace} deploy/git-mirror-http -- sh -lc ${shQuote(gitMirrorCacheProbeScript())}`, - ].join("\n"); - const result = await capture(config, g14K3sRoute, ["script", "--", script]); +async function readGitMirrorStatus(config: UniDeskConfig, target: { configPath: string; spec: AgentRunLaneSpec }): Promise & { result: SshCaptureResult; raw: string; summary: Record }> { + const spec = target.spec; + const result = await capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneGitMirrorStatusScript(spec)]); const raw = result.stdout; - const summary = gitMirrorStatusSummary(raw); + const summary = captureJsonPayload(result); return { - ok: result.exitCode === 0 && summary.localV01 !== null, - namespace: gitMirrorNamespace, - readUrl: gitMirrorReadUrl, - writeUrl: gitMirrorWriteUrl, + ok: result.exitCode === 0 && summary.ok !== false, + configPath: target.configPath, + target: agentRunLaneSummary(spec), + namespace: spec.gitMirror.namespace, + readUrl: spec.gitMirror.readUrl, + writeUrl: spec.gitMirror.writeUrl, raw, summary, result, @@ -4668,90 +4280,85 @@ async function readGitMirrorStatus(config: UniDeskConfig): Promise> { - const jobName = `${action === "sync" ? gitMirrorSyncJobPrefix : gitMirrorFlushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); - const manifest = gitMirrorJobManifest(action, jobName); - const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + const { configPath, spec } = resolveAgentRunLaneTarget(options); + const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); + const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); const command = `agentrun git-mirror ${action}`; if (options.dryRun || !options.confirm) { return { ok: true, command, + mode: "yaml-declared-node-lane", + configPath, + target: agentRunLaneSummary(spec), dryRun: true, - namespace: gitMirrorNamespace, + namespace: spec.gitMirror.namespace, jobName, manifest, - next: { confirm: `bun scripts/cli.ts ${command} --confirm` }, + next: { confirm: `bun scripts/cli.ts ${command} --node ${spec.nodeId} --lane ${spec.lane} --confirm` }, }; } - const created = await createGitMirrorJob(config, jobName, manifestB64); + const created = await capture(config, spec.nodeKubeRoute, ["script", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); if (created.exitCode !== 0 || !options.wait) { - const status = await gitMirrorStatus(config); + const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: created.exitCode === 0, command, - dryRun: false, mode: options.wait ? "create-failed" : "submitted", - namespace: gitMirrorNamespace, + configPath, + target: agentRunLaneSummary(spec), + dryRun: false, + namespace: spec.gitMirror.namespace, jobName, result: compactCapture(created), status, next: { - status: "bun scripts/cli.ts agentrun git-mirror status", - wait: `bun scripts/cli.ts agentrun git-mirror ${action} --confirm --wait`, + status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, + wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, }, }; } - const wait = await waitForGitMirrorJob(config, action, jobName, options.timeoutSeconds); - const status = await gitMirrorStatus(config); + const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds); + const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: created.exitCode === 0 && wait.ok === true, command, - dryRun: false, mode: "waited", - namespace: gitMirrorNamespace, + configPath, + target: agentRunLaneSummary(spec), + dryRun: false, + namespace: spec.gitMirror.namespace, jobName, result: compactCapture(created), wait, status, next: { - status: "bun scripts/cli.ts agentrun git-mirror status", - flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? "bun scripts/cli.ts agentrun git-mirror flush --confirm" : null, + status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, + flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, }, }; } -async function createGitMirrorJob(config: UniDeskConfig, jobName: string, manifestB64: string): Promise { - const script = [ - "set -eu", - `job=${shQuote(jobName)}`, - `manifest_b64=${shQuote(manifestB64)}`, - "manifest_path=\"/tmp/$job.json\"", - "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", - `kubectl delete job -n ${gitMirrorNamespace} "$job" --ignore-not-found=true >/dev/null`, - "kubectl create -f \"$manifest_path\"", - `kubectl get job -n ${gitMirrorNamespace} "$job" -o 'jsonpath=created={.metadata.name}{\"\\n\"}'`, - ].join("\n"); - return await capture(config, g14K3sRoute, ["script", "--", script]); -} - -async function waitForGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", jobName: string, timeoutSeconds: number): Promise> { +async function waitForGitMirrorJob(config: UniDeskConfig, spec: AgentRunLaneSpec, action: "sync" | "flush", jobName: string, timeoutSeconds: number): Promise> { const startedAtMs = Date.now(); let lastProbe: SshCaptureResult | null = null; let polls = 0; while (Date.now() - startedAtMs <= timeoutSeconds * 1000) { polls += 1; - lastProbe = await capture(config, g14K3sRoute, ["script", "--", gitMirrorJobProbeScript(jobName)]); - const summary = gitMirrorJobProbeSummary(lastProbe.stdout); + lastProbe = await capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); + const summary = captureJsonPayload(lastProbe); process.stderr.write(`${JSON.stringify({ event: `agentrun.git-mirror.${action}.progress`, at: new Date().toISOString(), stage: "k8s-job", - status: summary.succeeded ? "succeeded" : summary.failed ? "failed" : "running", + node: spec.nodeId, + lane: spec.lane, + status: summary.succeeded === true ? "succeeded" : summary.failed === true ? "failed" : "running", jobName, polls, elapsedMs: Date.now() - startedAtMs, })}\n`); - if (summary.succeeded) { + if (summary.succeeded === true) { return { ok: true, action, @@ -4762,7 +4369,7 @@ async function waitForGitMirrorJob(config: UniDeskConfig, action: "sync" | "flus probe: compactCapture(lastProbe), }; } - if (summary.failed) { + if (summary.failed === true) { return { ok: false, action, @@ -4787,317 +4394,6 @@ async function waitForGitMirrorJob(config: UniDeskConfig, action: "sync" | "flus }; } -function gitMirrorJobProbeScript(jobName: string): string { - return [ - "set +e", - `job=${shQuote(jobName)}`, - "printf 'jobStatus='", - `kubectl get job -n ${gitMirrorNamespace} "$job" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed} active={.status.active} start={.status.startTime} completion={.status.completionTime}' 2>/dev/null || true`, - "printf '\\n'", - "printf 'pods\\n'", - `kubectl get pod -n ${gitMirrorNamespace} -l job-name="$job" -o 'custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,STATUS:.status.phase,RESTARTS:.status.containerStatuses[*].restartCount,START:.status.startTime' --no-headers 2>/dev/null || true`, - "printf 'logs\\n'", - `kubectl logs -n ${gitMirrorNamespace} "job/$job" --tail=120 2>/dev/null || true`, - ].join("\n"); -} - -function gitMirrorJobProbeSummary(stdout: string): Record { - const line = matchLine(stdout, "jobStatus=") ?? ""; - const succeeded = /(?:^|\s)succeeded=1(?:\s|$)/u.test(line); - const failedRaw = firstMatch(line, /(?:^|\s)failed=([0-9]+)/u); - const failed = failedRaw !== null && Number(failedRaw) > 0; - const activeRaw = firstMatch(line, /(?:^|\s)active=([0-9]+)/u); - return { - statusLine: line, - succeeded, - failed, - active: activeRaw === null ? null : Number(activeRaw), - }; -} - -function gitMirrorJobManifest(action: "sync" | "flush", name: string): Record { - return { - apiVersion: "batch/v1", - kind: "Job", - metadata: { - name, - namespace: gitMirrorNamespace, - labels: { - "app.kubernetes.io/name": "git-mirror", - "app.kubernetes.io/part-of": "devops-infra", - "app.kubernetes.io/component": action === "sync" ? "sync-controller" : "flush-controller", - "agentrun.pikastech.local/trigger": "manual-cli", - }, - }, - spec: { - backoffLimit: 0, - activeDeadlineSeconds: 600, - ttlSecondsAfterFinished: 3600, - template: { - metadata: { - labels: { - "app.kubernetes.io/name": "git-mirror", - "app.kubernetes.io/part-of": "devops-infra", - "app.kubernetes.io/component": action === "sync" ? "sync-controller" : "flush-controller", - "agentrun.pikastech.local/trigger": "manual-cli", - }, - }, - spec: { - restartPolicy: "Never", - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet", - volumes: [ - { name: "cache", persistentVolumeClaim: { claimName: "git-mirror-cache" } }, - { name: "git-ssh", secret: { secretName: "git-mirror-github-ssh", defaultMode: 0o400 } }, - ], - containers: [{ - name: action, - image: mirrorToolsImage, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-ec", action === "sync" ? gitMirrorSyncShellScript() : gitMirrorFlushShellScript()], - volumeMounts: [ - { name: "cache", mountPath: "/cache" }, - { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - ], - }], - }, - }, - }, - }; -} - -function gitMirrorSshSetupShellLines(): string[] { - return [ - "mkdir -p /root/.ssh", - "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - "cat > /tmp/agentrun-github-proxy-connect.mjs <<'NODE_PROXY'", - "#!/usr/bin/env node", - "import net from \"node:net\";", - "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", - "const proxyPort = Number.parseInt(proxyPortRaw || \"\", 10);", - "const targetPort = Number.parseInt(targetPortRaw || \"\", 10);", - "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);", - "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", - "let buffer = Buffer.alloc(0);", - "socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });", - "socket.on(\"connect\", () => socket.write(\"CONNECT \" + targetHost + \":\" + targetPort + \" HTTP/1.1\\r\\nHost: \" + targetHost + \":\" + targetPort + \"\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n\"));", - "socket.on(\"error\", () => process.exit(66));", - "function onData(chunk) {", - " buffer = Buffer.concat([buffer, chunk]);", - " const headerEnd = buffer.indexOf(\"\\r\\n\\r\\n\");", - " if (headerEnd === -1 && buffer.length < 8192) return;", - " const head = buffer.slice(0, headerEnd + 4).toString(\"latin1\");", - " const statusLine = head.split(\"\\r\\n\", 1)[0] || \"\";", - " const statusCode = Number.parseInt(statusLine.split(\" \")[1] || \"\", 10);", - " if (!statusLine.startsWith(\"HTTP/1.\") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }", - " socket.off(\"data\", onData);", - " socket.setTimeout(0);", - " const rest = buffer.slice(headerEnd + 4);", - " if (rest.length) process.stdout.write(rest);", - " process.stdin.pipe(socket);", - " socket.pipe(process.stdout);", - "}", - "socket.on(\"data\", onData);", - "socket.on(\"close\", () => process.exit(0));", - "NODE_PROXY", - "chmod 0700 /tmp/agentrun-github-proxy-connect.mjs", - "export GIT_SSH_COMMAND=\"ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/agentrun-github-proxy-connect.mjs 127.0.0.1 10808 %h %p'\"", - ]; -} - -function gitMirrorSyncShellScript(): string { - const repoSpecs = gitMirrorRepositories.map((repo) => [repo.key, repo.repository, repo.sourceBranch, repo.gitopsBranch ?? ""].join("|")); - return [ - "set -eu", - "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", - "mkdir -p /cache/pikasTech", - ...gitMirrorSshSetupShellLines(), - "sync_repo() {", - " key=\"$1\"", - " repository=\"$2\"", - " source_branch=\"$3\"", - " gitops_branch=\"$4\"", - " repo=\"/cache/${repository}.git\"", - " remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", - " mkdir -p \"$(dirname \"$repo\")\"", - " if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", - " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", - " else", - " rm -rf \"$repo\"", - " git init --bare \"$repo\"", - " git --git-dir=\"$repo\" remote add origin \"$remote\"", - " fi", - " git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", - " git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", - " git --git-dir=\"$repo\" config http.receivepack true", - " timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", - " source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", - " git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", - " if [ -n \"$gitops_branch\" ]; then", - " if timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", - " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - " if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then", - " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", - " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", - " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", - " fi", - " fi", - " fi", - " git --git-dir=\"$repo\" update-server-info", - "}", - `for spec in ${repoSpecs.map(shQuote).join(" ")}; do`, - " IFS='|' read -r key repository source_branch gitops_branch </dev/null || true)", - "if [ -n \"$local_gitops\" ]; then", - " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin refs/heads/v0.1-gitops:refs/heads/v0.1-gitops", - " git --git-dir=\"$repo\" fetch origin +refs/heads/v0.1-gitops:refs/mirror-stage/heads/v0.1-gitops", - "fi", - "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify 'refs/mirror-stage/heads/v0.1-gitops^{commit}' 2>/dev/null || true)", - "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", - "json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }", - "cat > /cache/agentrun.last-flush.json <