diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 783b0d3f..675ae720 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2096,10 +2096,11 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar const spec = target.spec; const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)])); const sourcePayload = captureJsonPayload(sourceProbe.value); - const sourceCommit = options.sourceCommit + const branchTipCommit = stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead); + const initialSourceCommit = options.sourceCommit ?? stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead); - const pipelineRunName = options.pipelineRun ?? (sourceCommit ? agentRunPipelineRunName(spec, sourceCommit) : null); + const pipelineRunName = options.pipelineRun ?? (initialSourceCommit ? agentRunPipelineRunName(spec, initialSourceCommit) : null); const [runtimeProbe, mirrorProbe] = await Promise.all([ timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])), timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)])), @@ -2114,9 +2115,35 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar const secrets = record(runtimePayload.secrets); const localPostgres = record(runtimePayload.localPostgres); const expectedGitopsRevision = stringOrNull(mirrorPayload.gitopsCommit); + const pipelineRunSourceCommit = stringOrNull(pipelineRunStatus.sourceCommit); + const sourceCommit = options.sourceCommit + ?? (options.pipelineRun !== null ? pipelineRunSourceCommit ?? initialSourceCommit : initialSourceCommit); const managerSourceMatchesExpected = Boolean(sourceCommit && manager.sourceCommit === sourceCommit); const argoSyncedToGitops = Boolean(expectedGitopsRevision && argo.revision === expectedGitopsRevision); const pipelineSucceeded = pipelineRunStatus.status === "True"; + const sourceWorktreeDetached = sourcePayload.branch === "HEAD"; + const targetMode = options.pipelineRun !== null ? "pipeline-run" : options.sourceCommit !== null ? "source-commit" : "latest-source-head"; + const sourceBranchAdvanced = targetMode !== "latest-source-head" + && sourceCommit !== null + && branchTipCommit !== null + && sourceCommit !== branchTipCommit; + const branchDrift = { + targetMode, + sourceBranch: spec.source.branch, + currentBranchTipCommit: branchTipCommit, + targetSourceCommit: sourceCommit, + pipelineRunSourceCommit, + sourceBranchAdvanced, + targetSupersededByCurrentBranch: sourceBranchAdvanced, + closeoutHint: sourceBranchAdvanced + ? "target PipelineRun/source commit is not the current branch tip; verify the branch tip contains the fix, then trigger-current again for latest-tip closeout" + : null, + valuesPrinted: false, + }; + const warnings = [ + ...(sourceWorktreeDetached ? ["source-worktree-detached"] : []), + ...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []), + ]; const blockers = [ ...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]), ...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]), @@ -2151,13 +2178,17 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar const summary = { aligned, blockers, + warnings, sourceCommit, expectedPipelineRun: pipelineRunName, expectedGitopsRevision, runtimeAlignment, + branchDrift, source: { workspaceExists: sourcePayload.workspaceExists ?? false, workspaceClean: sourcePayload.workspaceClean ?? null, + branch: sourcePayload.branch ?? null, + workspaceDetached: sourceWorktreeDetached, localHead: sourcePayload.localHead ?? null, remoteBranchExists: sourcePayload.remoteBranchExists ?? false, remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, @@ -2204,10 +2235,12 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar alignment: { aligned, blockers, + warnings, sourceCommit, expectedPipelineRun: pipelineRunName, expectedGitopsRevision, runtimeAlignment, + branchDrift, }, timings: { sourceMs: sourceProbe.elapsedMs, @@ -2234,6 +2267,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`, statusFull: statusFullCommand, + triggerLatest: sourceBranchAdvanced ? `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, }, valuesPrinted: false, }; @@ -2678,6 +2712,21 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt } async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { + const result = await triggerCurrentYamlLaneConfirmedSteps(config, spec, configPath, waited); + const restore = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceRestoreScript(spec)]); + const restorePayload = captureJsonPayload(restore); + const restoreOk = restore.exitCode === 0 && restorePayload.ok === true; + const existingWarnings = Array.isArray(result.warnings) ? result.warnings.filter((item): item is string => typeof item === "string") : []; + return { + ...result, + warnings: restoreOk ? existingWarnings : Array.from(new Set([...existingWarnings, "source-worktree-restore-failed"])), + sourceWorkspaceRestore: restorePayload, + sourceWorkspaceRestoreCapture: compactCapture(restore, { full: !restoreOk, stdoutTailChars: 3000, stderrTailChars: 3000 }), + valuesPrinted: false, + }; +} + +async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { node: spec.nodeId, lane: spec.lane, @@ -3251,6 +3300,7 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string "function exists(exitName) { return process.env[exitName] === '0'; }", "function condition(obj) { return obj?.status?.conditions?.[0] || null; }", "function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }", + "function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }", "const pipelineRun = readJson('pipelinerun.json');", "const argo = readJson('argo.json');", "const managerDeploy = readJson('manager-deploy.json');", @@ -3265,7 +3315,7 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string " ciNamespaceExists: exists('CI_NS_EXIT'),", " serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),", " pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },", - " pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },", + " pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },", " argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },", " manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },", " database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },", @@ -3399,6 +3449,74 @@ function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: stri ].join("\n"); } +function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string { + return [ + "set +e", + `workspace=${shQuote(spec.source.workspace)}`, + `branch=${shQuote(spec.source.branch)}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "workspace_exists=false", + "if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi", + "if [ \"$workspace_exists\" != true ]; then", + " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'", + "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));", + "NODE", + " exit 0", + "fi", + "before_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", + "before_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", + "status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", + "if [ -n \"$status_short\" ]; then", + " WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'", + "console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));", + "NODE", + " exit 0", + "fi", + "git -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"", + "fetch_exit=$?", + "remote_branch_commit=$(git -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)", + "checkout_exit=1", + "if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then", + " git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"", + " checkout_exit=$?", + "else", + " : > \"$tmp_dir/checkout.out\"", + " : > \"$tmp_dir/checkout.err\"", + "fi", + "after_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", + "after_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", + "after_status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", + "fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", + "checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)", + "WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'", + "const fetchExit = Number(process.env.FETCH_EXIT || '1');", + "const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');", + "const afterClean = !process.env.AFTER_STATUS_SHORT;", + "const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;", + "let failureKind = null;", + "if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';", + "else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';", + "else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';", + "else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';", + "else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';", + "console.log(JSON.stringify({", + " ok,", + " status: ok ? 'restored' : 'failed',", + " failureKind,", + " workspace: process.env.WORKSPACE,", + " branch: process.env.BRANCH,", + " before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },", + " after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },", + " remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,", + " fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },", + " checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); +} + function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string { const build = spec.deployment.manager.imageBuild; const noProxy = build.noProxy.join(",");