fix: improve AgentRun trigger closeout visibility
This commit is contained in:
Codex
+121
-3
@@ -2096,10 +2096,11 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
const spec = target.spec;
|
||||
const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)]));
|
||||
const sourcePayload = captureJsonPayload(sourceProbe.value);
|
||||
const sourceCommit = options.sourceCommit
|
||||
const branchTipCommit = stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead);
|
||||
const initialSourceCommit = options.sourceCommit
|
||||
?? stringOrNull(sourcePayload.remoteBranchCommit)
|
||||
?? stringOrNull(sourcePayload.localHead);
|
||||
const pipelineRunName = options.pipelineRun ?? (sourceCommit ? agentRunPipelineRunName(spec, sourceCommit) : null);
|
||||
const pipelineRunName = options.pipelineRun ?? (initialSourceCommit ? agentRunPipelineRunName(spec, initialSourceCommit) : null);
|
||||
const [runtimeProbe, mirrorProbe] = await Promise.all([
|
||||
timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])),
|
||||
timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)])),
|
||||
@@ -2114,9 +2115,35 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
const secrets = record(runtimePayload.secrets);
|
||||
const localPostgres = record(runtimePayload.localPostgres);
|
||||
const expectedGitopsRevision = stringOrNull(mirrorPayload.gitopsCommit);
|
||||
const pipelineRunSourceCommit = stringOrNull(pipelineRunStatus.sourceCommit);
|
||||
const sourceCommit = options.sourceCommit
|
||||
?? (options.pipelineRun !== null ? pipelineRunSourceCommit ?? initialSourceCommit : initialSourceCommit);
|
||||
const managerSourceMatchesExpected = Boolean(sourceCommit && manager.sourceCommit === sourceCommit);
|
||||
const argoSyncedToGitops = Boolean(expectedGitopsRevision && argo.revision === expectedGitopsRevision);
|
||||
const pipelineSucceeded = pipelineRunStatus.status === "True";
|
||||
const sourceWorktreeDetached = sourcePayload.branch === "HEAD";
|
||||
const targetMode = options.pipelineRun !== null ? "pipeline-run" : options.sourceCommit !== null ? "source-commit" : "latest-source-head";
|
||||
const sourceBranchAdvanced = targetMode !== "latest-source-head"
|
||||
&& sourceCommit !== null
|
||||
&& branchTipCommit !== null
|
||||
&& sourceCommit !== branchTipCommit;
|
||||
const branchDrift = {
|
||||
targetMode,
|
||||
sourceBranch: spec.source.branch,
|
||||
currentBranchTipCommit: branchTipCommit,
|
||||
targetSourceCommit: sourceCommit,
|
||||
pipelineRunSourceCommit,
|
||||
sourceBranchAdvanced,
|
||||
targetSupersededByCurrentBranch: sourceBranchAdvanced,
|
||||
closeoutHint: sourceBranchAdvanced
|
||||
? "target PipelineRun/source commit is not the current branch tip; verify the branch tip contains the fix, then trigger-current again for latest-tip closeout"
|
||||
: null,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
const warnings = [
|
||||
...(sourceWorktreeDetached ? ["source-worktree-detached"] : []),
|
||||
...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []),
|
||||
];
|
||||
const blockers = [
|
||||
...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]),
|
||||
...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]),
|
||||
@@ -2151,13 +2178,17 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
const summary = {
|
||||
aligned,
|
||||
blockers,
|
||||
warnings,
|
||||
sourceCommit,
|
||||
expectedPipelineRun: pipelineRunName,
|
||||
expectedGitopsRevision,
|
||||
runtimeAlignment,
|
||||
branchDrift,
|
||||
source: {
|
||||
workspaceExists: sourcePayload.workspaceExists ?? false,
|
||||
workspaceClean: sourcePayload.workspaceClean ?? null,
|
||||
branch: sourcePayload.branch ?? null,
|
||||
workspaceDetached: sourceWorktreeDetached,
|
||||
localHead: sourcePayload.localHead ?? null,
|
||||
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
|
||||
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
|
||||
@@ -2204,10 +2235,12 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
alignment: {
|
||||
aligned,
|
||||
blockers,
|
||||
warnings,
|
||||
sourceCommit,
|
||||
expectedPipelineRun: pipelineRunName,
|
||||
expectedGitopsRevision,
|
||||
runtimeAlignment,
|
||||
branchDrift,
|
||||
},
|
||||
timings: {
|
||||
sourceMs: sourceProbe.elapsedMs,
|
||||
@@ -2234,6 +2267,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
||||
restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
||||
statusFull: statusFullCommand,
|
||||
triggerLatest: sourceBranchAdvanced ? `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
@@ -2678,6 +2712,21 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt
|
||||
}
|
||||
|
||||
async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise<Record<string, unknown>> {
|
||||
const result = await triggerCurrentYamlLaneConfirmedSteps(config, spec, configPath, waited);
|
||||
const restore = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceRestoreScript(spec)]);
|
||||
const restorePayload = captureJsonPayload(restore);
|
||||
const restoreOk = restore.exitCode === 0 && restorePayload.ok === true;
|
||||
const existingWarnings = Array.isArray(result.warnings) ? result.warnings.filter((item): item is string => typeof item === "string") : [];
|
||||
return {
|
||||
...result,
|
||||
warnings: restoreOk ? existingWarnings : Array.from(new Set([...existingWarnings, "source-worktree-restore-failed"])),
|
||||
sourceWorkspaceRestore: restorePayload,
|
||||
sourceWorkspaceRestoreCapture: compactCapture(restore, { full: !restoreOk, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise<Record<string, unknown>> {
|
||||
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
@@ -3251,6 +3300,7 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string
|
||||
"function exists(exitName) { return process.env[exitName] === '0'; }",
|
||||
"function condition(obj) { return obj?.status?.conditions?.[0] || null; }",
|
||||
"function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }",
|
||||
"function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }",
|
||||
"const pipelineRun = readJson('pipelinerun.json');",
|
||||
"const argo = readJson('argo.json');",
|
||||
"const managerDeploy = readJson('manager-deploy.json');",
|
||||
@@ -3265,7 +3315,7 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string
|
||||
" ciNamespaceExists: exists('CI_NS_EXIT'),",
|
||||
" serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),",
|
||||
" pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },",
|
||||
" pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },",
|
||||
" pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },",
|
||||
" argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },",
|
||||
" manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },",
|
||||
" database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },",
|
||||
@@ -3399,6 +3449,74 @@ function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: stri
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function yamlLaneSourceRestoreScript(spec: AgentRunLaneSpec): string {
|
||||
return [
|
||||
"set +e",
|
||||
`workspace=${shQuote(spec.source.workspace)}`,
|
||||
`branch=${shQuote(spec.source.branch)}`,
|
||||
"tmp_dir=$(mktemp -d)",
|
||||
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
||||
"workspace_exists=false",
|
||||
"if git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then workspace_exists=true; fi",
|
||||
"if [ \"$workspace_exists\" != true ]; then",
|
||||
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-missing', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"before_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
||||
"before_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
||||
"status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
||||
"if [ -n \"$status_short\" ]; then",
|
||||
" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" STATUS_SHORT=\"$status_short\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'skipped', failureKind: 'source-worktree-dirty', workspace: process.env.WORKSPACE, branch: process.env.BRANCH, before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' }, statusShort: process.env.STATUS_SHORT || null, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"git -C \"$workspace\" fetch origin \"$branch\" > \"$tmp_dir/fetch.out\" 2> \"$tmp_dir/fetch.err\"",
|
||||
"fetch_exit=$?",
|
||||
"remote_branch_commit=$(git -C \"$workspace\" rev-parse \"refs/remotes/origin/$branch\" 2>/dev/null || true)",
|
||||
"checkout_exit=1",
|
||||
"if [ \"$fetch_exit\" -eq 0 ] && [ -n \"$remote_branch_commit\" ]; then",
|
||||
" git -C \"$workspace\" checkout -B \"$branch\" \"refs/remotes/origin/$branch\" > \"$tmp_dir/checkout.out\" 2> \"$tmp_dir/checkout.err\"",
|
||||
" checkout_exit=$?",
|
||||
"else",
|
||||
" : > \"$tmp_dir/checkout.out\"",
|
||||
" : > \"$tmp_dir/checkout.err\"",
|
||||
"fi",
|
||||
"after_branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)",
|
||||
"after_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)",
|
||||
"after_status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)",
|
||||
"fetch_err=$(tail -n 20 \"$tmp_dir/fetch.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
|
||||
"checkout_err=$(tail -n 20 \"$tmp_dir/checkout.err\" 2>/dev/null | tr '\\n' ' ' | cut -c1-1200)",
|
||||
"WORKSPACE=\"$workspace\" BRANCH=\"$branch\" BEFORE_BRANCH=\"$before_branch\" BEFORE_HEAD=\"$before_head\" FETCH_EXIT=\"$fetch_exit\" CHECKOUT_EXIT=\"$checkout_exit\" REMOTE_BRANCH_COMMIT=\"$remote_branch_commit\" AFTER_BRANCH=\"$after_branch\" AFTER_HEAD=\"$after_head\" AFTER_STATUS_SHORT=\"$after_status_short\" FETCH_ERR=\"$fetch_err\" CHECKOUT_ERR=\"$checkout_err\" node <<'NODE'",
|
||||
"const fetchExit = Number(process.env.FETCH_EXIT || '1');",
|
||||
"const checkoutExit = Number(process.env.CHECKOUT_EXIT || '1');",
|
||||
"const afterClean = !process.env.AFTER_STATUS_SHORT;",
|
||||
"const ok = fetchExit === 0 && checkoutExit === 0 && afterClean && process.env.AFTER_BRANCH === process.env.BRANCH;",
|
||||
"let failureKind = null;",
|
||||
"if (fetchExit !== 0) failureKind = 'source-branch-fetch-failed';",
|
||||
"else if (!process.env.REMOTE_BRANCH_COMMIT) failureKind = 'source-branch-missing';",
|
||||
"else if (checkoutExit !== 0) failureKind = 'source-branch-checkout-failed';",
|
||||
"else if (!afterClean) failureKind = 'source-worktree-dirty-after-restore';",
|
||||
"else if (process.env.AFTER_BRANCH !== process.env.BRANCH) failureKind = 'source-worktree-branch-mismatch';",
|
||||
"console.log(JSON.stringify({",
|
||||
" ok,",
|
||||
" status: ok ? 'restored' : 'failed',",
|
||||
" failureKind,",
|
||||
" workspace: process.env.WORKSPACE,",
|
||||
" branch: process.env.BRANCH,",
|
||||
" before: { branch: process.env.BEFORE_BRANCH || null, head: process.env.BEFORE_HEAD || null, detached: process.env.BEFORE_BRANCH === 'HEAD' },",
|
||||
" after: { branch: process.env.AFTER_BRANCH || null, head: process.env.AFTER_HEAD || null, clean: afterClean },",
|
||||
" remoteBranchCommit: process.env.REMOTE_BRANCH_COMMIT || null,",
|
||||
" fetch: { exitCode: fetchExit, stderrTail: process.env.FETCH_ERR || null },",
|
||||
" checkout: { exitCode: checkoutExit, stderrTail: process.env.CHECKOUT_ERR || null },",
|
||||
" valuesPrinted: false",
|
||||
"}));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string {
|
||||
const build = spec.deployment.manager.imageBuild;
|
||||
const noProxy = build.noProxy.join(",");
|
||||
|
||||
Reference in New Issue
Block a user