From d513d560d9739dd716c5ab5b36a5770db79b6985 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 4 Jun 2026 23:21:52 +0000 Subject: [PATCH] fix: add hwlab v02 closeout verdict --- scripts/hwlab-g14-contract-test.ts | 116 +++++++++- scripts/src/hwlab-g14.ts | 328 +++++++++++++++++++++++++++-- 2 files changed, 419 insertions(+), 25 deletions(-) diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 661e61d4..bbb87dee 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -1,4 +1,4 @@ -import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, parseV02TriggerSnapshot, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02ExistingPipelineRunReuseDecision, v02FalseGreenGuard, v02GitMirrorPreSyncWaitMs, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; +import { activeV02PipelineRuns, gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, parseV02TriggerSnapshot, rolloutRecordBody, semanticChangelogBullets, summarizeV02CdStatus, v02CloseoutVerdict, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02ExistingPipelineRunReuseDecision, v02FalseGreenGuard, v02GitMirrorPreSyncWaitMs, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); @@ -34,8 +34,10 @@ assertCondition( const hwlabHelpUsage = Array.isArray(hwlabG14Help().usage) ? hwlabG14Help().usage.map(String) : []; assertCondition( hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v02 --pipeline-run")) - && hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v02 --source-commit")), - "v0.2 control-plane status help must expose targeted PipelineRun/source-commit inspection", + && hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v02 --source-commit")) + && hwlabHelpUsage.some((line) => line.includes("control-plane closeout --lane v02 --pipeline-run")) + && hwlabHelpUsage.some((line) => line.includes("control-plane closeout --lane v02 --source-commit")), + "v0.2 control-plane help must expose targeted PipelineRun/source-commit status and closeout inspection", hwlabHelpUsage, ); assertCondition( @@ -367,6 +369,110 @@ assertCondition( latestOnlyStalePassed, ); +const supersededCloseoutStatus = { + command: "hwlab g14 control-plane status --lane v02 --source-commit aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + statusTarget: { mode: "source-commit", sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pipelineRun: "hwlab-v02-ci-poll-aaaaaaaaaaaa" }, + pipelineRun: { exists: true, pipelineRun: "hwlab-v02-ci-poll-aaaaaaaaaaaa", status: "True", reason: "Completed" }, + targetValidation: { ok: true, state: "superseded", latestOnlySuperseded: true, latestOnlyReasons: ["origin-head-mismatch"], failures: [] }, + sourceHeads: { + originHead: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + target: { + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + objectExists: true, + ancestorOfOriginHead: true, + ancestorExitCode: 0, + }, + }, + commitAlignment: { originHead: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", staleReasons: ["origin-head-mismatch"] }, + gitMirror: { + summary: { + pendingFlush: false, + githubInSync: true, + flushNeeded: false, + flushCommand: null, + lastFlush: { status: "flushed", at: "2026-06-04T00:00:00Z" }, + }, + }, + argo: { fields: { syncStatus: "Synced", health: "Healthy", syncRevision: "gitopsbbbb", targetRevision: "v0.2-gitops", path: "deploy/gitops/g14/runtime-v02" } }, + webAssets: { ok: true, summary: "19666/19667 React/Vite asset probes passed", apiRevision: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, + activePipelineRuns: [], +}; +const supersededCloseout = v02CloseoutVerdict(supersededCloseoutStatus); +assertCondition( + supersededCloseout.ok === true + && supersededCloseout.closeable === true + && supersededCloseout.state === "superseded" + && record(record(supersededCloseout.latestOnly).ancestorCheck).ancestorOfLatest === true + && String(supersededCloseout.issueCommentMarkdown).includes("ancestorOfLatest=`true`") + && String(record(supersededCloseout.recommendedNext).action) === "comment-and-close-issue", + "v0.2 closeout verdict must close historical successful runs only after ancestor, Argo, and git mirror checks pass", + supersededCloseout, +); + +const unresolvedAncestorCloseout = v02CloseoutVerdict({ + ...supersededCloseoutStatus, + sourceHeads: { originHead: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", target: { sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", objectExists: true, ancestorOfOriginHead: null } }, +}); +assertCondition( + unresolvedAncestorCloseout.closeable === false + && unresolvedAncestorCloseout.state === "pending" + && JSON.stringify(unresolvedAncestorCloseout.pendingReasons).includes("ancestor-check-unresolved"), + "v0.2 closeout verdict must not close superseded targets when ancestor evidence is missing", + unresolvedAncestorCloseout, +); + +const pendingFlushCloseout = v02CloseoutVerdict({ + ...supersededCloseoutStatus, + targetValidation: { + ok: false, + state: "failed", + failures: [{ reason: "git-mirror-not-flushed", pendingFlush: true, githubInSync: false }], + }, + gitMirror: { + summary: { + pendingFlush: true, + githubInSync: false, + flushNeeded: true, + flushCommand: "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm", + }, + }, +}); +assertCondition( + pendingFlushCloseout.closeable === false + && pendingFlushCloseout.state === "pending" + && String(record(pendingFlushCloseout.recommendedNext).action) === "flush-git-mirror" + && String(record(pendingFlushCloseout.recommendedNext).command).includes("git-mirror flush --confirm --wait"), + "v0.2 closeout verdict must turn pendingFlush into an explicit controlled flush command", + pendingFlushCloseout, +); + +const cdSummaryWithFields = summarizeV02CdStatus({ + pipelineRun: { pipelineRun: "hwlab-v02-ci-poll-aaaaaaaaaaaa", status: "True", reason: "Completed" }, + targetValidation: { state: "passed", ok: true, failures: [] }, + gitMirror: { summary: { pendingFlush: false, githubInSync: true } }, + argo: { fields: { syncStatus: "Synced", health: "Healthy" } }, + webAssets: { ok: true }, + planArtifacts: { buildServices: ["hwlab-cloud-api"], reusedServices: ["hwlab-cloud-web"], rolloutServices: ["hwlab-cloud-api"] }, +}); +assertCondition( + cdSummaryWithFields.argoSync === "Synced" + && cdSummaryWithFields.argoHealth === "Healthy", + "v0.2 CD summary must read Argo sync/health from status.argo.fields", + cdSummaryWithFields, +); +const activeRunSummary = activeV02PipelineRuns({ + activePipelineRuns: [ + { name: "hwlab-v02-ci-poll-aaaaaaaaaaaa", status: "Unknown" }, + { name: "hwlab-v02-ci-poll-bbbbbbbbbbbb", status: "True" }, + ], +}); +assertCondition( + activeRunSummary.length === 1 && activeRunSummary[0]?.name === "hwlab-v02-ci-poll-aaaaaaaaaaaa", + "v0.2 active PipelineRun helper must accept the status.activePipelineRuns array shape", + activeRunSummary, +); + const falseGreenPassed = v02FalseGreenGuard({ sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pipelineRun: { exists: true, status: "True" }, @@ -570,12 +676,14 @@ console.log(JSON.stringify({ "trigger-current can decide whether v0.2 git mirror pre-sync is required", "v0.2 git mirror pre-sync marker only reuses fresh same-commit post-observation markers", "git mirror status exposes source and gitops GitHub sync state plus controlled flush command", - "v0.2 control-plane status help exposes targeted PipelineRun/source-commit inspection", + "v0.2 control-plane status and closeout expose targeted PipelineRun/source-commit inspection", "v0.2 control-plane render uses an isolated temp clone from a CI/CD dedicated bare repo", "v0.2 control-plane refresh marker only reuses recent same-contract refreshes", "trigger-current rechecks latest v0.2 head before reusing an existing PipelineRun", "trigger-current fast snapshot parses source head and PipelineRun status in one G14:k3s route while latest head is checked locally", "v0.2 status alignment reports stale-success without coupling CI to dirty workspace state", + "v0.2 closeout verdicts distinguish passed, superseded, pending flush, and unresolved ancestor states", + "v0.2 CD summary and active run helpers expose Argo fields plus concurrent PipelineRun visibility", "v0.2 PipelineRun service matrix excludes hwlab-cli", "v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance", "v0.2 status warns on slow build TaskRuns", diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 0cdb74cd..c014bb0b 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -92,7 +92,7 @@ interface G14RecordRolloutOptions { } interface G14ControlPlaneOptions { - action: "status" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration"; + action: "status" | "closeout" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration"; lane: "v02" | "g14" | "all"; dryRun: boolean; confirm: boolean; @@ -193,7 +193,7 @@ interface CiPipelineMetrics { export interface V02PrCommentInput { pr: OpenPullRequest; phase: string; - state: "waiting-ci" | "blocked" | "merge-failed" | "cd-started" | "cd-succeeded" | "cd-failed" | "cd-timeout" | "cd-blocked"; + state: "waiting-ci" | "blocked" | "merge-failed" | "cd-started" | "cd-succeeded" | "cd-superseded" | "cd-failed" | "cd-timeout" | "cd-blocked"; startedAt: string; observedAt: string; elapsedSeconds: number | null; @@ -281,13 +281,14 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { const actionRaw = rawAction === "rerun-current" ? "trigger-current" : rawAction; if ( actionRaw !== "status" && + actionRaw !== "closeout" && actionRaw !== "apply" && actionRaw !== "trigger-current" && actionRaw !== "cleanup-runs" && actionRaw !== "cleanup-released-pvs" && actionRaw !== "runtime-migration" ) { - throw new Error("control-plane usage: status|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); + throw new Error("control-plane usage: status|closeout|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); } const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02"); if (actionRaw === "cleanup-runs") { @@ -295,7 +296,7 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { } else if (actionRaw === "cleanup-released-pvs") { if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane"); } else if (lane !== "v02") { - throw new Error("control-plane status/apply/trigger-current/runtime-migration currently requires --lane v02"); + throw new Error("control-plane status/closeout/apply/trigger-current/runtime-migration currently requires --lane v02"); } const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); @@ -305,17 +306,20 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { if (allowLiveDbRead && actionRaw !== "runtime-migration") throw new Error("--allow-live-db-read is only valid for control-plane runtime-migration"); const sourceCommitRaw = optionValue(args, "--source-commit"); const pipelineRunRaw = optionValue(args, "--pipeline-run"); - if ((sourceCommitRaw !== undefined || pipelineRunRaw !== undefined) && actionRaw !== "status") { - throw new Error("--source-commit and --pipeline-run are only valid for control-plane status"); + if ((sourceCommitRaw !== undefined || pipelineRunRaw !== undefined) && actionRaw !== "status" && actionRaw !== "closeout") { + throw new Error("--source-commit and --pipeline-run are only valid for control-plane status/closeout"); } - if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) throw new Error("control-plane status accepts only one of --source-commit or --pipeline-run"); + if (actionRaw === "closeout" && sourceCommitRaw === undefined && pipelineRunRaw === undefined) { + throw new Error("control-plane closeout requires --source-commit or --pipeline-run hwlab-v02-ci-poll-"); + } + if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) throw new Error("control-plane status/closeout accepts only one of --source-commit or --pipeline-run"); return { action: actionRaw, lane, confirm, wait: args.includes("--wait"), allowLiveDbRead, - dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, + dryRun: actionRaw === "status" || actionRaw === "closeout" ? true : explicitDryRun || !confirm, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(args, "--limit", 20, 200), @@ -436,6 +440,15 @@ function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } +function fieldBoolean(value: unknown): boolean | null { + if (value === true || value === false) return value; + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + if (normalized === "true" || normalized === "yes" || normalized === "1") return true; + if (normalized === "false" || normalized === "no" || normalized === "0") return false; + return null; +} + function nested(value: unknown, keys: string[]): unknown { let current = value; for (const key of keys) current = record(current)[key]; @@ -1003,7 +1016,7 @@ function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): "section statusTarget printf 'mode\\t%s\\nsourceCommit\\t%s\\npipelineRun\\t%s\\n' \"$target_mode\" \"$source_commit\" \"$pipeline_run\"", "section sourceCommit printf '%s\\n' \"$source_commit\"", "section pipelineRunName printf '%s\\n' \"$pipeline_run\"", - `section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())}`, + `section sourceHeads sh -c ${shellQuote(v02SourceHeadsProbeScript())} sh "$source_commit"`, "section queryNow date -u +%Y-%m-%dT%H:%M:%SZ", `section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`, `section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`, @@ -1022,14 +1035,33 @@ function v02ControlPlaneStatusBundle(target: V02ControlPlaneStatusTarget = {}): function v02SourceHeadsProbeScript(): string { return [ "set +e", + "target_source_commit=${1:-}", `cicd_repo=${shellQuote(V02_CICD_REPO)}`, `workspace=${shellQuote(V02_WORKSPACE)}`, "rev_cicd() { git --git-dir=\"$cicd_repo\" rev-parse \"$1\" 2>/dev/null || true; }", "rev_workspace() { git -C \"$workspace\" rev-parse \"$1\" 2>/dev/null || true; }", + "origin_head=$(rev_cicd refs/remotes/origin/v0.2)", "printf 'cicdRepo\\t%s\\n' \"$cicd_repo\"", "printf 'cicdRepoExists\\t%s\\n' \"$([ -d \"$cicd_repo/objects\" ] && printf yes || printf no)\"", - "printf 'cicdSourceHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"", - "printf 'originHead\\t%s\\n' \"$(rev_cicd refs/remotes/origin/v0.2)\"", + "printf 'cicdSourceHead\\t%s\\n' \"$origin_head\"", + "printf 'originHead\\t%s\\n' \"$origin_head\"", + "printf 'targetSourceCommit\\t%s\\n' \"$target_source_commit\"", + "target_object_exists=", + "target_ancestor=", + "target_ancestor_exit=", + "if [ -n \"$target_source_commit\" ] && git --git-dir=\"$cicd_repo\" cat-file -e \"$target_source_commit^{commit}\" 2>/dev/null; then", + " target_object_exists=true", + " if [ -n \"$origin_head\" ]; then", + " git --git-dir=\"$cicd_repo\" merge-base --is-ancestor \"$target_source_commit\" \"$origin_head\" >/dev/null 2>&1", + " target_ancestor_exit=$?", + " if [ \"$target_ancestor_exit\" = 0 ]; then target_ancestor=true; elif [ \"$target_ancestor_exit\" = 1 ]; then target_ancestor=false; else target_ancestor=unknown; fi", + " fi", + "elif [ -n \"$target_source_commit\" ]; then", + " target_object_exists=false", + "fi", + "printf 'targetObjectExists\\t%s\\n' \"$target_object_exists\"", + "printf 'targetAncestorOfOriginHead\\t%s\\n' \"$target_ancestor\"", + "printf 'targetAncestorExitCode\\t%s\\n' \"$target_ancestor_exit\"", "printf 'workspacePath\\t%s\\n' \"$workspace\"", "printf 'workspaceBranch\\t%s\\n' \"$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)\"", "printf 'workspaceHead\\t%s\\n' \"$(rev_workspace HEAD)\"", @@ -1387,12 +1419,21 @@ function keyValueLinesFromText(text: string): Record { function v02SourceHeadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record { const fields = keyValueLinesFromText(text); const dirtyCount = numericField(fields.workspaceDirtyCount); + const targetSourceCommit = fields.targetSourceCommit || null; return { ok: commandOk, cicdRepo: fields.cicdRepo || V02_CICD_REPO, cicdRepoExists: fields.cicdRepoExists === "yes", cicdSourceHead: fields.cicdSourceHead || null, originHead: fields.originHead || null, + target: { + sourceCommit: targetSourceCommit, + objectExists: fieldBoolean(fields.targetObjectExists), + ancestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead), + ancestorExitCode: numericField(fields.targetAncestorExitCode), + }, + targetObjectExists: fieldBoolean(fields.targetObjectExists), + targetAncestorOfOriginHead: fieldBoolean(fields.targetAncestorOfOriginHead), workspace: { path: fields.workspacePath || V02_WORKSPACE, branch: fields.workspaceBranch || null, @@ -1622,6 +1663,200 @@ export function v02LatestOnlyTargetValidation(input: { }; } +function v02CloseoutRecommendedNext(input: { + closeable: boolean; + blockedReasons: string[]; + pendingReasons: string[]; + statusCommand: string; + sourceCommit: string | null; + gitMirrorSummary: Record; +}): Record { + const rawFlushCommand = stringOrNull(input.gitMirrorSummary.flushCommand) ?? "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm"; + const flushCommand = rawFlushCommand.includes("--wait") ? rawFlushCommand : `${rawFlushCommand} --wait`; + if (input.closeable) { + return { + action: "comment-and-close-issue", + command: input.sourceCommit === null + ? input.statusCommand + : `bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit ${input.sourceCommit}`, + }; + } + if (input.pendingReasons.includes("git-mirror-not-flushed")) { + return { action: "flush-git-mirror", command: flushCommand }; + } + if (input.pendingReasons.includes("argo-not-healthy")) { + return { action: "wait-and-recheck", command: input.statusCommand }; + } + if (input.pendingReasons.includes("pipeline-run-not-final")) { + return { action: "wait-pipelinerun", command: input.statusCommand }; + } + if (input.blockedReasons.includes("target-pipelinerun-missing")) { + return { action: "trigger-current-or-inspect-target", command: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }; + } + return { action: "inspect-status", command: input.statusCommand }; +} + +export function v02CloseoutVerdict(status: Record): Record { + const statusTarget = record(status.statusTarget); + const targetMode = stringOrNull(statusTarget.mode) ?? "latest-source-head"; + const sourceCommit = stringOrNull(status.sourceCommit) ?? stringOrNull(statusTarget.sourceCommit); + const pipelineRun = record(status.pipelineRun); + const targetValidation = record(status.targetValidation); + const targetValidationState = stringOrNull(targetValidation.state); + const argoFields = record(record(status.argo).fields); + const gitMirrorSummary = record(record(status.gitMirror).summary); + const sourceHeadsTarget = record(record(status.sourceHeads).target); + const commitAlignment = record(status.commitAlignment); + const latestHead = stringOrNull(commitAlignment.originHead) ?? stringOrNull(record(status.sourceHeads).originHead); + const ancestorOfLatest = sourceHeadsTarget.ancestorOfOriginHead ?? status.sourceHeadsTargetAncestorOfOriginHead ?? null; + const activePipelineRuns = Array.isArray(status.activePipelineRuns) + ? status.activePipelineRuns.map((item) => record(item)) + : []; + const pipelineStatus = stringOrNull(pipelineRun.status); + const pipelineExists = pipelineRun.exists !== false && Object.keys(pipelineRun).length > 0; + const pendingReasons: string[] = []; + const blockedReasons: string[] = []; + if (targetMode === "latest-source-head") { + pendingReasons.push("target-not-explicit"); + } + if (!pipelineExists) { + blockedReasons.push("target-pipelinerun-missing"); + } else if (pipelineStatus === "Unknown" || pipelineStatus === null) { + pendingReasons.push("pipeline-run-not-final"); + } else if (pipelineStatus === "False") { + blockedReasons.push("pipeline-run-failed"); + } + if (targetValidationState !== "passed" && targetValidationState !== "superseded") { + const failures = Array.isArray(targetValidation.failures) ? targetValidation.failures.map((item) => record(item)) : []; + for (const failure of failures) { + const reason = stringOrNull(failure.reason); + if (reason === "git-mirror-not-flushed") pendingReasons.push(reason); + else if (reason === "argo-not-synced-healthy") pendingReasons.push("argo-not-healthy"); + else if (reason === "pipeline-run-not-succeeded" && pipelineStatus === "Unknown") pendingReasons.push("pipeline-run-not-final"); + else if (reason !== null) blockedReasons.push(reason); + } + if (failures.length === 0 && targetMode !== "latest-source-head") blockedReasons.push("target-validation-not-passed"); + } + if (gitMirrorSummary.pendingFlush !== false || gitMirrorSummary.githubInSync !== true) { + pendingReasons.push("git-mirror-not-flushed"); + } + if (argoFields.syncStatus !== "Synced" || argoFields.health !== "Healthy") { + pendingReasons.push("argo-not-healthy"); + } + if (targetValidationState === "superseded" && ancestorOfLatest !== true) { + if (ancestorOfLatest === false) blockedReasons.push("target-not-ancestor-of-latest-head"); + else pendingReasons.push("ancestor-check-unresolved"); + } + const uniquePendingReasons = Array.from(new Set(pendingReasons)); + const uniqueBlockedReasons = Array.from(new Set(blockedReasons)); + const terminalState = targetValidationState === "passed" || targetValidationState === "superseded"; + const state = uniqueBlockedReasons.length > 0 + ? "blocked" + : uniquePendingReasons.length > 0 + ? "pending" + : terminalState + ? targetValidationState + : "blocked"; + const closeable = (state === "passed" || state === "superseded") && uniquePendingReasons.length === 0 && uniqueBlockedReasons.length === 0; + const rawStatusCommand = String(status.command ?? ( + sourceCommit === null + ? "bun scripts/cli.ts hwlab g14 control-plane status --lane v02" + : `bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}` + )); + const statusCommand = rawStatusCommand.startsWith("bun ") ? rawStatusCommand : `bun scripts/cli.ts ${rawStatusCommand}`; + const ancestorCheck = { + required: state === "superseded", + sourceCommit, + latestHead, + objectExists: sourceHeadsTarget.objectExists ?? null, + ancestorOfLatest, + exitCode: sourceHeadsTarget.ancestorExitCode ?? null, + command: sourceCommit === null + ? null + : `trans G14:/root/hwlab-v02 script -- 'git fetch origin v0.2 && git merge-base --is-ancestor ${sourceCommit} origin/v0.2'`, + }; + const recommendedNext = v02CloseoutRecommendedNext({ + closeable, + blockedReasons: uniqueBlockedReasons, + pendingReasons: uniquePendingReasons, + statusCommand, + sourceCommit, + gitMirrorSummary, + }); + const summary = closeable + ? state === "superseded" + ? `closeout ready: target ${shortSha(sourceCommit ?? "")} passed and is superseded by latest v0.2 head ${shortSha(latestHead ?? "")}` + : `closeout ready: target ${shortSha(sourceCommit ?? "")} passed on v0.2` + : uniquePendingReasons.length > 0 + ? `closeout pending: ${uniquePendingReasons.join(",")}` + : `closeout blocked: ${uniqueBlockedReasons.join(",") || "unknown"}`; + const issueCommentMarkdown = [ + `v0.2 closeout verdict: \`${state}\` (${summary})`, + "", + `- target sourceCommit: \`${sourceCommit ?? "n/a"}\``, + `- target PipelineRun: \`${String(pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? "n/a")}\`; status=\`${pipelineStatus ?? "n/a"}\`; reason=\`${String(pipelineRun.reason ?? "n/a")}\``, + `- latest head: \`${latestHead ?? "n/a"}\`; ancestorOfLatest=\`${String(ancestorCheck.ancestorOfLatest ?? "n/a")}\``, + `- targetValidation: \`${targetValidationState ?? "n/a"}\`; pending=\`${uniquePendingReasons.join(",") || "none"}\`; blocked=\`${uniqueBlockedReasons.join(",") || "none"}\``, + `- Git mirror: pendingFlush=\`${String(gitMirrorSummary.pendingFlush ?? "n/a")}\`, githubInSync=\`${String(gitMirrorSummary.githubInSync ?? "n/a")}\`, flushCommand=\`${String(gitMirrorSummary.flushCommand ?? "none")}\``, + `- Argo: sync=\`${String(argoFields.syncStatus ?? "n/a")}\`, health=\`${String(argoFields.health ?? "n/a")}\`, revision=\`${String(argoFields.syncRevision ?? "n/a")}\``, + `- active v0.2 PipelineRuns: \`${activePipelineRuns.length}\``, + `- recommended next: \`${String(recommendedNext.action ?? "inspect-status")}\` -> \`${String(recommendedNext.command ?? statusCommand)}\``, + ].join("\n"); + return { + ok: closeable, + closeable, + state, + summary, + targetMode, + sourceCommit, + latestHead, + pipelineRun: { + name: pipelineRun.pipelineRun ?? pipelineRun.name ?? statusTarget.pipelineRun ?? null, + exists: pipelineExists, + status: pipelineStatus, + reason: pipelineRun.reason ?? null, + }, + targetValidation: { + state: targetValidationState, + ok: targetValidation.ok ?? null, + failures: Array.isArray(targetValidation.failures) ? targetValidation.failures.slice(0, 10) : [], + }, + latestOnly: { + superseded: targetValidation.latestOnlySuperseded === true || state === "superseded", + reasons: stringArray(targetValidation.latestOnlyReasons), + ancestorCheck, + }, + gitMirror: { + pendingFlush: gitMirrorSummary.pendingFlush ?? null, + githubInSync: gitMirrorSummary.githubInSync ?? null, + flushNeeded: gitMirrorSummary.flushNeeded ?? null, + flushCommand: gitMirrorSummary.flushCommand ?? null, + lastSync: gitMirrorSummary.lastSync ?? null, + lastWrite: gitMirrorSummary.lastWrite ?? null, + lastFlush: gitMirrorSummary.lastFlush ?? null, + }, + argo: { + syncStatus: argoFields.syncStatus ?? null, + health: argoFields.health ?? null, + syncRevision: argoFields.syncRevision ?? null, + targetRevision: argoFields.targetRevision ?? null, + path: argoFields.path ?? null, + }, + runtime: { + webAssetsOk: nested(status, ["webAssets", "ok"]) ?? null, + webAssetsSummary: nested(status, ["webAssets", "summary"]) ?? null, + apiRevision: nested(status, ["webAssets", "apiRevision"]) ?? nested(status, ["commitAlignment", "apiRevision"]) ?? null, + serviceSourceCommits: nested(status, ["commitAlignment", "serviceSourceCommits"]) ?? null, + }, + activePipelineRuns, + recentPipelineRuns: status.recentPipelineRuns ?? null, + pendingReasons: uniquePendingReasons, + blockedReasons: uniqueBlockedReasons, + recommendedNext, + issueCommentMarkdown, + }; +} + export function v02CommitAlignment(input: { expectedSourceHead: string | null; sourceHeads: Record; @@ -2533,6 +2768,7 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record commitAlignment, targetValidation: targetValidationBase, }); + const sourceHeadsTarget = record(sourceHeads.target); const falseGreenGuard = targetValidation.state === "superseded" ? { ok: null, @@ -2560,7 +2796,7 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record : targetMode === "source-commit" ? "explicit --source-commit" : "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated"; - return { + const status: Record = { ok: baseOk && targetPipelineRunOk && (!strictHeadAlignment || commitAlignment.aligned !== false), command: statusCommand, lane: "v02", @@ -2571,6 +2807,7 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record sourceCommit, pipelineRun, strictHeadAlignment, + ancestorOfLatest: sourceHeadsTarget.ancestorOfOriginHead ?? null, note: strictHeadAlignment ? "default status validates the latest v0.2 source head alignment" : "targeted status inspects the requested PipelineRun/source commit without failing merely because origin/v0.2 advanced later", @@ -2591,6 +2828,7 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record commitAlignment, targetValidation, sourceHeads, + sourceHeadsTargetAncestorOfOriginHead: sourceHeadsTarget.ancestorOfOriginHead ?? null, gitMirror, controlPlane: { ok: shellSectionOk(controlPlane), @@ -2626,13 +2864,29 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record ? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger" : "no active v02 PipelineRun observed", }; + return { + ...status, + closeout: v02CloseoutVerdict(status), + }; } function runV02ControlPlane(options: G14ControlPlaneOptions): Record { if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options); - if (options.action === "status" && options.pipelineRun !== undefined) return v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run" }); - if (options.action === "status" && options.sourceCommit !== undefined) return v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit" }); + if ((options.action === "status" || options.action === "closeout") && options.pipelineRun !== undefined) { + const status = v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run" }); + if (options.action === "closeout") return v02ControlPlaneCloseout(status); + return status; + } + if ((options.action === "status" || options.action === "closeout") && options.sourceCommit !== undefined) { + const status = v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit" }); + if (options.action === "closeout") return v02ControlPlaneCloseout(status); + return status; + } + if (options.action === "closeout") { + const status = v02ControlPlaneStatus(); + return v02ControlPlaneCloseout(status); + } const snapshot = options.action === "trigger-current" ? getV02TriggerSnapshot() : null; const latestHead = options.action === "trigger-current" ? resolveV02LatestRemoteHead() : null; const head = snapshot === null @@ -4760,7 +5014,7 @@ function v02PrConflictState(summary: Record): string { return "clear-or-unknown"; } -function summarizeV02CdStatus(status: Record): Record { +export function summarizeV02CdStatus(status: Record): Record { const pipelineRun = record(status.pipelineRun); const targetValidation = record(status.targetValidation); const gitMirror = record(status.gitMirror); @@ -4779,8 +5033,8 @@ function summarizeV02CdStatus(status: Record): Record): boolean { return state === "passed" || state === "superseded"; } +function v02ControlPlaneCloseout(status: Record): Record { + const closeout = record(status.closeout); + const sourceCommit = stringOrNull(status.sourceCommit); + const pipelineRun = stringOrNull(nested(status, ["statusTarget", "pipelineRun"])) ?? stringOrNull(nested(status, ["pipelineRun", "pipelineRun"])); + const command = sourceCommit !== null + ? `hwlab g14 control-plane closeout --lane v02 --source-commit ${sourceCommit}` + : pipelineRun !== null + ? `hwlab g14 control-plane closeout --lane v02 --pipeline-run ${pipelineRun}` + : "hwlab g14 control-plane closeout --lane v02"; + return { + ok: closeout.closeable === true, + command, + lane: "v02", + closeout, + target: status.statusTarget ?? null, + sourceCommit: status.sourceCommit ?? null, + pipelineRun: status.pipelineRun ?? null, + targetValidation: status.targetValidation ?? null, + latestHead: nested(status, ["commitAlignment", "originHead"]) ?? nested(status, ["sourceHeads", "originHead"]) ?? null, + gitMirror: record(record(status.gitMirror).summary), + argo: record(status.argo).fields ?? null, + activePipelineRuns: status.activePipelineRuns ?? [], + recommendedNext: closeout.recommendedNext ?? null, + issueCommentMarkdown: closeout.issueCommentMarkdown ?? null, + }; +} + function v02PipelineSucceeded(status: Record): boolean { return String(nested(status, ["pipelineRun", "status"]) ?? "") === "True"; } @@ -4804,9 +5085,12 @@ function v02CdFailed(status: Record): boolean { return pipelineStatus === "False"; } -function activeV02PipelineRuns(status: Record): Record[] { - const active = record(status.activePipelineRuns); - const items = Array.isArray(active.items) ? active.items : []; +export function activeV02PipelineRuns(status: Record): Record[] { + const items = Array.isArray(status.activePipelineRuns) + ? status.activePipelineRuns + : Array.isArray(record(status.activePipelineRuns).items) + ? record(status.activePipelineRuns).items + : []; return items.map((item) => record(item)).filter((item) => String(item.status ?? "") === "Unknown"); } @@ -5352,6 +5636,8 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 control-plane status --lane v02", "bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --pipeline-run hwlab-v02-ci-poll-", "bun scripts/cli.ts hwlab g14 control-plane status --lane v02 --source-commit ", + "bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --pipeline-run hwlab-v02-ci-poll-", + "bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit ", "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm", @@ -5376,7 +5662,7 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm", "bun scripts/cli.ts job status --tail-bytes 30000", ], - description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.", + description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror maintenance, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/closeout/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, historical PipelineRun/source-commit closeout verdicts, GitOps mirror flush state, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob.", defaults: { repo: HWLAB_REPO, base: G14_SOURCE_BRANCH,