fix: formalize pac closeout guidance
This commit is contained in:
Codex
@@ -356,6 +356,52 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio
|
||||
const deadline = startedAt + budgetSeconds * 1000;
|
||||
const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds);
|
||||
let controlResult: Record<string, unknown> | null = null;
|
||||
const inCluster = Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT);
|
||||
if (!inCluster && !options.manualRecovery) {
|
||||
const next = publishCurrentNext(state);
|
||||
const result = {
|
||||
ok: false,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
mode: "debug-step-only-blocked",
|
||||
mutation: false,
|
||||
deliverySource: "auto-pac-required",
|
||||
specRef: SPEC_REF,
|
||||
source: state.sourceHead,
|
||||
image: state.image,
|
||||
pipelineRun: sentinelPipelineRunName(state, options.rerun),
|
||||
budget,
|
||||
validationPlan: publishCurrentHealthValidationPlan(state),
|
||||
stageBudgets: publishCurrentStageBudgets(state),
|
||||
warnings: [
|
||||
"operator-side publish-current is debug/test/recovery only; formal delivery must use platform-infra pipelines-as-code closeout/status/history.",
|
||||
],
|
||||
blocker: {
|
||||
code: "sentinel-publish-current-debug-step-only",
|
||||
reason: "this operator-side command no longer performs formal Web sentinel delivery; use PaC closeout for automatic branch following",
|
||||
valuesRedacted: true,
|
||||
},
|
||||
recoveryNext: {
|
||||
reason: "manual recovery is only allowed after PaC is terminally failed, half-applied, or explicitly over budget",
|
||||
pipelineRun: null,
|
||||
digestRef: null,
|
||||
gitopsCommit: null,
|
||||
pacCloseout: next.pacCloseout,
|
||||
publishCurrent: next.manualRecovery,
|
||||
nextStatus: next.pacStatus,
|
||||
gitMirrorStatus: next.gitMirrorStatus,
|
||||
gitMirrorSync: null,
|
||||
gitMirrorFlush: null,
|
||||
controlPlaneApply: null,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
next,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(false, command, renderPublishCurrentResult(result));
|
||||
}
|
||||
sentinelProgressEvent("sentinel.publish.progress", {
|
||||
phase: "publish-current-start",
|
||||
status: "running",
|
||||
@@ -365,7 +411,7 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
});
|
||||
if (process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) {
|
||||
if (inCluster) {
|
||||
const publish = runSentinelPublishJob(state, true, Math.max(1, remainingBudgetSeconds()), options.rerun);
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
const ok = record(publish).ok === true;
|
||||
@@ -1049,8 +1095,13 @@ function publishCurrentNext(state: SentinelCicdState): Record<string, string> {
|
||||
const node = state.spec.nodeId;
|
||||
const lane = state.spec.lane;
|
||||
const suffix = sentinelCliSuffix(state);
|
||||
const consumer = sentinelPacConsumerId(state);
|
||||
const sourceCommit = state.sourceHead.commit === null ? "" : ` --source-commit ${state.sourceHead.commit}`;
|
||||
return {
|
||||
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`,
|
||||
pacCloseout: `bun scripts/cli.ts platform-infra pipelines-as-code closeout --target ${node} --consumer ${consumer}${sourceCommit} --wait`,
|
||||
pacStatus: `bun scripts/cli.ts platform-infra pipelines-as-code status --target ${node} --consumer ${consumer}`,
|
||||
pacHistory: `bun scripts/cli.ts platform-infra pipelines-as-code history --target ${node} --consumer ${consumer} --limit 10`,
|
||||
manualRecovery: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait --manual-recovery --reason <reason>`,
|
||||
controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
|
||||
dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`,
|
||||
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`,
|
||||
@@ -1059,6 +1110,10 @@ function publishCurrentNext(state: SentinelCicdState): Record<string, string> {
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelPacConsumerId(state: SentinelCicdState): string {
|
||||
return `sentinel-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}`;
|
||||
}
|
||||
|
||||
function renderSentinelManifests(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
sentinelId: string,
|
||||
@@ -1963,7 +2018,7 @@ function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknow
|
||||
? "sentinel-cadence-not-ready"
|
||||
: "sentinel-control-plane-observed-not-ready";
|
||||
const reason = code === "sentinel-publish-half-state-registry-missing"
|
||||
? "source mirror contains the selected commit, but the expected registry tag is missing; retry publish-current so Tekton builds/pushes the image and advances GitOps/runtime"
|
||||
? "source mirror contains the selected commit, but the expected registry tag is missing; inspect PaC closeout/history before considering explicit manual recovery"
|
||||
: code === "sentinel-gitops-manifest-not-updated"
|
||||
? "registry contains the selected image, but GitOps manifest is not yet updated to a digest-pinned runtime image"
|
||||
: code === "sentinel-runtime-not-aligned"
|
||||
@@ -1998,7 +2053,7 @@ function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknow
|
||||
runtime: `ready=${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"} image=${short(runtimeDeployment.image)} expected=${short(runtimeDeployment.expectedImage)}`,
|
||||
pipelineRun,
|
||||
warning: code === "sentinel-publish-half-state-registry-missing"
|
||||
? `source mirror already exposes ${short(state.sourceHead.commit)} but registry tag ${state.image.tag} is missing; rerun ${next.publishCurrent} and then recheck ${next.controlPlaneStatus}.`
|
||||
? `source mirror already exposes ${short(state.sourceHead.commit)} but registry tag ${state.image.tag} is missing; run ${next.pacCloseout} and inspect ${next.pacHistory}.`
|
||||
: code === "sentinel-git-mirror-not-in-sync"
|
||||
? `runtime is aligned but git-mirror is not in sync; run ${next.gitMirrorSync} and then recheck ${next.controlPlaneStatus}.`
|
||||
: null,
|
||||
@@ -2008,8 +2063,9 @@ function sentinelObservedStatusDiagnosis(state: SentinelCicdState, value: unknow
|
||||
pipelineRun,
|
||||
digestRef: registryPresent ? expectedRuntimeImageFromRegistry(state, record(observed.registry)) : null,
|
||||
gitopsCommit: gitops.revision ?? null,
|
||||
publishCurrent: shouldRetryPublish ? next.publishCurrent : null,
|
||||
nextStatus: next.controlPlaneStatus,
|
||||
pacCloseout: next.pacCloseout,
|
||||
publishCurrent: shouldRetryPublish ? next.manualRecovery : null,
|
||||
nextStatus: next.pacStatus,
|
||||
gitMirrorStatus: next.gitMirrorStatus,
|
||||
gitMirrorSync: !gitMirrorReady ? next.gitMirrorSync : null,
|
||||
gitMirrorFlush: null,
|
||||
@@ -2389,8 +2445,9 @@ function installSentinelPublishInterruptHandler(state: SentinelCicdState, contex
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
recoveryNext: {
|
||||
status: next.controlPlaneStatus,
|
||||
retry: next.publishCurrent,
|
||||
pacCloseout: next.pacCloseout,
|
||||
pacStatus: next.pacStatus,
|
||||
retry: next.manualRecovery,
|
||||
gitMirrorStatus: next.gitMirrorStatus,
|
||||
gitMirrorSync: next.gitMirrorSync,
|
||||
gitMirrorFlush: next.gitMirrorFlush,
|
||||
@@ -2428,7 +2485,12 @@ function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelCont
|
||||
const node = state.spec.nodeId;
|
||||
const lane = state.spec.lane;
|
||||
const suffix = sentinelCliSuffix(state);
|
||||
const consumer = sentinelPacConsumerId(state);
|
||||
const sourceCommit = state.sourceHead.commit === null ? "" : ` --source-commit ${state.sourceHead.commit}`;
|
||||
return {
|
||||
pacCloseout: `bun scripts/cli.ts platform-infra pipelines-as-code closeout --target ${node} --consumer ${consumer}${sourceCommit} --wait`,
|
||||
pacStatus: `bun scripts/cli.ts platform-infra pipelines-as-code status --target ${node} --consumer ${consumer}`,
|
||||
pacHistory: `bun scripts/cli.ts platform-infra pipelines-as-code history --target ${node} --consumer ${consumer} --limit 10`,
|
||||
plan: `bun scripts/cli.ts web-probe sentinel control-plane plan --node ${node} --lane ${lane}${suffix} --dry-run`,
|
||||
status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
|
||||
image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`,
|
||||
@@ -2457,8 +2519,9 @@ function controlPlaneRecoveryNext(state: SentinelCicdState, ok: boolean, publish
|
||||
gitopsCommit: payload.gitopsCommit ?? null,
|
||||
flushMode: flushRecord.mode ?? null,
|
||||
observedReady: sentinelObservedReady(observedRecord),
|
||||
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
||||
nextStatus: next.status,
|
||||
pacCloseout: next.pacCloseout,
|
||||
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait --manual-recovery --reason <reason>`,
|
||||
nextStatus: next.pacStatus,
|
||||
gitMirrorStatus: next.gitMirrorStatus,
|
||||
gitMirrorSync: next.gitMirrorSync,
|
||||
gitMirrorFlush: next.gitMirrorFlush,
|
||||
|
||||
Reference in New Issue
Block a user