|
|
|
@@ -19,7 +19,7 @@ import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry,
|
|
|
|
|
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
|
|
|
|
import type { RenderedCliResult } from "./output";
|
|
|
|
|
import { probeSentinelDashboardBrowser, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
|
|
|
|
import { remainingSeconds, runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
|
|
|
|
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
|
|
|
|
|
|
|
|
|
export type WebProbeSentinelConfigAction = "plan" | "status";
|
|
|
|
|
export type WebProbeSentinelImageAction = "status" | "build";
|
|
|
|
@@ -368,8 +368,26 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
const budget = publishCurrentBudget(state);
|
|
|
|
|
const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds"));
|
|
|
|
|
const deadline = startedAt + budgetSeconds * 1000;
|
|
|
|
|
const remainingBudgetSeconds = () => remainingSeconds(deadline, budgetSeconds);
|
|
|
|
|
const controlResult = sentinelControlPlaneConfirmedResult(state, {
|
|
|
|
|
const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds);
|
|
|
|
|
let controlResult: Record<string, unknown> | null = null;
|
|
|
|
|
if (state.configReady && state.sourceHead.ok && remainingBudgetSeconds() >= 5) {
|
|
|
|
|
const registryProbe = probeImageRegistry(state, Math.max(1, Math.min(remainingBudgetSeconds(), 5)));
|
|
|
|
|
if (record(record(registryProbe).probe).present === true && remainingBudgetSeconds() >= 5) {
|
|
|
|
|
const preflightStartedAt = Date.now();
|
|
|
|
|
const preflightTimeoutSeconds = Math.max(1, Math.min(remainingBudgetSeconds(), 10));
|
|
|
|
|
const preflightObserved = withObservedWait(
|
|
|
|
|
collectSentinelObservedStatus(state, preflightTimeoutSeconds, undefined, true),
|
|
|
|
|
preflightStartedAt,
|
|
|
|
|
preflightTimeoutSeconds,
|
|
|
|
|
true,
|
|
|
|
|
);
|
|
|
|
|
if (sentinelObservedReady(preflightObserved)) {
|
|
|
|
|
controlResult = sentinelAlreadyCurrentControlResult(state, preflightObserved, Date.now() - preflightStartedAt);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const dashboardReserveSeconds = publishCurrentDashboardReserveSeconds(state);
|
|
|
|
|
controlResult ??= sentinelControlPlaneConfirmedResult(state, {
|
|
|
|
|
kind: "control-plane",
|
|
|
|
|
action: "trigger-current",
|
|
|
|
|
node: options.node,
|
|
|
|
@@ -378,7 +396,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
dryRun: false,
|
|
|
|
|
confirm: true,
|
|
|
|
|
wait: true,
|
|
|
|
|
timeoutSeconds: remainingBudgetSeconds(),
|
|
|
|
|
timeoutSeconds: Math.max(1, remainingBudgetSeconds() - dashboardReserveSeconds),
|
|
|
|
|
});
|
|
|
|
|
const dashboardRequired = publishCurrentDashboardRequired(state);
|
|
|
|
|
let dashboard: Record<string, unknown>;
|
|
|
|
@@ -408,8 +426,8 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
node: state.spec.nodeId,
|
|
|
|
|
lane: state.spec.lane,
|
|
|
|
|
sentinelId: state.sentinelId,
|
|
|
|
|
mode: "confirm-wait",
|
|
|
|
|
mutation: true,
|
|
|
|
|
mode: controlResult.mode === "already-current" ? "already-current" : "confirm-wait",
|
|
|
|
|
mutation: controlResult.mutation !== false,
|
|
|
|
|
specRef: SPEC_REF,
|
|
|
|
|
source: state.sourceHead,
|
|
|
|
|
image: state.image,
|
|
|
|
@@ -431,6 +449,120 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
return rendered(ok, command, renderPublishCurrentResult(result));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function strictRemainingSeconds(deadline: number, cap: number): number {
|
|
|
|
|
return Math.max(0, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function withObservedWait(observed: SentinelObservedStatus, startedAt: number, timeoutSeconds: number, includeGitMirror: boolean): SentinelObservedStatus {
|
|
|
|
|
const elapsedMs = Date.now() - startedAt;
|
|
|
|
|
return {
|
|
|
|
|
...observed,
|
|
|
|
|
wait: {
|
|
|
|
|
polls: 1,
|
|
|
|
|
elapsedMs,
|
|
|
|
|
timeoutMs: Math.max(1, timeoutSeconds) * 1000,
|
|
|
|
|
ready: sentinelObservedReady(observed),
|
|
|
|
|
includeGitMirror,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: SentinelObservedStatus, elapsedMs: number): Record<string, unknown> {
|
|
|
|
|
const registryProbe = record(record(observed.registry).probe);
|
|
|
|
|
const gitops = record(observed.gitops);
|
|
|
|
|
const argo = record(observed.argo);
|
|
|
|
|
const digest = text(registryProbe.digest);
|
|
|
|
|
const digestRef = digest === "-" ? null : `${state.image.repository}@${digest}`;
|
|
|
|
|
const stageTimings = {
|
|
|
|
|
sourceFetchMs: 0,
|
|
|
|
|
monitorWebVerifyMs: 0,
|
|
|
|
|
imageBuildMs: 0,
|
|
|
|
|
gitopsMs: 0,
|
|
|
|
|
totalMs: 0,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
return {
|
|
|
|
|
ok: true,
|
|
|
|
|
command: "web-probe sentinel control-plane trigger-current",
|
|
|
|
|
node: state.spec.nodeId,
|
|
|
|
|
lane: state.spec.lane,
|
|
|
|
|
mode: "already-current",
|
|
|
|
|
mutation: false,
|
|
|
|
|
specRef: SPEC_REF,
|
|
|
|
|
source: state.sourceHead,
|
|
|
|
|
image: state.image,
|
|
|
|
|
pipelineRun: "already-current",
|
|
|
|
|
gitops: {
|
|
|
|
|
path: stringAt(state.cicd, "gitopsPath"),
|
|
|
|
|
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
|
|
|
|
manifestObjects: state.manifests.length,
|
|
|
|
|
manifestSha256: state.manifestSha256,
|
|
|
|
|
},
|
|
|
|
|
argo: {
|
|
|
|
|
namespace: stringAt(state.cicd, "argo.namespace"),
|
|
|
|
|
projectName: stringAt(state.cicd, "argo.projectName"),
|
|
|
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
|
|
|
},
|
|
|
|
|
validation: {
|
|
|
|
|
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
|
|
|
|
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
|
|
|
|
controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state),
|
|
|
|
|
quickVerifyMode: "manual-validate",
|
|
|
|
|
automaticSecondPath: false,
|
|
|
|
|
},
|
|
|
|
|
manifests: {
|
|
|
|
|
objects: manifestObjectSummary(state.manifests),
|
|
|
|
|
sha256: state.manifestSha256,
|
|
|
|
|
},
|
|
|
|
|
sourceMirrorSync: { ok: true, phase: "already-current", jobName: "-", elapsedMs: 0, valuesRedacted: true },
|
|
|
|
|
publish: {
|
|
|
|
|
ok: true,
|
|
|
|
|
phase: "already-current",
|
|
|
|
|
resourceKind: "PipelineRun",
|
|
|
|
|
jobName: "already-current",
|
|
|
|
|
elapsedMs: 0,
|
|
|
|
|
payload: {
|
|
|
|
|
ok: true,
|
|
|
|
|
status: "already-current",
|
|
|
|
|
sourceCommit: state.sourceHead.commit,
|
|
|
|
|
imageRef: state.image.ref,
|
|
|
|
|
digestRef,
|
|
|
|
|
gitopsCommit: gitops.revision ?? argo.revision ?? null,
|
|
|
|
|
stageTimings,
|
|
|
|
|
completedStages: ["already-current"],
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
},
|
|
|
|
|
diagnostics: {
|
|
|
|
|
domain: "publish",
|
|
|
|
|
resourceKind: "PipelineRun",
|
|
|
|
|
pipelineRun: "already-current",
|
|
|
|
|
currentPhase: "already-current",
|
|
|
|
|
completedStages: ["already-current:skipped"],
|
|
|
|
|
stageTimings,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
},
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
},
|
|
|
|
|
flush: { ok: true, skipped: true, reason: "already-current", valuesRedacted: true },
|
|
|
|
|
runtimeSecretsApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
|
|
|
|
|
publicExposureApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
|
|
|
|
|
argoApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true },
|
|
|
|
|
observed,
|
|
|
|
|
targetValidation: null,
|
|
|
|
|
elapsedMs,
|
|
|
|
|
warnings: [
|
|
|
|
|
"publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used dashboard verification only.",
|
|
|
|
|
...sentinelObservedWarnings(observed),
|
|
|
|
|
...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)),
|
|
|
|
|
],
|
|
|
|
|
blocker: null,
|
|
|
|
|
recoveryNext: controlPlaneRecoveryNext(state, true, {}, { ok: true }, observed),
|
|
|
|
|
next: controlPlaneNext(state, "trigger-current"),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState {
|
|
|
|
|
const sentinel = resolveWebProbeSentinel(spec, sentinelId);
|
|
|
|
|
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
|
|
|
|
@@ -536,6 +668,7 @@ function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unk
|
|
|
|
|
imageBuildNetworkMode: monitorWebImageBuildNetworkMode(cicd),
|
|
|
|
|
imageBuildProxySource: stringAtNullable(cicd, "monitorWeb.imageBuild.proxySource") ?? "node.networkProfile.imageBuildProxy",
|
|
|
|
|
imageBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.imageBuild.contextIgnore") ?? "generated",
|
|
|
|
|
imageBuildState: monitorWebBuildkitStatePlan(cicd),
|
|
|
|
|
ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
@@ -547,6 +680,34 @@ function monitorWebImageBuildNetworkMode(cicd: Record<string, unknown>): "defaul
|
|
|
|
|
return value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function monitorWebBuildkitStatePlan(cicd: Record<string, unknown>): Record<string, unknown> {
|
|
|
|
|
const state = recordTarget(valueAtPath(cicd, "monitorWeb.imageBuild.buildkitState"), "monitorWeb.imageBuild.buildkitState");
|
|
|
|
|
const mode = stringAt(state, "mode");
|
|
|
|
|
if (mode === "hostPath") {
|
|
|
|
|
return {
|
|
|
|
|
mode,
|
|
|
|
|
path: stringAt(state, "path"),
|
|
|
|
|
type: stringAt(state, "type"),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (mode === "persistentVolumeClaim") {
|
|
|
|
|
return {
|
|
|
|
|
mode,
|
|
|
|
|
claimName: stringAt(state, "claimName"),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (mode === "emptyDir") {
|
|
|
|
|
return {
|
|
|
|
|
mode,
|
|
|
|
|
sizeLimit: stringAt(state, "sizeLimit"),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentBudget(state: SentinelCicdState): Record<string, unknown> {
|
|
|
|
|
const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget");
|
|
|
|
|
return {
|
|
|
|
@@ -593,6 +754,17 @@ function publishCurrentDashboardRequired(state: SentinelCicdState): boolean {
|
|
|
|
|
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number {
|
|
|
|
|
if (!publishCurrentDashboardEnabled(state)) return 0;
|
|
|
|
|
const dashboard = publishCurrentDashboardPlan(state);
|
|
|
|
|
const budgets = publishCurrentStageBudgets(state);
|
|
|
|
|
return Math.max(0, Math.min(
|
|
|
|
|
numberAt(budgets, "dashboardVerifySeconds"),
|
|
|
|
|
numberAt(dashboard, "commandTimeoutSeconds"),
|
|
|
|
|
Math.ceil(numberAt(dashboard, "waitTimeoutMs") / 1000),
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract<WebProbeSentinelOptions, { kind: "dashboard" }> {
|
|
|
|
|
const dashboard = publishCurrentDashboardPlan(state);
|
|
|
|
|
const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000));
|
|
|
|
@@ -619,7 +791,9 @@ function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds
|
|
|
|
|
function publishCurrentStageTimings(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, elapsedMs: number): Record<string, unknown> {
|
|
|
|
|
const publish = record(controlResult.publish);
|
|
|
|
|
const payload = record(publish.payload);
|
|
|
|
|
const stageTimings = record(payload.stageTimings);
|
|
|
|
|
const payloadStageTimings = record(payload.stageTimings);
|
|
|
|
|
const diagnosticStageTimings = record(record(publish.diagnostics).stageTimings);
|
|
|
|
|
const stageTimings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings;
|
|
|
|
|
const observedWait = record(record(controlResult.observed).wait);
|
|
|
|
|
return {
|
|
|
|
|
sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs),
|
|
|
|
@@ -680,9 +854,14 @@ function publishCurrentBlocker(controlResult: Record<string, unknown>, dashboard
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (dashboard.ok !== true) {
|
|
|
|
|
const degradedReason = text(dashboard.degradedReason);
|
|
|
|
|
return {
|
|
|
|
|
code: dashboard.skipped === true ? text(dashboard.reason) : "sentinel-publish-current-dashboard-verify-failed",
|
|
|
|
|
reason: dashboard.skipped === true ? "dashboard verification did not run" : "dashboard verification did not pass",
|
|
|
|
|
reason: dashboard.skipped === true
|
|
|
|
|
? "dashboard verification did not run"
|
|
|
|
|
: degradedReason === "-"
|
|
|
|
|
? "dashboard verification did not pass"
|
|
|
|
|
: `dashboard verification did not pass: ${degradedReason}`,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
@@ -1222,23 +1401,34 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
|
|
|
|
|
const command = `web-probe sentinel control-plane ${options.action}`;
|
|
|
|
|
const applyOnly = options.action === "apply";
|
|
|
|
|
const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state);
|
|
|
|
|
const deadline = startedAt + cicdWaitWarningSeconds * 1000;
|
|
|
|
|
const remainingCicdWaitSeconds = () => remainingSeconds(deadline, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds));
|
|
|
|
|
const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCicdWaitSeconds(), 20));
|
|
|
|
|
const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCicdWaitSeconds());
|
|
|
|
|
const waitBudgetSeconds = Math.max(1, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds));
|
|
|
|
|
const deadline = startedAt + waitBudgetSeconds * 1000;
|
|
|
|
|
const remainingCicdWaitSeconds = () => strictRemainingSeconds(deadline, waitBudgetSeconds);
|
|
|
|
|
const remainingCommandSeconds = () => Math.max(1, remainingCicdWaitSeconds());
|
|
|
|
|
const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCommandSeconds(), 20));
|
|
|
|
|
const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCommandSeconds());
|
|
|
|
|
const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true;
|
|
|
|
|
const publish = applyOnly
|
|
|
|
|
? null
|
|
|
|
|
: sourceMirrorReady
|
|
|
|
|
? runSentinelPublishJob(state, true, remainingCicdWaitSeconds())
|
|
|
|
|
? runSentinelPublishJob(state, true, remainingCommandSeconds())
|
|
|
|
|
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
|
|
|
|
const flush = !applyOnly && record(publish).ok === true
|
|
|
|
|
const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8;
|
|
|
|
|
const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true
|
|
|
|
|
? startSentinelGitMirrorFlushAsync(state)
|
|
|
|
|
: null;
|
|
|
|
|
const runtimeSecretsApply = applySentinelRuntimeSecrets(state, remainingCicdWaitSeconds());
|
|
|
|
|
const publicExposureApply = applySentinelPublicExposure(state, remainingCicdWaitSeconds());
|
|
|
|
|
const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds());
|
|
|
|
|
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false);
|
|
|
|
|
const runtimeSecretsApply = publishWaitBudgetExhausted
|
|
|
|
|
? sentinelSkippedControlStep("publish-wait-budget-exhausted")
|
|
|
|
|
: applySentinelRuntimeSecrets(state, remainingCommandSeconds());
|
|
|
|
|
const publicExposureApply = publishWaitBudgetExhausted
|
|
|
|
|
? sentinelSkippedControlStep("publish-wait-budget-exhausted")
|
|
|
|
|
: applySentinelPublicExposure(state, remainingCommandSeconds());
|
|
|
|
|
const argoApply = publishWaitBudgetExhausted
|
|
|
|
|
? sentinelSkippedControlStep("publish-wait-budget-exhausted")
|
|
|
|
|
: applySentinelArgoApplication(state, remainingCommandSeconds());
|
|
|
|
|
const observed = publishWaitBudgetExhausted
|
|
|
|
|
? sentinelSkippedObservedStatus("publish-wait-budget-exhausted")
|
|
|
|
|
: waitForSentinelObservedStatus(state, remainingCommandSeconds(), undefined, false);
|
|
|
|
|
const observedReady = sentinelObservedReady(observed);
|
|
|
|
|
const publishReady = applyOnly || record(publish).ok === true || observedReady;
|
|
|
|
|
const flushReady = applyOnly || record(flush).ok === true || observedReady;
|
|
|
|
@@ -1323,6 +1513,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
|
|
|
|
|
...publishSatisfiedByObservedWarnings(publish, flush, observedReady),
|
|
|
|
|
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
|
|
|
|
|
...sentinelObservedWarnings(observed),
|
|
|
|
|
...(publishWaitBudgetExhausted ? [`sentinel publish consumed the configured ${waitBudgetSeconds}s confirm-wait budget; skipped runtime Secret, public exposure, Argo apply and observed wait to avoid blind over-budget waiting. Use the reported status/log drill-down after the PipelineRun advances.`] : []),
|
|
|
|
|
...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
|
|
|
|
|
...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []),
|
|
|
|
|
...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []),
|
|
|
|
@@ -1388,6 +1579,31 @@ function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): str
|
|
|
|
|
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelSkippedControlStep(reason: string): Record<string, unknown> {
|
|
|
|
|
return { ok: false, skipped: true, reason, valuesRedacted: true };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus {
|
|
|
|
|
const skipped = { ok: false, skipped: true, reason, valuesRedacted: true };
|
|
|
|
|
return {
|
|
|
|
|
sourceMirror: skipped,
|
|
|
|
|
registry: skipped,
|
|
|
|
|
gitMirror: { skipped: true, reason, valuesRedacted: true },
|
|
|
|
|
gitops: skipped,
|
|
|
|
|
argo: skipped,
|
|
|
|
|
runtime: skipped,
|
|
|
|
|
wait: {
|
|
|
|
|
polls: 0,
|
|
|
|
|
elapsedMs: 0,
|
|
|
|
|
timeoutMs: 0,
|
|
|
|
|
ready: false,
|
|
|
|
|
includeGitMirror: false,
|
|
|
|
|
reason,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
|
|
|
|
|
const registry = probeImageRegistry(state, timeoutSeconds);
|
|
|
|
|
const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds);
|
|
|
|
@@ -1969,7 +2185,7 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu
|
|
|
|
|
sentinelGitMirrorCacheVolume(state),
|
|
|
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
|
|
|
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
|
|
|
|
|
{ name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } },
|
|
|
|
|
sentinelBuildkitStateVolume(state),
|
|
|
|
|
{ name: "tmp", emptyDir: {} },
|
|
|
|
|
],
|
|
|
|
|
steps: [
|
|
|
|
@@ -1984,6 +2200,16 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu
|
|
|
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "prepare-buildkit-state",
|
|
|
|
|
image: state.image.baseImage,
|
|
|
|
|
imagePullPolicy: "IfNotPresent",
|
|
|
|
|
script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"),
|
|
|
|
|
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
|
|
|
|
volumeMounts: [
|
|
|
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "image-build",
|
|
|
|
|
image: buildkitImage,
|
|
|
|
@@ -2030,6 +2256,27 @@ function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string,
|
|
|
|
|
return { name: "cache", persistentVolumeClaim: { claimName: stringAt(state.controlPlaneTarget, "gitMirror.cachePvcName") } };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelBuildkitStateVolume(state: SentinelCicdState): Record<string, unknown> {
|
|
|
|
|
const buildkitState = monitorWebBuildkitStatePlan(state.cicd);
|
|
|
|
|
const mode = stringAt(buildkitState, "mode");
|
|
|
|
|
if (mode === "hostPath") {
|
|
|
|
|
return {
|
|
|
|
|
name: "buildkit-state",
|
|
|
|
|
hostPath: {
|
|
|
|
|
path: stringAt(buildkitState, "path"),
|
|
|
|
|
type: stringAt(buildkitState, "type"),
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (mode === "persistentVolumeClaim") {
|
|
|
|
|
return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } };
|
|
|
|
|
}
|
|
|
|
|
if (mode === "emptyDir") {
|
|
|
|
|
return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } };
|
|
|
|
|
}
|
|
|
|
|
throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function requireSentinelBuildkitImage(state: SentinelCicdState): string {
|
|
|
|
|
const image = state.spec.buildkit?.sidecarImage;
|
|
|
|
|
if (typeof image !== "string" || image.length === 0) {
|
|
|
|
@@ -2453,6 +2700,7 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel
|
|
|
|
|
const events = sentinelStageEventsFromLogs(logsTail, domain);
|
|
|
|
|
const envReuse = sentinelEnvReuseFromLogs(logsTail);
|
|
|
|
|
const completedStages = sentinelCompletedStages(events, record(result.payload));
|
|
|
|
|
const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs);
|
|
|
|
|
const currentPhase = sentinelCurrentRemotePhase(result, events, domain);
|
|
|
|
|
const isPipelineRun = result.resourceKind === "PipelineRun";
|
|
|
|
|
const commands = {
|
|
|
|
@@ -2481,6 +2729,7 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel
|
|
|
|
|
taskRun: probe.taskRun ?? null,
|
|
|
|
|
currentPhase,
|
|
|
|
|
completedStages,
|
|
|
|
|
stageTimings,
|
|
|
|
|
envReuse,
|
|
|
|
|
pod: probe.pod ?? null,
|
|
|
|
|
podPhase: probe.podPhase ?? null,
|
|
|
|
@@ -2518,6 +2767,33 @@ function sentinelCompletedStages(events: readonly Record<string, unknown>[], pay
|
|
|
|
|
return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelStageTimingSummary(events: readonly Record<string, unknown>[], payload: Record<string, unknown>, fallbackTotalMs: unknown): Record<string, unknown> {
|
|
|
|
|
const payloadTimings = record(payload.stageTimings);
|
|
|
|
|
const eventElapsed = (stage: string): number | null => {
|
|
|
|
|
const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed"));
|
|
|
|
|
return event === undefined ? null : finiteNumberOrNull(event.elapsedMs);
|
|
|
|
|
};
|
|
|
|
|
const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch");
|
|
|
|
|
const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify");
|
|
|
|
|
const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build");
|
|
|
|
|
const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops");
|
|
|
|
|
const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null);
|
|
|
|
|
const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0);
|
|
|
|
|
const totalMs = finiteNumberOrNull(payloadTimings.totalMs)
|
|
|
|
|
?? finiteNumberOrNull(payload.elapsedMs)
|
|
|
|
|
?? finiteNumberOrNull(fallbackTotalMs)
|
|
|
|
|
?? summedTotalMs;
|
|
|
|
|
const result = {
|
|
|
|
|
sourceFetchMs,
|
|
|
|
|
monitorWebVerifyMs,
|
|
|
|
|
imageBuildMs,
|
|
|
|
|
gitopsMs,
|
|
|
|
|
totalMs,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
return Object.values(result).some((item) => item !== null && item !== true) ? result : {};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record<string, unknown>[], domain: "source-mirror" | "publish"): string {
|
|
|
|
|
if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed";
|
|
|
|
|
if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase;
|
|
|
|
@@ -3182,7 +3458,9 @@ function renderPublishResult(publish: Record<string, unknown>): string {
|
|
|
|
|
const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse;
|
|
|
|
|
const imageBuild = record(payload.imageBuild);
|
|
|
|
|
const imageBuildProxy = record(imageBuild.proxy);
|
|
|
|
|
const timings = record(payload.stageTimings);
|
|
|
|
|
const payloadStageTimings = record(payload.stageTimings);
|
|
|
|
|
const diagnosticStageTimings = record(diagnostics.stageTimings);
|
|
|
|
|
const timings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings;
|
|
|
|
|
const commands = record(diagnostics.commands);
|
|
|
|
|
const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none";
|
|
|
|
|
const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB";
|
|
|
|
@@ -3413,7 +3691,7 @@ function renderImageResult(result: Record<string, unknown>): string {
|
|
|
|
|
"",
|
|
|
|
|
table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]),
|
|
|
|
|
"",
|
|
|
|
|
Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, monitorWeb.imageBuildContextIgnore]]),
|
|
|
|
|
Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "BUILD_STATE", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, `${record(monitorWeb.imageBuildState).mode ?? "-"}:${record(monitorWeb.imageBuildState).path ?? record(monitorWeb.imageBuildState).claimName ?? record(monitorWeb.imageBuildState).sizeLimit ?? "-"}`, monitorWeb.imageBuildContextIgnore]]),
|
|
|
|
|
"",
|
|
|
|
|
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
|
|
|
|
|
"",
|
|
|
|
|