diff --git a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml index ad7aa503..4ea50718 100644 --- a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml @@ -57,6 +57,10 @@ sentinel: proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated verifyPhase: pre-image-build + buildkitState: + mode: hostPath + path: /var/lib/unidesk/web-probe-sentinel/buildkit-${nodeLower} + type: DirectoryOrCreate gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml index 16342568..8e242a9e 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml @@ -57,6 +57,10 @@ sentinel: proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated verifyPhase: pre-image-build + buildkitState: + mode: hostPath + path: /var/lib/unidesk/web-probe-sentinel/buildkit-${nodeLower} + type: DirectoryOrCreate gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml index 6becc59c..7c140648 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml @@ -57,6 +57,10 @@ sentinel: proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated verifyPhase: pre-image-build + buildkitState: + mode: hostPath + path: /var/lib/unidesk/web-probe-sentinel/buildkit-${nodeLower} + type: DirectoryOrCreate gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml index 43431803..6fad8450 100644 --- a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml @@ -58,6 +58,10 @@ sentinel: proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated verifyPhase: pre-image-build + buildkitState: + mode: hostPath + path: /var/lib/unidesk/web-probe-sentinel/buildkit-${nodeLower} + type: DirectoryOrCreate gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml index 6d3db31a..7cedfe22 100644 --- a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml @@ -57,6 +57,10 @@ sentinel: proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated verifyPhase: pre-image-build + buildkitState: + mode: hostPath + path: /var/lib/unidesk/web-probe-sentinel/buildkit-${nodeLower} + type: DirectoryOrCreate gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml index 5e034221..081c1c37 100644 --- a/config/hwlab-web-probe-sentinel/profiles.yaml +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -68,6 +68,10 @@ baselines: proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated verifyPhase: pre-image-build + buildkitState: + mode: hostPath + path: /var/lib/unidesk/web-probe-sentinel/buildkit-${nodeLower} + type: DirectoryOrCreate gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 548d217d..3e932d4d 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -19,7 +19,7 @@ import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; import { probeSentinelDashboardBrowser, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; -import { remainingSeconds, runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; +import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; export type WebProbeSentinelConfigAction = "plan" | "status"; export type WebProbeSentinelImageAction = "status" | "build"; @@ -368,8 +368,26 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E const budget = publishCurrentBudget(state); const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds")); const deadline = startedAt + budgetSeconds * 1000; - const remainingBudgetSeconds = () => remainingSeconds(deadline, budgetSeconds); - const controlResult = sentinelControlPlaneConfirmedResult(state, { + const remainingBudgetSeconds = () => strictRemainingSeconds(deadline, budgetSeconds); + let controlResult: Record | null = null; + if (state.configReady && state.sourceHead.ok && remainingBudgetSeconds() >= 5) { + const registryProbe = probeImageRegistry(state, Math.max(1, Math.min(remainingBudgetSeconds(), 5))); + if (record(record(registryProbe).probe).present === true && remainingBudgetSeconds() >= 5) { + const preflightStartedAt = Date.now(); + const preflightTimeoutSeconds = Math.max(1, Math.min(remainingBudgetSeconds(), 10)); + const preflightObserved = withObservedWait( + collectSentinelObservedStatus(state, preflightTimeoutSeconds, undefined, true), + preflightStartedAt, + preflightTimeoutSeconds, + true, + ); + if (sentinelObservedReady(preflightObserved)) { + controlResult = sentinelAlreadyCurrentControlResult(state, preflightObserved, Date.now() - preflightStartedAt); + } + } + } + const dashboardReserveSeconds = publishCurrentDashboardReserveSeconds(state); + controlResult ??= sentinelControlPlaneConfirmedResult(state, { kind: "control-plane", action: "trigger-current", node: options.node, @@ -378,7 +396,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E dryRun: false, confirm: true, wait: true, - timeoutSeconds: remainingBudgetSeconds(), + timeoutSeconds: Math.max(1, remainingBudgetSeconds() - dashboardReserveSeconds), }); const dashboardRequired = publishCurrentDashboardRequired(state); let dashboard: Record; @@ -408,8 +426,8 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, - mode: "confirm-wait", - mutation: true, + mode: controlResult.mode === "already-current" ? "already-current" : "confirm-wait", + mutation: controlResult.mutation !== false, specRef: SPEC_REF, source: state.sourceHead, image: state.image, @@ -431,6 +449,120 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E return rendered(ok, command, renderPublishCurrentResult(result)); } +function strictRemainingSeconds(deadline: number, cap: number): number { + return Math.max(0, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); +} + +function withObservedWait(observed: SentinelObservedStatus, startedAt: number, timeoutSeconds: number, includeGitMirror: boolean): SentinelObservedStatus { + const elapsedMs = Date.now() - startedAt; + return { + ...observed, + wait: { + polls: 1, + elapsedMs, + timeoutMs: Math.max(1, timeoutSeconds) * 1000, + ready: sentinelObservedReady(observed), + includeGitMirror, + valuesRedacted: true, + }, + }; +} + +function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: SentinelObservedStatus, elapsedMs: number): Record { + const registryProbe = record(record(observed.registry).probe); + const gitops = record(observed.gitops); + const argo = record(observed.argo); + const digest = text(registryProbe.digest); + const digestRef = digest === "-" ? null : `${state.image.repository}@${digest}`; + const stageTimings = { + sourceFetchMs: 0, + monitorWebVerifyMs: 0, + imageBuildMs: 0, + gitopsMs: 0, + totalMs: 0, + valuesRedacted: true, + }; + return { + ok: true, + command: "web-probe sentinel control-plane trigger-current", + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "already-current", + mutation: false, + specRef: SPEC_REF, + source: state.sourceHead, + image: state.image, + pipelineRun: "already-current", + gitops: { + path: stringAt(state.cicd, "gitopsPath"), + targetRevision: stringAt(state.cicd, "argo.targetRevision"), + manifestObjects: state.manifests.length, + manifestSha256: state.manifestSha256, + }, + argo: { + namespace: stringAt(state.cicd, "argo.namespace"), + projectName: stringAt(state.cicd, "argo.projectName"), + applicationName: stringAt(state.cicd, "argo.applicationName"), + }, + validation: { + scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"), + maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"), + controlPlaneWaitMaxSeconds: controlPlaneWaitWarningSeconds(state), + quickVerifyMode: "manual-validate", + automaticSecondPath: false, + }, + manifests: { + objects: manifestObjectSummary(state.manifests), + sha256: state.manifestSha256, + }, + sourceMirrorSync: { ok: true, phase: "already-current", jobName: "-", elapsedMs: 0, valuesRedacted: true }, + publish: { + ok: true, + phase: "already-current", + resourceKind: "PipelineRun", + jobName: "already-current", + elapsedMs: 0, + payload: { + ok: true, + status: "already-current", + sourceCommit: state.sourceHead.commit, + imageRef: state.image.ref, + digestRef, + gitopsCommit: gitops.revision ?? argo.revision ?? null, + stageTimings, + completedStages: ["already-current"], + valuesRedacted: true, + }, + diagnostics: { + domain: "publish", + resourceKind: "PipelineRun", + pipelineRun: "already-current", + currentPhase: "already-current", + completedStages: ["already-current:skipped"], + stageTimings, + valuesRedacted: true, + }, + valuesRedacted: true, + }, + flush: { ok: true, skipped: true, reason: "already-current", valuesRedacted: true }, + runtimeSecretsApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, + publicExposureApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, + argoApply: { ok: true, skipped: true, reason: "already-current-observed-ready", valuesRedacted: true }, + observed, + targetValidation: null, + elapsedMs, + warnings: [ + "publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used dashboard verification only.", + ...sentinelObservedWarnings(observed), + ...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)), + ], + blocker: null, + recoveryNext: controlPlaneRecoveryNext(state, true, {}, { ok: true }, observed), + next: controlPlaneNext(state, "trigger-current"), + valuesRedacted: true, + }; +} + function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState { const sentinel = resolveWebProbeSentinel(spec, sentinelId); const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); @@ -536,6 +668,7 @@ function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record): "defaul return value; } +function monitorWebBuildkitStatePlan(cicd: Record): Record { + const state = recordTarget(valueAtPath(cicd, "monitorWeb.imageBuild.buildkitState"), "monitorWeb.imageBuild.buildkitState"); + const mode = stringAt(state, "mode"); + if (mode === "hostPath") { + return { + mode, + path: stringAt(state, "path"), + type: stringAt(state, "type"), + valuesRedacted: true, + }; + } + if (mode === "persistentVolumeClaim") { + return { + mode, + claimName: stringAt(state, "claimName"), + valuesRedacted: true, + }; + } + if (mode === "emptyDir") { + return { + mode, + sizeLimit: stringAt(state, "sizeLimit"), + valuesRedacted: true, + }; + } + throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); +} + function publishCurrentBudget(state: SentinelCicdState): Record { const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget"); return { @@ -593,6 +754,17 @@ function publishCurrentDashboardRequired(state: SentinelCicdState): boolean { return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required"); } +function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number { + if (!publishCurrentDashboardEnabled(state)) return 0; + const dashboard = publishCurrentDashboardPlan(state); + const budgets = publishCurrentStageBudgets(state); + return Math.max(0, Math.min( + numberAt(budgets, "dashboardVerifySeconds"), + numberAt(dashboard, "commandTimeoutSeconds"), + Math.ceil(numberAt(dashboard, "waitTimeoutMs") / 1000), + )); +} + function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract { const dashboard = publishCurrentDashboardPlan(state); const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000)); @@ -619,7 +791,9 @@ function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds function publishCurrentStageTimings(controlResult: Record, dashboard: Record, elapsedMs: number): Record { const publish = record(controlResult.publish); const payload = record(publish.payload); - const stageTimings = record(payload.stageTimings); + const payloadStageTimings = record(payload.stageTimings); + const diagnosticStageTimings = record(record(publish.diagnostics).stageTimings); + const stageTimings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; const observedWait = record(record(controlResult.observed).wait); return { sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs), @@ -680,9 +854,14 @@ function publishCurrentBlocker(controlResult: Record, dashboard }; } if (dashboard.ok !== true) { + const degradedReason = text(dashboard.degradedReason); return { code: dashboard.skipped === true ? text(dashboard.reason) : "sentinel-publish-current-dashboard-verify-failed", - reason: dashboard.skipped === true ? "dashboard verification did not run" : "dashboard verification did not pass", + reason: dashboard.skipped === true + ? "dashboard verification did not run" + : degradedReason === "-" + ? "dashboard verification did not pass" + : `dashboard verification did not pass: ${degradedReason}`, valuesRedacted: true, }; } @@ -1222,23 +1401,34 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: const command = `web-probe sentinel control-plane ${options.action}`; const applyOnly = options.action === "apply"; const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); - const deadline = startedAt + cicdWaitWarningSeconds * 1000; - const remainingCicdWaitSeconds = () => remainingSeconds(deadline, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds)); - const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCicdWaitSeconds(), 20)); - const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCicdWaitSeconds()); + const waitBudgetSeconds = Math.max(1, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds)); + const deadline = startedAt + waitBudgetSeconds * 1000; + const remainingCicdWaitSeconds = () => strictRemainingSeconds(deadline, waitBudgetSeconds); + const remainingCommandSeconds = () => Math.max(1, remainingCicdWaitSeconds()); + const sourceMirrorProbe = applyOnly ? null : probeSourceMirror(state, Math.min(remainingCommandSeconds(), 20)); + const sourceMirrorSync = applyOnly ? null : record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, remainingCommandSeconds()); const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true; const publish = applyOnly ? null : sourceMirrorReady - ? runSentinelPublishJob(state, true, remainingCicdWaitSeconds()) + ? runSentinelPublishJob(state, true, remainingCommandSeconds()) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); - const flush = !applyOnly && record(publish).ok === true + const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8; + const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true ? startSentinelGitMirrorFlushAsync(state) : null; - const runtimeSecretsApply = applySentinelRuntimeSecrets(state, remainingCicdWaitSeconds()); - const publicExposureApply = applySentinelPublicExposure(state, remainingCicdWaitSeconds()); - const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds()); - const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false); + const runtimeSecretsApply = publishWaitBudgetExhausted + ? sentinelSkippedControlStep("publish-wait-budget-exhausted") + : applySentinelRuntimeSecrets(state, remainingCommandSeconds()); + const publicExposureApply = publishWaitBudgetExhausted + ? sentinelSkippedControlStep("publish-wait-budget-exhausted") + : applySentinelPublicExposure(state, remainingCommandSeconds()); + const argoApply = publishWaitBudgetExhausted + ? sentinelSkippedControlStep("publish-wait-budget-exhausted") + : applySentinelArgoApplication(state, remainingCommandSeconds()); + const observed = publishWaitBudgetExhausted + ? sentinelSkippedObservedStatus("publish-wait-budget-exhausted") + : waitForSentinelObservedStatus(state, remainingCommandSeconds(), undefined, false); const observedReady = sentinelObservedReady(observed); const publishReady = applyOnly || record(publish).ok === true || observedReady; const flushReady = applyOnly || record(flush).ok === true || observedReady; @@ -1323,6 +1513,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: ...publishSatisfiedByObservedWarnings(publish, flush, observedReady), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ...sentinelObservedWarnings(observed), + ...(publishWaitBudgetExhausted ? [`sentinel publish consumed the configured ${waitBudgetSeconds}s confirm-wait budget; skipped runtime Secret, public exposure, Argo apply and observed wait to avoid blind over-budget waiting. Use the reported status/log drill-down after the PipelineRun advances.`] : []), ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds), ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []), @@ -1388,6 +1579,31 @@ function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): str return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`]; } +function sentinelSkippedControlStep(reason: string): Record { + return { ok: false, skipped: true, reason, valuesRedacted: true }; +} + +function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus { + const skipped = { ok: false, skipped: true, reason, valuesRedacted: true }; + return { + sourceMirror: skipped, + registry: skipped, + gitMirror: { skipped: true, reason, valuesRedacted: true }, + gitops: skipped, + argo: skipped, + runtime: skipped, + wait: { + polls: 0, + elapsedMs: 0, + timeoutMs: 0, + ready: false, + includeGitMirror: false, + reason, + valuesRedacted: true, + }, + }; +} + function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const registry = probeImageRegistry(state, timeoutSeconds); const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds); @@ -1969,7 +2185,7 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu sentinelGitMirrorCacheVolume(state), { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, - { name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } }, + sentinelBuildkitStateVolume(state), { name: "tmp", emptyDir: {} }, ], steps: [ @@ -1984,6 +2200,16 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }, + { + name: "prepare-buildkit-state", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"), + securityContext: { runAsUser: 0, runAsGroup: 0 }, + volumeMounts: [ + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + ], + }, { name: "image-build", image: buildkitImage, @@ -2030,6 +2256,27 @@ function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record { + const buildkitState = monitorWebBuildkitStatePlan(state.cicd); + const mode = stringAt(buildkitState, "mode"); + if (mode === "hostPath") { + return { + name: "buildkit-state", + hostPath: { + path: stringAt(buildkitState, "path"), + type: stringAt(buildkitState, "type"), + }, + }; + } + if (mode === "persistentVolumeClaim") { + return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } }; + } + if (mode === "emptyDir") { + return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } }; + } + throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); +} + function requireSentinelBuildkitImage(state: SentinelCicdState): string { const image = state.spec.buildkit?.sidecarImage; if (typeof image !== "string" || image.length === 0) { @@ -2453,6 +2700,7 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel const events = sentinelStageEventsFromLogs(logsTail, domain); const envReuse = sentinelEnvReuseFromLogs(logsTail); const completedStages = sentinelCompletedStages(events, record(result.payload)); + const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs); const currentPhase = sentinelCurrentRemotePhase(result, events, domain); const isPipelineRun = result.resourceKind === "PipelineRun"; const commands = { @@ -2481,6 +2729,7 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel taskRun: probe.taskRun ?? null, currentPhase, completedStages, + stageTimings, envReuse, pod: probe.pod ?? null, podPhase: probe.podPhase ?? null, @@ -2518,6 +2767,33 @@ function sentinelCompletedStages(events: readonly Record[], pay return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-"); } +function sentinelStageTimingSummary(events: readonly Record[], payload: Record, fallbackTotalMs: unknown): Record { + const payloadTimings = record(payload.stageTimings); + const eventElapsed = (stage: string): number | null => { + const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed")); + return event === undefined ? null : finiteNumberOrNull(event.elapsedMs); + }; + const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch"); + const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify"); + const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build"); + const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops"); + const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null); + const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0); + const totalMs = finiteNumberOrNull(payloadTimings.totalMs) + ?? finiteNumberOrNull(payload.elapsedMs) + ?? finiteNumberOrNull(fallbackTotalMs) + ?? summedTotalMs; + const result = { + sourceFetchMs, + monitorWebVerifyMs, + imageBuildMs, + gitopsMs, + totalMs, + valuesRedacted: true, + }; + return Object.values(result).some((item) => item !== null && item !== true) ? result : {}; +} + function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record[], domain: "source-mirror" | "publish"): string { if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed"; if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase; @@ -3182,7 +3458,9 @@ function renderPublishResult(publish: Record): string { const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse; const imageBuild = record(payload.imageBuild); const imageBuildProxy = record(imageBuild.proxy); - const timings = record(payload.stageTimings); + const payloadStageTimings = record(payload.stageTimings); + const diagnosticStageTimings = record(diagnostics.stageTimings); + const timings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; const commands = record(diagnostics.commands); const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none"; const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB"; @@ -3413,7 +3691,7 @@ function renderImageResult(result: Record): string { "", table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]), "", - Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, monitorWeb.imageBuildContextIgnore]]), + Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "BUILD_STATE", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, `${record(monitorWeb.imageBuildState).mode ?? "-"}:${record(monitorWeb.imageBuildState).path ?? record(monitorWeb.imageBuildState).claimName ?? record(monitorWeb.imageBuildState).sizeLimit ?? "-"}`, monitorWeb.imageBuildContextIgnore]]), "", Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), "",