diff --git a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml index 29362818..ad7aa503 100644 --- a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml @@ -65,6 +65,25 @@ sentinel: maxSeconds: 120 confirmWait: maxSeconds: 120 + publishCurrent: + endToEndBudget: + maxSeconds: 120 + stageBudgets: + sourceSyncSeconds: 20 + sourceFetchSeconds: 20 + monitorWebVerifySeconds: 15 + imageBuildSeconds: 45 + gitopsSeconds: 15 + argoRuntimeSeconds: 30 + dashboardVerifySeconds: 30 + dashboard: + enabled: true + required: true + viewport: 1440x900 + timeoutMs: 30000 + waitTimeoutMs: 60000 + commandTimeoutSeconds: 90 + fullPage: false targetValidation: scenarioId: workbench-auth-session-switch-2users maxSeconds: 300 diff --git a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml index da35fb45..16342568 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml @@ -65,6 +65,25 @@ sentinel: maxSeconds: 120 confirmWait: maxSeconds: 120 + publishCurrent: + endToEndBudget: + maxSeconds: 120 + stageBudgets: + sourceSyncSeconds: 20 + sourceFetchSeconds: 20 + monitorWebVerifySeconds: 15 + imageBuildSeconds: 45 + gitopsSeconds: 15 + argoRuntimeSeconds: 30 + dashboardVerifySeconds: 30 + dashboard: + enabled: true + required: true + viewport: 1440x900 + timeoutMs: 30000 + waitTimeoutMs: 60000 + commandTimeoutSeconds: 90 + fullPage: false targetValidation: scenarioId: workbench-dsflash-go-tool-call-10x maxSeconds: 300 diff --git a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml index 71d5e7df..6becc59c 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml @@ -65,6 +65,25 @@ sentinel: maxSeconds: 120 confirmWait: maxSeconds: 120 + publishCurrent: + endToEndBudget: + maxSeconds: 120 + stageBudgets: + sourceSyncSeconds: 20 + sourceFetchSeconds: 20 + monitorWebVerifySeconds: 15 + imageBuildSeconds: 45 + gitopsSeconds: 15 + argoRuntimeSeconds: 30 + dashboardVerifySeconds: 30 + dashboard: + enabled: true + required: true + viewport: 1440x900 + timeoutMs: 30000 + waitTimeoutMs: 60000 + commandTimeoutSeconds: 90 + fullPage: false targetValidation: scenarioId: workbench-dsflash-go-tool-call-10x maxSeconds: 300 diff --git a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml index 2b18041f..43431803 100644 --- a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml @@ -66,6 +66,25 @@ sentinel: maxSeconds: 120 confirmWait: maxSeconds: 120 + publishCurrent: + endToEndBudget: + maxSeconds: 120 + stageBudgets: + sourceSyncSeconds: 20 + sourceFetchSeconds: 20 + monitorWebVerifySeconds: 15 + imageBuildSeconds: 45 + gitopsSeconds: 15 + argoRuntimeSeconds: 30 + dashboardVerifySeconds: 30 + dashboard: + enabled: true + required: true + viewport: 1440x900 + timeoutMs: 30000 + waitTimeoutMs: 60000 + commandTimeoutSeconds: 90 + fullPage: false targetValidation: scenarioId: workbench-fake-echo-session-invariance-10x maxSeconds: 300 diff --git a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml index 9167c03b..6d3db31a 100644 --- a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml @@ -65,6 +65,25 @@ sentinel: maxSeconds: 120 confirmWait: maxSeconds: 120 + publishCurrent: + endToEndBudget: + maxSeconds: 120 + stageBudgets: + sourceSyncSeconds: 20 + sourceFetchSeconds: 20 + monitorWebVerifySeconds: 15 + imageBuildSeconds: 45 + gitopsSeconds: 15 + argoRuntimeSeconds: 30 + dashboardVerifySeconds: 30 + dashboard: + enabled: true + required: true + viewport: 1440x900 + timeoutMs: 30000 + waitTimeoutMs: 60000 + commandTimeoutSeconds: 90 + fullPage: false targetValidation: scenarioId: mdtodo-visual-regression maxSeconds: 360 diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml index a128f412..5e034221 100644 --- a/config/hwlab-web-probe-sentinel/profiles.yaml +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -79,6 +79,25 @@ baselines: stopCommand: sentinel maintenance stop confirmWait: &confirm-wait maxSeconds: 120 + publishCurrent: &publish-current + endToEndBudget: + maxSeconds: 120 + stageBudgets: + sourceSyncSeconds: 20 + sourceFetchSeconds: 20 + monitorWebVerifySeconds: 15 + imageBuildSeconds: 45 + gitopsSeconds: 15 + argoRuntimeSeconds: 30 + dashboardVerifySeconds: 30 + dashboard: + enabled: true + required: true + viewport: 1440x900 + timeoutMs: 30000 + waitTimeoutMs: 60000 + commandTimeoutSeconds: 90 + fullPage: false publicExposure: common: &public-exposure-common @@ -139,6 +158,8 @@ nodes: <<: *monitor-web confirmWait: <<: *confirm-wait + publishCurrent: + <<: *publish-current sentinels: jd01-web-probe-sentinel: diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index c2a1c070..6e0e623a 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -65,6 +65,7 @@ export function hwlabNodeWebProbeHelp(): Record { "bun scripts/cli.ts web-probe observe analyze webobs-xxxx", "bun scripts/cli.ts web-probe sentinel plan --node D601 --lane v03 --dry-run", "bun scripts/cli.ts web-probe sentinel plan --node D601 --lane v03 --sentinel workbench-auth-session-switch-2users", + "bun scripts/cli.ts web-probe sentinel publish-current --node JD01 --lane v03 --sentinel jd01-web-probe-sentinel --confirm --wait", "bun scripts/cli.ts web-probe sentinel dashboard verify --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x", "bun scripts/cli.ts web-probe sentinel dashboard screenshot --node D601 --lane v03 --sentinel workbench-auth-session-switch-2users", "bun scripts/cli.ts web-probe sentinel report --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --latest --view summary --raw", @@ -77,7 +78,7 @@ export function hwlabNodeWebProbeHelp(): Record { script: "Run caller-provided Playwright JS after CLI-managed /auth/login; scripts must not handle secrets themselves.", screenshot: "Capture a no-auth or public page through the selected node/lane remote browser and download PNG artifacts to the caller /tmp by default.", observe: "Start, inspect, control, stop, collect, and analyze a long-running observer that writes JSONL artifacts.", - sentinel: "Render and operate the YAML-first web-probe sentinel wrapper, image, GitOps, dashboard verification, maintenance and report views.", + sentinel: "Render and operate the YAML-first web-probe sentinel wrapper, one-click publish, image, GitOps, dashboard verification, maintenance and report views.", }, notes: [ "Default URL, browser proxy mode, observe/analyze thresholds, and project-management command allowlist come from config/hwlab-node-lanes.yaml webProbe.", diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 43d97c0b..548d217d 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -18,12 +18,13 @@ import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-c import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; -import { runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; +import { probeSentinelDashboardBrowser, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; import { remainingSeconds, runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; export type WebProbeSentinelConfigAction = "plan" | "status"; export type WebProbeSentinelImageAction = "status" | "build"; export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current"; +export type WebProbeSentinelPublishAction = "publish-current"; export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; export type WebProbeSentinelDashboardAction = "verify" | "screenshot"; export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary"; @@ -59,6 +60,17 @@ export type WebProbeSentinelOptions = readonly wait: boolean; readonly timeoutSeconds: number; } + | { + readonly kind: "publish"; + readonly action: WebProbeSentinelPublishAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + } | { readonly kind: "maintenance"; readonly action: WebProbeSentinelMaintenanceAction; @@ -165,6 +177,7 @@ interface SentinelObservedStatus { readonly gitops: Record; readonly argo: Record; readonly runtime: Record; + readonly wait?: Record; } interface SentinelObservedExpectation { @@ -209,6 +222,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds); if (options.kind === "image") return runSentinelImage(state, options); if (options.kind === "control-plane") return runSentinelControlPlane(state, options); + if (options.kind === "publish") return runSentinelPublishCurrent(state, options); if (options.kind === "maintenance") return runSentinelMaintenance(state, options); if (options.kind === "validate") return runSentinelValidate(state, options); if (options.kind === "dashboard") return runSentinelDashboard(state, options); @@ -311,6 +325,112 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = "web-probe sentinel publish-current"; + if (options.confirm) { + if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds); + return runSentinelPublishCurrentConfirmed(state, options); + } + const result = { + ok: state.configReady && state.sourceHead.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + sentinelId: state.sentinelId, + mode: "dry-run", + mutation: false, + specRef: SPEC_REF, + source: state.sourceHead, + image: state.image, + pipelineRun: sentinelPipelineRunName(state), + gitops: { + path: stringAt(state.cicd, "gitopsPath"), + targetRevision: stringAt(state.cicd, "argo.targetRevision"), + manifestSha256: state.manifestSha256, + }, + argo: { + namespace: stringAt(state.cicd, "argo.namespace"), + applicationName: stringAt(state.cicd, "argo.applicationName"), + }, + budget: publishCurrentBudget(state), + dashboardPlan: publishCurrentDashboardPlan(state), + stageBudgets: publishCurrentStageBudgets(state), + blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" }, + next: publishCurrentNext(state), + valuesRedacted: true, + }; + return rendered(result.ok === true, command, renderPublishCurrentResult(result)); +} + +function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { + const startedAt = Date.now(); + const command = "web-probe sentinel publish-current"; + const budget = publishCurrentBudget(state); + const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds")); + const deadline = startedAt + budgetSeconds * 1000; + const remainingBudgetSeconds = () => remainingSeconds(deadline, budgetSeconds); + const controlResult = sentinelControlPlaneConfirmedResult(state, { + kind: "control-plane", + action: "trigger-current", + node: options.node, + lane: options.lane, + sentinelId: options.sentinelId, + dryRun: false, + confirm: true, + wait: true, + timeoutSeconds: remainingBudgetSeconds(), + }); + const dashboardRequired = publishCurrentDashboardRequired(state); + let dashboard: Record; + let dashboardElapsedMs: number | null = null; + if (controlResult.ok !== true) { + dashboard = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true }; + } else if (!publishCurrentDashboardEnabled(state)) { + dashboard = { ok: !dashboardRequired, skipped: true, reason: "disabled-by-yaml", valuesRedacted: true }; + } else if (remainingBudgetSeconds() < 2) { + dashboard = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-dashboard", valuesRedacted: true }; + } else { + const dashboardStartedAt = Date.now(); + dashboard = probeSentinelDashboardBrowser(state, publishCurrentDashboardOptions(state, remainingBudgetSeconds())); + dashboardElapsedMs = Date.now() - dashboardStartedAt; + dashboard = { ...dashboard, elapsedMs: dashboardElapsedMs, valuesRedacted: true }; + } + const elapsedMs = Date.now() - startedAt; + const timings = publishCurrentStageTimings(controlResult, dashboard, elapsedMs); + const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds); + const withinBudget = elapsedMs <= budgetSeconds * 1000; + const dashboardOk = dashboardRequired ? dashboard.ok === true : dashboard.ok !== false; + const ok = controlResult.ok === true && dashboardOk && withinBudget; + const blocker = ok ? null : publishCurrentBlocker(controlResult, dashboard, withinBudget); + const result = { + ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + sentinelId: state.sentinelId, + mode: "confirm-wait", + mutation: true, + specRef: SPEC_REF, + source: state.sourceHead, + image: state.image, + pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state), + controlPlane: controlResult, + dashboard, + budget, + dashboardPlan: publishCurrentDashboardPlan(state), + stageBudgets: publishCurrentStageBudgets(state), + elapsedMs, + withinBudget, + timings, + slowStages, + warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)), + blocker, + next: publishCurrentNext(state), + valuesRedacted: true, + }; + return rendered(ok, command, renderPublishCurrentResult(result)); +} + function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState { const sentinel = resolveWebProbeSentinel(spec, sentinelId); const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); @@ -427,6 +547,168 @@ function monitorWebImageBuildNetworkMode(cicd: Record): "defaul return value; } +function publishCurrentBudget(state: SentinelCicdState): Record { + const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget"); + return { + maxSeconds: numberAt(budget, "maxSeconds"), + valuesRedacted: true, + }; +} + +function publishCurrentStageBudgets(state: SentinelCicdState): Record { + const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets"); + return { + sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"), + sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"), + monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"), + imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"), + gitopsSeconds: numberAt(budgets, "gitopsSeconds"), + argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"), + dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"), + valuesRedacted: true, + }; +} + +function publishCurrentDashboardPlan(state: SentinelCicdState): Record { + const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"); + const viewport = stringAt(dashboard, "viewport"); + if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`); + return { + enabled: booleanAt(dashboard, "enabled"), + required: booleanAt(dashboard, "required"), + viewport, + timeoutMs: numberAt(dashboard, "timeoutMs"), + waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"), + commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"), + fullPage: booleanAt(dashboard, "fullPage"), + valuesRedacted: true, + }; +} + +function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean { + return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled"); +} + +function publishCurrentDashboardRequired(state: SentinelCicdState): boolean { + return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required"); +} + +function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract { + const dashboard = publishCurrentDashboardPlan(state); + const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000)); + const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs); + const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs); + return { + kind: "dashboard", + action: "verify", + node: state.spec.nodeId, + lane: state.spec.lane, + sentinelId: state.sentinelId, + viewport: stringAt(dashboard, "viewport"), + localDir: "/tmp", + name: null, + timeoutMs, + waitTimeoutMs, + timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))), + commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))), + fullPage: booleanAt(dashboard, "fullPage"), + raw: false, + }; +} + +function publishCurrentStageTimings(controlResult: Record, dashboard: Record, elapsedMs: number): Record { + const publish = record(controlResult.publish); + const payload = record(publish.payload); + const stageTimings = record(payload.stageTimings); + const observedWait = record(record(controlResult.observed).wait); + return { + sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs), + sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs), + monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs), + imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs), + gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs), + argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs), + dashboardVerifyMs: finiteNumberOrNull(dashboard.elapsedMs), + totalMs: elapsedMs, + valuesRedacted: true, + }; +} + +function publishCurrentSlowStages(state: SentinelCicdState, timings: Record, budgetSeconds: number): Record[] { + const budgets = publishCurrentStageBudgets(state); + const stageMap: Array<[string, string, string, string]> = [ + ["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"], + ["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"], + ["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"], + ["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"], + ["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"], + ["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"], + ["dashboard-verify", "dashboardVerifyMs", "dashboardVerifySeconds", "inspect remote browser startup, monitor-web public route and dashboard API latency"], + ]; + const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => { + const elapsed = finiteNumberOrNull(timings[timingKey]); + const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]); + if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return []; + return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }]; + }); + const total = finiteNumberOrNull(timings.totalMs); + if (total !== null && total > budgetSeconds * 1000) { + slow.push({ + stage: "total", + elapsedMs: total, + budgetSeconds, + suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or dashboard segment before rerun", + valuesRedacted: true, + }); + } + return slow; +} + +function publishCurrentBudgetWarnings(slowStages: readonly Record[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] { + const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`); + if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`); + return warnings; +} + +function publishCurrentBlocker(controlResult: Record, dashboard: Record, withinBudget: boolean): Record { + if (controlResult.ok !== true) { + const blocker = record(controlResult.blocker); + return { + code: blocker.code ?? "sentinel-publish-current-control-plane-blocked", + reason: blocker.reason ?? "control-plane publish did not converge", + valuesRedacted: true, + }; + } + if (dashboard.ok !== true) { + return { + code: dashboard.skipped === true ? text(dashboard.reason) : "sentinel-publish-current-dashboard-verify-failed", + reason: dashboard.skipped === true ? "dashboard verification did not run" : "dashboard verification did not pass", + valuesRedacted: true, + }; + } + if (!withinBudget) { + return { + code: "sentinel-publish-current-over-budget", + reason: "runtime and dashboard converged, but the one-click CI/CD path exceeded the YAML end-to-end budget", + valuesRedacted: true, + }; + } + return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true }; +} + +function publishCurrentNext(state: SentinelCicdState): Record { + const node = state.spec.nodeId; + const lane = state.spec.lane; + const suffix = sentinelCliSuffix(state); + return { + publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`, + controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`, + dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`, + gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`, + gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`, + }; +} + function renderSentinelManifests( spec: HwlabRuntimeLaneSpec, sentinelId: string, @@ -931,6 +1213,11 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra } function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { + const result = sentinelControlPlaneConfirmedResult(state, options); + return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result)); +} + +function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract): Record { const startedAt = Date.now(); const command = `web-probe sentinel control-plane ${options.action}`; const applyOnly = options.action === "apply"; @@ -953,25 +1240,35 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds()); const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false); const observedReady = sentinelObservedReady(observed); + const publishReady = applyOnly || record(publish).ok === true || observedReady; + const flushReady = applyOnly || record(flush).ok === true || observedReady; const targetValidation = null; const targetValidationBlocked = false; const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady - && (applyOnly || record(publish).ok === true) - && (applyOnly || record(flush).ok === true) + && publishReady + && flushReady && record(runtimeSecretsApply).ok === true && record(publicExposureApply).ok === true && record(argoApply).ok === true && observedReady; const elapsedMs = Date.now() - startedAt; const blocker = ok ? null : { - code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready", + code: !sourceMirrorReady + ? "sentinel-source-mirror-sync-failed" + : !publishReady + ? "sentinel-image-gitops-publish-not-ready" + : record(runtimeSecretsApply).ok === false + ? "sentinel-runtime-secret-sync-failed" + : "sentinel-control-plane-not-ready", reason: !sourceMirrorReady ? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish" - : record(runtimeSecretsApply).ok === false - ? "one or more YAML-declared runtime Secrets were not synced from sourceRef" - : "one or more publish, publicExposure, Argo or runtime observation checks did not pass", + : !publishReady + ? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet" + : record(runtimeSecretsApply).ok === false + ? "one or more YAML-declared runtime Secrets were not synced from sourceRef" + : "one or more publicExposure, Argo or runtime observation checks did not pass", }; const result = { ok, @@ -1022,7 +1319,8 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds), - ...asyncGitMirrorFlushWarnings(flush), + ...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds), + ...publishSatisfiedByObservedWarnings(publish, flush, observedReady), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ...sentinelObservedWarnings(observed), ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds), @@ -1034,15 +1332,17 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext next: controlPlaneNext(state, options.action), valuesRedacted: true, }; - return rendered(ok, command, renderControlPlaneResult(result)); + return result; } -function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane", action: string, timeoutSeconds: number): RenderedCliResult { +function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult { const args = domain === "image" ? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)] - : ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; + : domain === "control-plane" + ? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)] + : ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`); - const command = `web-probe sentinel ${domain} ${action}`; + const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`; const result = { ok: true, command, @@ -1081,11 +1381,11 @@ function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record | SentinelObservedStatus): boolean { @@ -2238,11 +2550,18 @@ function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`]; } -function sentinelElapsedWarnings(value: unknown, subject = "sentinel confirmed operation", budgetSeconds = 120): string[] { - const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; - const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; - if (elapsedMs === null || elapsedMs <= budgetMs) return []; - return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s timing budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, investigate wait-stage latency without treating timing alone as HWLAB business blockage.`]; +function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] { + if (!observedReady) return []; + const warnings: string[] = []; + const publishRecord = record(publish); + if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) { + warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`); + } + const flushRecord = record(flush); + if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) { + warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout."); + } + return warnings; } function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number { @@ -2944,6 +3263,134 @@ function renderPublishResult(publish: Record): string { return lines.join("\n"); } +function renderPublishCurrentResult(result: Record): string { + const source = record(result.source); + const image = record(result.image); + const controlPlane = record(result.controlPlane); + const publish = record(controlPlane.publish); + const publishPayload = record(publish.payload); + const observed = record(controlPlane.observed); + const gitops = record(observed.gitops); + const argo = record(observed.argo); + const runtime = record(observed.runtime); + const runtimeDeployment = record(record(runtime.probe).deployment); + const dashboard = record(result.dashboard); + const dashboardPage = record(dashboard.page); + const dashboardDom = record(dashboardPage.dom); + const latestRunCounts = record(dashboardDom.latestRunCounts); + const checkScope = record(dashboardDom.checkScope); + const timings = record(result.timings); + const budget = record(result.budget); + const stageBudgets = record(result.stageBudgets); + const dashboardPlan = record(result.dashboardPlan); + const blocker = record(result.blocker); + const next = record(result.next); + const warnings = Array.isArray(result.warnings) ? result.warnings : []; + const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : []; + const lines = [ + String(result.command), + "", + table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[ + result.node, + result.lane, + result.sentinelId, + result.ok === true ? "ok" : "blocked", + result.mode, + budget.maxSeconds ?? "-", + finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000), + ]]), + "", + table(["SOURCE", "COMMIT", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[ + `${source.repository ?? "-"}@${source.branch ?? "-"}`, + short(source.commit), + image.ref ?? "-", + short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest), + result.pipelineRun ?? publish.jobName ?? "-", + ]]), + "", + table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "DASHBOARD"], [[ + short(gitops.revision), + short(argo.revision), + `${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`, + short(runtimeDeployment.image), + `${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`, + dashboard.ok === true ? "pass" : dashboard.skipped === true ? `skipped:${text(dashboard.reason)}` : Object.keys(dashboard).length === 0 ? "planned" : "blocked", + ]]), + "", + table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "DASHBOARD_MS", "TOTAL_MS"], [[ + timings.sourceSyncMs ?? "-", + timings.sourceFetchMs ?? "-", + timings.monitorWebVerifyMs ?? "-", + timings.imageBuildMs ?? "-", + timings.gitopsMs ?? "-", + timings.argoRuntimeMs ?? "-", + timings.dashboardVerifyMs ?? "-", + timings.totalMs ?? "-", + ]]), + "", + table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "DASHBOARD"], [[ + "YAML publishCurrent", + stageBudgets.sourceSyncSeconds ?? "-", + stageBudgets.sourceFetchSeconds ?? "-", + stageBudgets.monitorWebVerifySeconds ?? "-", + stageBudgets.imageBuildSeconds ?? "-", + stageBudgets.gitopsSeconds ?? "-", + stageBudgets.argoRuntimeSeconds ?? "-", + stageBudgets.dashboardVerifySeconds ?? "-", + ]]), + ]; + if (Object.keys(publish).length > 0) { + const payloadImageBuild = record(publishPayload.imageBuild); + const payloadEnvReuse = record(publishPayload.envReuse); + lines.push( + "", + table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[ + payloadEnvReuse.dependencyReuse ?? "-", + payloadEnvReuse.nodeDepsPresent ?? "-", + payloadImageBuild.packageMode ?? "-", + payloadImageBuild.networkMode ?? "-", + payloadImageBuild.layerCache ?? "-", + payloadImageBuild.cacheHitLines ?? "-", + ]]), + ); + } + lines.push( + "", + Object.keys(dashboard).length === 0 + ? "DASHBOARD_VERIFY\n-" + : table(["URL", "HTTP", "LATEST_RUN", "CHECK_SCOPE", "CHECK_MATCH", "REQ_FAIL", "CONSOLE_ERR"], [[ + dashboard.publicUrl ?? "-", + dashboardPage.httpStatus ?? "-", + latestRunCounts.runId ?? "-", + checkScope.scope ?? "-", + checkScope.matchesRunDetail ?? "-", + dashboardPage.requestFailureCount ?? "-", + dashboardPage.consoleErrorCount ?? "-", + ]]), + "", + slowStages.length === 0 ? "SLOW_STAGES\n-" : [ + "SLOW_STAGES", + table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])), + ].join("\n"), + "", + warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), + "", + "NEXT", + ` publish-current: ${next.publishCurrent ?? "-"}`, + ` status: ${next.controlPlaneStatus ?? "-"}`, + ` dashboard: ${next.dashboardVerify ?? "-"}`, + ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, + ` flush: ${next.gitMirrorFlush ?? "-"}`, + "", + "DISCLOSURE", + ` end-to-end and stage budgets are read from ${Object.keys(dashboardPlan).length > 0 ? "publishCurrent YAML" : "YAML-required publishCurrent fields"}.`, + " image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.", + ); + return lines.join("\n"); +} + function renderImageResult(result: Record): string { const source = record(result.source); const sourceMirror = record(result.sourceMirror); @@ -3197,6 +3644,16 @@ export function numberAt(value: unknown, path: string): number { return found; } +function booleanAt(value: unknown, path: string): boolean { + const found = valueAtPath(value, path); + if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`); + return found; +} + +function finiteNumberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + export function arrayAt(value: unknown, path: string): unknown[] { const found = valueAtPath(value, path); if (!Array.isArray(found)) throw new Error(`${path} must be an array`); diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index 53a9f7e6..b2e35a26 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -391,7 +391,7 @@ export function runSentinelDashboard(state: SentinelCicdState, options: Extract< return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result)); } -function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract): Record { +export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const [widthRaw, heightRaw] = options.viewport.split("x"); const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : ""; diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 5eff4a52..290e0ebb 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -45,12 +45,13 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe && sentinelActionRaw !== "status" && sentinelActionRaw !== "image" && sentinelActionRaw !== "control-plane" + && sentinelActionRaw !== "publish-current" && sentinelActionRaw !== "validate" && sentinelActionRaw !== "maintenance" && sentinelActionRaw !== "dashboard" && sentinelActionRaw !== "report" ) { - throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|validate|maintenance|dashboard|report --node NODE --lane vNN [--dry-run|--confirm]"); + throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|publish-current|validate|maintenance|dashboard|report --node NODE --lane vNN [--dry-run|--confirm]"); } assertKnownOptions(args, new Set([ "--node", @@ -96,6 +97,8 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]"); } sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds }; + } else if (sentinelActionRaw === "publish-current") { + sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds }; } else if (sentinelActionRaw === "maintenance") { const maintenanceAction = args[1]; if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") {