fix: add one-click web sentinel publish

This commit is contained in:
Codex
2026-07-01 00:56:28 +00:00
parent 0f087898b9
commit 98c59c9468
10 changed files with 600 additions and 23 deletions
@@ -65,6 +65,25 @@ sentinel:
maxSeconds: 120
confirmWait:
maxSeconds: 120
publishCurrent:
endToEndBudget:
maxSeconds: 120
stageBudgets:
sourceSyncSeconds: 20
sourceFetchSeconds: 20
monitorWebVerifySeconds: 15
imageBuildSeconds: 45
gitopsSeconds: 15
argoRuntimeSeconds: 30
dashboardVerifySeconds: 30
dashboard:
enabled: true
required: true
viewport: 1440x900
timeoutMs: 30000
waitTimeoutMs: 60000
commandTimeoutSeconds: 90
fullPage: false
targetValidation:
scenarioId: workbench-auth-session-switch-2users
maxSeconds: 300
@@ -65,6 +65,25 @@ sentinel:
maxSeconds: 120
confirmWait:
maxSeconds: 120
publishCurrent:
endToEndBudget:
maxSeconds: 120
stageBudgets:
sourceSyncSeconds: 20
sourceFetchSeconds: 20
monitorWebVerifySeconds: 15
imageBuildSeconds: 45
gitopsSeconds: 15
argoRuntimeSeconds: 30
dashboardVerifySeconds: 30
dashboard:
enabled: true
required: true
viewport: 1440x900
timeoutMs: 30000
waitTimeoutMs: 60000
commandTimeoutSeconds: 90
fullPage: false
targetValidation:
scenarioId: workbench-dsflash-go-tool-call-10x
maxSeconds: 300
@@ -65,6 +65,25 @@ sentinel:
maxSeconds: 120
confirmWait:
maxSeconds: 120
publishCurrent:
endToEndBudget:
maxSeconds: 120
stageBudgets:
sourceSyncSeconds: 20
sourceFetchSeconds: 20
monitorWebVerifySeconds: 15
imageBuildSeconds: 45
gitopsSeconds: 15
argoRuntimeSeconds: 30
dashboardVerifySeconds: 30
dashboard:
enabled: true
required: true
viewport: 1440x900
timeoutMs: 30000
waitTimeoutMs: 60000
commandTimeoutSeconds: 90
fullPage: false
targetValidation:
scenarioId: workbench-dsflash-go-tool-call-10x
maxSeconds: 300
@@ -66,6 +66,25 @@ sentinel:
maxSeconds: 120
confirmWait:
maxSeconds: 120
publishCurrent:
endToEndBudget:
maxSeconds: 120
stageBudgets:
sourceSyncSeconds: 20
sourceFetchSeconds: 20
monitorWebVerifySeconds: 15
imageBuildSeconds: 45
gitopsSeconds: 15
argoRuntimeSeconds: 30
dashboardVerifySeconds: 30
dashboard:
enabled: true
required: true
viewport: 1440x900
timeoutMs: 30000
waitTimeoutMs: 60000
commandTimeoutSeconds: 90
fullPage: false
targetValidation:
scenarioId: workbench-fake-echo-session-invariance-10x
maxSeconds: 300
@@ -65,6 +65,25 @@ sentinel:
maxSeconds: 120
confirmWait:
maxSeconds: 120
publishCurrent:
endToEndBudget:
maxSeconds: 120
stageBudgets:
sourceSyncSeconds: 20
sourceFetchSeconds: 20
monitorWebVerifySeconds: 15
imageBuildSeconds: 45
gitopsSeconds: 15
argoRuntimeSeconds: 30
dashboardVerifySeconds: 30
dashboard:
enabled: true
required: true
viewport: 1440x900
timeoutMs: 30000
waitTimeoutMs: 60000
commandTimeoutSeconds: 90
fullPage: false
targetValidation:
scenarioId: mdtodo-visual-regression
maxSeconds: 360
@@ -79,6 +79,25 @@ baselines:
stopCommand: sentinel maintenance stop
confirmWait: &confirm-wait
maxSeconds: 120
publishCurrent: &publish-current
endToEndBudget:
maxSeconds: 120
stageBudgets:
sourceSyncSeconds: 20
sourceFetchSeconds: 20
monitorWebVerifySeconds: 15
imageBuildSeconds: 45
gitopsSeconds: 15
argoRuntimeSeconds: 30
dashboardVerifySeconds: 30
dashboard:
enabled: true
required: true
viewport: 1440x900
timeoutMs: 30000
waitTimeoutMs: 60000
commandTimeoutSeconds: 90
fullPage: false
publicExposure:
common: &public-exposure-common
@@ -139,6 +158,8 @@ nodes:
<<: *monitor-web
confirmWait:
<<: *confirm-wait
publishCurrent:
<<: *publish-current
sentinels:
jd01-web-probe-sentinel:
+2 -1
View File
@@ -65,6 +65,7 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
"bun scripts/cli.ts web-probe observe analyze webobs-xxxx",
"bun scripts/cli.ts web-probe sentinel plan --node D601 --lane v03 --dry-run",
"bun scripts/cli.ts web-probe sentinel plan --node D601 --lane v03 --sentinel workbench-auth-session-switch-2users",
"bun scripts/cli.ts web-probe sentinel publish-current --node JD01 --lane v03 --sentinel jd01-web-probe-sentinel --confirm --wait",
"bun scripts/cli.ts web-probe sentinel dashboard verify --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x",
"bun scripts/cli.ts web-probe sentinel dashboard screenshot --node D601 --lane v03 --sentinel workbench-auth-session-switch-2users",
"bun scripts/cli.ts web-probe sentinel report --node D601 --lane v03 --sentinel workbench-dsflash-go-tool-call-10x --latest --view summary --raw",
@@ -77,7 +78,7 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
script: "Run caller-provided Playwright JS after CLI-managed /auth/login; scripts must not handle secrets themselves.",
screenshot: "Capture a no-auth or public page through the selected node/lane remote browser and download PNG artifacts to the caller /tmp by default.",
observe: "Start, inspect, control, stop, collect, and analyze a long-running observer that writes JSONL artifacts.",
sentinel: "Render and operate the YAML-first web-probe sentinel wrapper, image, GitOps, dashboard verification, maintenance and report views.",
sentinel: "Render and operate the YAML-first web-probe sentinel wrapper, one-click publish, image, GitOps, dashboard verification, maintenance and report views.",
},
notes: [
"Default URL, browser proxy mode, observe/analyze thresholds, and project-management command allowlist come from config/hwlab-node-lanes.yaml webProbe.",
+477 -20
View File
@@ -18,12 +18,13 @@ import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-c
import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver";
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import type { RenderedCliResult } from "./output";
import { runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
import { probeSentinelDashboardBrowser, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
import { remainingSeconds, runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
export type WebProbeSentinelConfigAction = "plan" | "status";
export type WebProbeSentinelImageAction = "status" | "build";
export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current";
export type WebProbeSentinelPublishAction = "publish-current";
export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop";
export type WebProbeSentinelDashboardAction = "verify" | "screenshot";
export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary";
@@ -59,6 +60,17 @@ export type WebProbeSentinelOptions =
readonly wait: boolean;
readonly timeoutSeconds: number;
}
| {
readonly kind: "publish";
readonly action: WebProbeSentinelPublishAction;
readonly node: string;
readonly lane: string;
readonly sentinelId: string | null;
readonly dryRun: boolean;
readonly confirm: boolean;
readonly wait: boolean;
readonly timeoutSeconds: number;
}
| {
readonly kind: "maintenance";
readonly action: WebProbeSentinelMaintenanceAction;
@@ -165,6 +177,7 @@ interface SentinelObservedStatus {
readonly gitops: Record<string, unknown>;
readonly argo: Record<string, unknown>;
readonly runtime: Record<string, unknown>;
readonly wait?: Record<string, unknown>;
}
interface SentinelObservedExpectation {
@@ -209,6 +222,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds);
if (options.kind === "image") return runSentinelImage(state, options);
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
if (options.kind === "maintenance") return runSentinelMaintenance(state, options);
if (options.kind === "validate") return runSentinelValidate(state, options);
if (options.kind === "dashboard") return runSentinelDashboard(state, options);
@@ -311,6 +325,112 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
return rendered(result.ok, command, renderControlPlaneResult(result));
}
function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
const command = "web-probe sentinel publish-current";
if (options.confirm) {
if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds);
return runSentinelPublishCurrentConfirmed(state, options);
}
const result = {
ok: state.configReady && state.sourceHead.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
mode: "dry-run",
mutation: false,
specRef: SPEC_REF,
source: state.sourceHead,
image: state.image,
pipelineRun: sentinelPipelineRunName(state),
gitops: {
path: stringAt(state.cicd, "gitopsPath"),
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
manifestSha256: state.manifestSha256,
},
argo: {
namespace: stringAt(state.cicd, "argo.namespace"),
applicationName: stringAt(state.cicd, "argo.applicationName"),
},
budget: publishCurrentBudget(state),
dashboardPlan: publishCurrentDashboardPlan(state),
stageBudgets: publishCurrentStageBudgets(state),
blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" },
next: publishCurrentNext(state),
valuesRedacted: true,
};
return rendered(result.ok === true, command, renderPublishCurrentResult(result));
}
function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
const startedAt = Date.now();
const command = "web-probe sentinel publish-current";
const budget = publishCurrentBudget(state);
const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds"));
const deadline = startedAt + budgetSeconds * 1000;
const remainingBudgetSeconds = () => remainingSeconds(deadline, budgetSeconds);
const controlResult = sentinelControlPlaneConfirmedResult(state, {
kind: "control-plane",
action: "trigger-current",
node: options.node,
lane: options.lane,
sentinelId: options.sentinelId,
dryRun: false,
confirm: true,
wait: true,
timeoutSeconds: remainingBudgetSeconds(),
});
const dashboardRequired = publishCurrentDashboardRequired(state);
let dashboard: Record<string, unknown>;
let dashboardElapsedMs: number | null = null;
if (controlResult.ok !== true) {
dashboard = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true };
} else if (!publishCurrentDashboardEnabled(state)) {
dashboard = { ok: !dashboardRequired, skipped: true, reason: "disabled-by-yaml", valuesRedacted: true };
} else if (remainingBudgetSeconds() < 2) {
dashboard = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-dashboard", valuesRedacted: true };
} else {
const dashboardStartedAt = Date.now();
dashboard = probeSentinelDashboardBrowser(state, publishCurrentDashboardOptions(state, remainingBudgetSeconds()));
dashboardElapsedMs = Date.now() - dashboardStartedAt;
dashboard = { ...dashboard, elapsedMs: dashboardElapsedMs, valuesRedacted: true };
}
const elapsedMs = Date.now() - startedAt;
const timings = publishCurrentStageTimings(controlResult, dashboard, elapsedMs);
const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds);
const withinBudget = elapsedMs <= budgetSeconds * 1000;
const dashboardOk = dashboardRequired ? dashboard.ok === true : dashboard.ok !== false;
const ok = controlResult.ok === true && dashboardOk && withinBudget;
const blocker = ok ? null : publishCurrentBlocker(controlResult, dashboard, withinBudget);
const result = {
ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
mode: "confirm-wait",
mutation: true,
specRef: SPEC_REF,
source: state.sourceHead,
image: state.image,
pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state),
controlPlane: controlResult,
dashboard,
budget,
dashboardPlan: publishCurrentDashboardPlan(state),
stageBudgets: publishCurrentStageBudgets(state),
elapsedMs,
withinBudget,
timings,
slowStages,
warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)),
blocker,
next: publishCurrentNext(state),
valuesRedacted: true,
};
return rendered(ok, command, renderPublishCurrentResult(result));
}
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState {
const sentinel = resolveWebProbeSentinel(spec, sentinelId);
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
@@ -427,6 +547,168 @@ function monitorWebImageBuildNetworkMode(cicd: Record<string, unknown>): "defaul
return value;
}
function publishCurrentBudget(state: SentinelCicdState): Record<string, unknown> {
const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget");
return {
maxSeconds: numberAt(budget, "maxSeconds"),
valuesRedacted: true,
};
}
function publishCurrentStageBudgets(state: SentinelCicdState): Record<string, unknown> {
const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets");
return {
sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"),
sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"),
monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"),
imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"),
gitopsSeconds: numberAt(budgets, "gitopsSeconds"),
argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"),
dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"),
valuesRedacted: true,
};
}
function publishCurrentDashboardPlan(state: SentinelCicdState): Record<string, unknown> {
const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard");
const viewport = stringAt(dashboard, "viewport");
if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`);
return {
enabled: booleanAt(dashboard, "enabled"),
required: booleanAt(dashboard, "required"),
viewport,
timeoutMs: numberAt(dashboard, "timeoutMs"),
waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"),
commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"),
fullPage: booleanAt(dashboard, "fullPage"),
valuesRedacted: true,
};
}
function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean {
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled");
}
function publishCurrentDashboardRequired(state: SentinelCicdState): boolean {
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required");
}
function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract<WebProbeSentinelOptions, { kind: "dashboard" }> {
const dashboard = publishCurrentDashboardPlan(state);
const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000));
const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs);
const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs);
return {
kind: "dashboard",
action: "verify",
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
viewport: stringAt(dashboard, "viewport"),
localDir: "/tmp",
name: null,
timeoutMs,
waitTimeoutMs,
timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
fullPage: booleanAt(dashboard, "fullPage"),
raw: false,
};
}
function publishCurrentStageTimings(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, elapsedMs: number): Record<string, unknown> {
const publish = record(controlResult.publish);
const payload = record(publish.payload);
const stageTimings = record(payload.stageTimings);
const observedWait = record(record(controlResult.observed).wait);
return {
sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs),
sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs),
monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs),
imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs),
gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs),
argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs),
dashboardVerifyMs: finiteNumberOrNull(dashboard.elapsedMs),
totalMs: elapsedMs,
valuesRedacted: true,
};
}
function publishCurrentSlowStages(state: SentinelCicdState, timings: Record<string, unknown>, budgetSeconds: number): Record<string, unknown>[] {
const budgets = publishCurrentStageBudgets(state);
const stageMap: Array<[string, string, string, string]> = [
["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"],
["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"],
["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"],
["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"],
["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"],
["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"],
["dashboard-verify", "dashboardVerifyMs", "dashboardVerifySeconds", "inspect remote browser startup, monitor-web public route and dashboard API latency"],
];
const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => {
const elapsed = finiteNumberOrNull(timings[timingKey]);
const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]);
if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return [];
return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }];
});
const total = finiteNumberOrNull(timings.totalMs);
if (total !== null && total > budgetSeconds * 1000) {
slow.push({
stage: "total",
elapsedMs: total,
budgetSeconds,
suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or dashboard segment before rerun",
valuesRedacted: true,
});
}
return slow;
}
function publishCurrentBudgetWarnings(slowStages: readonly Record<string, unknown>[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] {
const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`);
if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`);
return warnings;
}
function publishCurrentBlocker(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, withinBudget: boolean): Record<string, unknown> {
if (controlResult.ok !== true) {
const blocker = record(controlResult.blocker);
return {
code: blocker.code ?? "sentinel-publish-current-control-plane-blocked",
reason: blocker.reason ?? "control-plane publish did not converge",
valuesRedacted: true,
};
}
if (dashboard.ok !== true) {
return {
code: dashboard.skipped === true ? text(dashboard.reason) : "sentinel-publish-current-dashboard-verify-failed",
reason: dashboard.skipped === true ? "dashboard verification did not run" : "dashboard verification did not pass",
valuesRedacted: true,
};
}
if (!withinBudget) {
return {
code: "sentinel-publish-current-over-budget",
reason: "runtime and dashboard converged, but the one-click CI/CD path exceeded the YAML end-to-end budget",
valuesRedacted: true,
};
}
return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true };
}
function publishCurrentNext(state: SentinelCicdState): Record<string, string> {
const node = state.spec.nodeId;
const lane = state.spec.lane;
const suffix = sentinelCliSuffix(state);
return {
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`,
controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`,
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`,
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`,
};
}
function renderSentinelManifests(
spec: HwlabRuntimeLaneSpec,
sentinelId: string,
@@ -931,6 +1213,11 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra
}
function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
const result = sentinelControlPlaneConfirmedResult(state, options);
return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result));
}
function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): Record<string, unknown> {
const startedAt = Date.now();
const command = `web-probe sentinel control-plane ${options.action}`;
const applyOnly = options.action === "apply";
@@ -953,25 +1240,35 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds());
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false);
const observedReady = sentinelObservedReady(observed);
const publishReady = applyOnly || record(publish).ok === true || observedReady;
const flushReady = applyOnly || record(flush).ok === true || observedReady;
const targetValidation = null;
const targetValidationBlocked = false;
const ok = state.configReady
&& state.sourceHead.ok
&& sourceMirrorReady
&& (applyOnly || record(publish).ok === true)
&& (applyOnly || record(flush).ok === true)
&& publishReady
&& flushReady
&& record(runtimeSecretsApply).ok === true
&& record(publicExposureApply).ok === true
&& record(argoApply).ok === true
&& observedReady;
const elapsedMs = Date.now() - startedAt;
const blocker = ok ? null : {
code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready",
code: !sourceMirrorReady
? "sentinel-source-mirror-sync-failed"
: !publishReady
? "sentinel-image-gitops-publish-not-ready"
: record(runtimeSecretsApply).ok === false
? "sentinel-runtime-secret-sync-failed"
: "sentinel-control-plane-not-ready",
reason: !sourceMirrorReady
? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
: record(runtimeSecretsApply).ok === false
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
: "one or more publish, publicExposure, Argo or runtime observation checks did not pass",
: !publishReady
? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet"
: record(runtimeSecretsApply).ok === false
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
: "one or more publicExposure, Argo or runtime observation checks did not pass",
};
const result = {
ok,
@@ -1022,7 +1319,8 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"),
...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds),
...asyncGitMirrorFlushWarnings(flush),
...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds),
...publishSatisfiedByObservedWarnings(publish, flush, observedReady),
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
...sentinelObservedWarnings(observed),
...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
@@ -1034,15 +1332,17 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
next: controlPlaneNext(state, options.action),
valuesRedacted: true,
};
return rendered(ok, command, renderControlPlaneResult(result));
return result;
}
function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane", action: string, timeoutSeconds: number): RenderedCliResult {
function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult {
const args = domain === "image"
? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
: ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
: domain === "control-plane"
? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
: ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`);
const command = `web-probe sentinel ${domain} ${action}`;
const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`;
const result = {
ok: true,
command,
@@ -1081,11 +1381,11 @@ function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record<stri
};
}
function asyncGitMirrorFlushWarnings(flush: unknown): string[] {
function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): string[] {
const item = record(flush);
if (item.mode !== "async-job") return [];
const next = record(item.next);
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under 120s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
}
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
@@ -1111,11 +1411,23 @@ function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
const startedAt = Date.now();
const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
let polls = 1;
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
polls += 1;
}
return observed;
return {
...observed,
wait: {
polls,
elapsedMs: Date.now() - startedAt,
timeoutMs,
ready: sentinelObservedReady(observed),
includeGitMirror,
valuesRedacted: true,
},
};
}
function sentinelObservedReady(value: Record<string, unknown> | SentinelObservedStatus): boolean {
@@ -2238,11 +2550,18 @@ function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string
return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`];
}
function sentinelElapsedWarnings(value: unknown, subject = "sentinel confirmed operation", budgetSeconds = 120): string[] {
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s timing budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, investigate wait-stage latency without treating timing alone as HWLAB business blockage.`];
function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] {
if (!observedReady) return [];
const warnings: string[] = [];
const publishRecord = record(publish);
if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) {
warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`);
}
const flushRecord = record(flush);
if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) {
warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout.");
}
return warnings;
}
function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number {
@@ -2944,6 +3263,134 @@ function renderPublishResult(publish: Record<string, unknown>): string {
return lines.join("\n");
}
function renderPublishCurrentResult(result: Record<string, unknown>): string {
const source = record(result.source);
const image = record(result.image);
const controlPlane = record(result.controlPlane);
const publish = record(controlPlane.publish);
const publishPayload = record(publish.payload);
const observed = record(controlPlane.observed);
const gitops = record(observed.gitops);
const argo = record(observed.argo);
const runtime = record(observed.runtime);
const runtimeDeployment = record(record(runtime.probe).deployment);
const dashboard = record(result.dashboard);
const dashboardPage = record(dashboard.page);
const dashboardDom = record(dashboardPage.dom);
const latestRunCounts = record(dashboardDom.latestRunCounts);
const checkScope = record(dashboardDom.checkScope);
const timings = record(result.timings);
const budget = record(result.budget);
const stageBudgets = record(result.stageBudgets);
const dashboardPlan = record(result.dashboardPlan);
const blocker = record(result.blocker);
const next = record(result.next);
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : [];
const lines = [
String(result.command),
"",
table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[
result.node,
result.lane,
result.sentinelId,
result.ok === true ? "ok" : "blocked",
result.mode,
budget.maxSeconds ?? "-",
finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000),
]]),
"",
table(["SOURCE", "COMMIT", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
`${source.repository ?? "-"}@${source.branch ?? "-"}`,
short(source.commit),
image.ref ?? "-",
short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest),
result.pipelineRun ?? publish.jobName ?? "-",
]]),
"",
table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "DASHBOARD"], [[
short(gitops.revision),
short(argo.revision),
`${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`,
short(runtimeDeployment.image),
`${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`,
dashboard.ok === true ? "pass" : dashboard.skipped === true ? `skipped:${text(dashboard.reason)}` : Object.keys(dashboard).length === 0 ? "planned" : "blocked",
]]),
"",
table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "DASHBOARD_MS", "TOTAL_MS"], [[
timings.sourceSyncMs ?? "-",
timings.sourceFetchMs ?? "-",
timings.monitorWebVerifyMs ?? "-",
timings.imageBuildMs ?? "-",
timings.gitopsMs ?? "-",
timings.argoRuntimeMs ?? "-",
timings.dashboardVerifyMs ?? "-",
timings.totalMs ?? "-",
]]),
"",
table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "DASHBOARD"], [[
"YAML publishCurrent",
stageBudgets.sourceSyncSeconds ?? "-",
stageBudgets.sourceFetchSeconds ?? "-",
stageBudgets.monitorWebVerifySeconds ?? "-",
stageBudgets.imageBuildSeconds ?? "-",
stageBudgets.gitopsSeconds ?? "-",
stageBudgets.argoRuntimeSeconds ?? "-",
stageBudgets.dashboardVerifySeconds ?? "-",
]]),
];
if (Object.keys(publish).length > 0) {
const payloadImageBuild = record(publishPayload.imageBuild);
const payloadEnvReuse = record(publishPayload.envReuse);
lines.push(
"",
table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[
payloadEnvReuse.dependencyReuse ?? "-",
payloadEnvReuse.nodeDepsPresent ?? "-",
payloadImageBuild.packageMode ?? "-",
payloadImageBuild.networkMode ?? "-",
payloadImageBuild.layerCache ?? "-",
payloadImageBuild.cacheHitLines ?? "-",
]]),
);
}
lines.push(
"",
Object.keys(dashboard).length === 0
? "DASHBOARD_VERIFY\n-"
: table(["URL", "HTTP", "LATEST_RUN", "CHECK_SCOPE", "CHECK_MATCH", "REQ_FAIL", "CONSOLE_ERR"], [[
dashboard.publicUrl ?? "-",
dashboardPage.httpStatus ?? "-",
latestRunCounts.runId ?? "-",
checkScope.scope ?? "-",
checkScope.matchesRunDetail ?? "-",
dashboardPage.requestFailureCount ?? "-",
dashboardPage.consoleErrorCount ?? "-",
]]),
"",
slowStages.length === 0 ? "SLOW_STAGES\n-" : [
"SLOW_STAGES",
table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])),
].join("\n"),
"",
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
"",
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
"",
"NEXT",
` publish-current: ${next.publishCurrent ?? "-"}`,
` status: ${next.controlPlaneStatus ?? "-"}`,
` dashboard: ${next.dashboardVerify ?? "-"}`,
` git-mirror: ${next.gitMirrorStatus ?? "-"}`,
` flush: ${next.gitMirrorFlush ?? "-"}`,
"",
"DISCLOSURE",
` end-to-end and stage budgets are read from ${Object.keys(dashboardPlan).length > 0 ? "publishCurrent YAML" : "YAML-required publishCurrent fields"}.`,
" image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.",
);
return lines.join("\n");
}
function renderImageResult(result: Record<string, unknown>): string {
const source = record(result.source);
const sourceMirror = record(result.sourceMirror);
@@ -3197,6 +3644,16 @@ export function numberAt(value: unknown, path: string): number {
return found;
}
function booleanAt(value: unknown, path: string): boolean {
const found = valueAtPath(value, path);
if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`);
return found;
}
function finiteNumberOrNull(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
export function arrayAt(value: unknown, path: string): unknown[] {
const found = valueAtPath(value, path);
if (!Array.isArray(found)) throw new Error(`${path} must be an array`);
+1 -1
View File
@@ -391,7 +391,7 @@ export function runSentinelDashboard(state: SentinelCicdState, options: Extract<
return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result));
}
function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const [widthRaw, heightRaw] = options.viewport.split("x");
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
+4 -1
View File
@@ -45,12 +45,13 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
&& sentinelActionRaw !== "status"
&& sentinelActionRaw !== "image"
&& sentinelActionRaw !== "control-plane"
&& sentinelActionRaw !== "publish-current"
&& sentinelActionRaw !== "validate"
&& sentinelActionRaw !== "maintenance"
&& sentinelActionRaw !== "dashboard"
&& sentinelActionRaw !== "report"
) {
throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|validate|maintenance|dashboard|report --node NODE --lane vNN [--dry-run|--confirm]");
throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|publish-current|validate|maintenance|dashboard|report --node NODE --lane vNN [--dry-run|--confirm]");
}
assertKnownOptions(args, new Set([
"--node",
@@ -96,6 +97,8 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]");
}
sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds };
} else if (sentinelActionRaw === "publish-current") {
sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds };
} else if (sentinelActionRaw === "maintenance") {
const maintenanceAction = args[1];
if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") {