|
|
|
@@ -18,7 +18,7 @@ import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-c
|
|
|
|
|
import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver";
|
|
|
|
|
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
|
|
|
|
import type { RenderedCliResult } from "./output";
|
|
|
|
|
import { probeSentinelDashboardBrowser, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
|
|
|
|
import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
|
|
|
|
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
|
|
|
|
|
|
|
|
|
export type WebProbeSentinelConfigAction = "plan" | "status";
|
|
|
|
@@ -353,7 +353,7 @@ function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<We
|
|
|
|
|
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
|
|
|
|
},
|
|
|
|
|
budget: publishCurrentBudget(state),
|
|
|
|
|
dashboardPlan: publishCurrentDashboardPlan(state),
|
|
|
|
|
validationPlan: publishCurrentHealthValidationPlan(state),
|
|
|
|
|
stageBudgets: publishCurrentStageBudgets(state),
|
|
|
|
|
blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" },
|
|
|
|
|
next: publishCurrentNext(state),
|
|
|
|
@@ -386,7 +386,6 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const dashboardReserveSeconds = publishCurrentDashboardReserveSeconds(state);
|
|
|
|
|
controlResult ??= sentinelControlPlaneConfirmedResult(state, {
|
|
|
|
|
kind: "control-plane",
|
|
|
|
|
action: "trigger-current",
|
|
|
|
@@ -396,30 +395,27 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
dryRun: false,
|
|
|
|
|
confirm: true,
|
|
|
|
|
wait: true,
|
|
|
|
|
timeoutSeconds: Math.max(1, remainingBudgetSeconds() - dashboardReserveSeconds),
|
|
|
|
|
timeoutSeconds: Math.max(1, remainingBudgetSeconds()),
|
|
|
|
|
});
|
|
|
|
|
const dashboardRequired = publishCurrentDashboardRequired(state);
|
|
|
|
|
let dashboard: Record<string, unknown>;
|
|
|
|
|
let dashboardElapsedMs: number | null = null;
|
|
|
|
|
let health: Record<string, unknown>;
|
|
|
|
|
let healthElapsedMs: number | null = null;
|
|
|
|
|
if (controlResult.ok !== true) {
|
|
|
|
|
dashboard = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true };
|
|
|
|
|
} else if (!publishCurrentDashboardEnabled(state)) {
|
|
|
|
|
dashboard = { ok: !dashboardRequired, skipped: true, reason: "disabled-by-yaml", valuesRedacted: true };
|
|
|
|
|
health = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true };
|
|
|
|
|
} else if (remainingBudgetSeconds() < 2) {
|
|
|
|
|
dashboard = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-dashboard", valuesRedacted: true };
|
|
|
|
|
health = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-health", valuesRedacted: true };
|
|
|
|
|
} else {
|
|
|
|
|
const dashboardStartedAt = Date.now();
|
|
|
|
|
dashboard = probeSentinelDashboardBrowser(state, publishCurrentDashboardOptions(state, remainingBudgetSeconds()));
|
|
|
|
|
dashboardElapsedMs = Date.now() - dashboardStartedAt;
|
|
|
|
|
dashboard = { ...dashboard, elapsedMs: dashboardElapsedMs, valuesRedacted: true };
|
|
|
|
|
const healthStartedAt = Date.now();
|
|
|
|
|
health = probeSentinelRuntimeHealthEndpoint(state, remainingBudgetSeconds());
|
|
|
|
|
healthElapsedMs = Date.now() - healthStartedAt;
|
|
|
|
|
health = { ...health, elapsedMs: healthElapsedMs, valuesRedacted: true };
|
|
|
|
|
}
|
|
|
|
|
const elapsedMs = Date.now() - startedAt;
|
|
|
|
|
const timings = publishCurrentStageTimings(controlResult, dashboard, elapsedMs);
|
|
|
|
|
const timings = publishCurrentStageTimings(controlResult, health, elapsedMs);
|
|
|
|
|
const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds);
|
|
|
|
|
const withinBudget = elapsedMs <= budgetSeconds * 1000;
|
|
|
|
|
const dashboardOk = dashboardRequired ? dashboard.ok === true : dashboard.ok !== false;
|
|
|
|
|
const ok = controlResult.ok === true && dashboardOk && withinBudget;
|
|
|
|
|
const blocker = ok ? null : publishCurrentBlocker(controlResult, dashboard, withinBudget);
|
|
|
|
|
const healthOk = health.ok === true;
|
|
|
|
|
const ok = controlResult.ok === true && healthOk && withinBudget;
|
|
|
|
|
const blocker = ok ? null : publishCurrentBlocker(controlResult, health, withinBudget);
|
|
|
|
|
const result = {
|
|
|
|
|
ok,
|
|
|
|
|
command,
|
|
|
|
@@ -433,9 +429,9 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
|
|
|
|
|
image: state.image,
|
|
|
|
|
pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state),
|
|
|
|
|
controlPlane: controlResult,
|
|
|
|
|
dashboard,
|
|
|
|
|
health,
|
|
|
|
|
budget,
|
|
|
|
|
dashboardPlan: publishCurrentDashboardPlan(state),
|
|
|
|
|
validationPlan: publishCurrentHealthValidationPlan(state),
|
|
|
|
|
stageBudgets: publishCurrentStageBudgets(state),
|
|
|
|
|
elapsedMs,
|
|
|
|
|
withinBudget,
|
|
|
|
@@ -552,7 +548,7 @@ function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed:
|
|
|
|
|
targetValidation: null,
|
|
|
|
|
elapsedMs,
|
|
|
|
|
warnings: [
|
|
|
|
|
"publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used dashboard verification only.",
|
|
|
|
|
"publish-current already-current fast path: source mirror, registry, GitOps, Argo and runtime already match the selected source; skipped Tekton publish and used health endpoint validation only.",
|
|
|
|
|
...sentinelObservedWarnings(observed),
|
|
|
|
|
...targetValidationDeferredWarnings(state, false, controlPlaneWaitWarningSeconds(state)),
|
|
|
|
|
],
|
|
|
|
@@ -754,6 +750,19 @@ function publishCurrentDashboardRequired(state: SentinelCicdState): boolean {
|
|
|
|
|
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentHealthValidationPlan(state: SentinelCicdState): Record<string, unknown> {
|
|
|
|
|
return {
|
|
|
|
|
enabled: true,
|
|
|
|
|
required: true,
|
|
|
|
|
endpoint: stringAt(state.runtime, "healthPath"),
|
|
|
|
|
source: "runtime.healthPath",
|
|
|
|
|
browser: false,
|
|
|
|
|
playwright: false,
|
|
|
|
|
webProbe: false,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentDashboardReserveSeconds(state: SentinelCicdState): number {
|
|
|
|
|
if (!publishCurrentDashboardEnabled(state)) return 0;
|
|
|
|
|
const dashboard = publishCurrentDashboardPlan(state);
|
|
|
|
@@ -788,7 +797,7 @@ function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentStageTimings(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, elapsedMs: number): Record<string, unknown> {
|
|
|
|
|
function publishCurrentStageTimings(controlResult: Record<string, unknown>, health: Record<string, unknown>, elapsedMs: number): Record<string, unknown> {
|
|
|
|
|
const publish = record(controlResult.publish);
|
|
|
|
|
const payload = record(publish.payload);
|
|
|
|
|
const payloadStageTimings = record(payload.stageTimings);
|
|
|
|
@@ -802,7 +811,7 @@ function publishCurrentStageTimings(controlResult: Record<string, unknown>, dash
|
|
|
|
|
imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs),
|
|
|
|
|
gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs),
|
|
|
|
|
argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs),
|
|
|
|
|
dashboardVerifyMs: finiteNumberOrNull(dashboard.elapsedMs),
|
|
|
|
|
healthValidationMs: finiteNumberOrNull(health.elapsedMs),
|
|
|
|
|
totalMs: elapsedMs,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
@@ -817,7 +826,7 @@ function publishCurrentSlowStages(state: SentinelCicdState, timings: Record<stri
|
|
|
|
|
["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"],
|
|
|
|
|
["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"],
|
|
|
|
|
["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"],
|
|
|
|
|
["dashboard-verify", "dashboardVerifyMs", "dashboardVerifySeconds", "inspect remote browser startup, monitor-web public route and dashboard API latency"],
|
|
|
|
|
["health-validation", "healthValidationMs", "dashboardVerifySeconds", "inspect runtime health endpoint latency and service routing"],
|
|
|
|
|
];
|
|
|
|
|
const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => {
|
|
|
|
|
const elapsed = finiteNumberOrNull(timings[timingKey]);
|
|
|
|
@@ -831,7 +840,7 @@ function publishCurrentSlowStages(state: SentinelCicdState, timings: Record<stri
|
|
|
|
|
stage: "total",
|
|
|
|
|
elapsedMs: total,
|
|
|
|
|
budgetSeconds,
|
|
|
|
|
suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or dashboard segment before rerun",
|
|
|
|
|
suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or health endpoint segment before rerun",
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
@@ -844,7 +853,7 @@ function publishCurrentBudgetWarnings(slowStages: readonly Record<string, unknow
|
|
|
|
|
return warnings;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function publishCurrentBlocker(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, withinBudget: boolean): Record<string, unknown> {
|
|
|
|
|
function publishCurrentBlocker(controlResult: Record<string, unknown>, health: Record<string, unknown>, withinBudget: boolean): Record<string, unknown> {
|
|
|
|
|
if (controlResult.ok !== true) {
|
|
|
|
|
const blocker = record(controlResult.blocker);
|
|
|
|
|
return {
|
|
|
|
@@ -853,22 +862,22 @@ function publishCurrentBlocker(controlResult: Record<string, unknown>, dashboard
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (dashboard.ok !== true) {
|
|
|
|
|
const degradedReason = text(dashboard.degradedReason);
|
|
|
|
|
if (health.ok !== true) {
|
|
|
|
|
const degradedReason = text(health.degradedReason);
|
|
|
|
|
return {
|
|
|
|
|
code: dashboard.skipped === true ? text(dashboard.reason) : "sentinel-publish-current-dashboard-verify-failed",
|
|
|
|
|
reason: dashboard.skipped === true
|
|
|
|
|
? "dashboard verification did not run"
|
|
|
|
|
code: health.skipped === true ? text(health.reason) : "sentinel-publish-current-health-endpoint-failed",
|
|
|
|
|
reason: health.skipped === true
|
|
|
|
|
? "health endpoint validation did not run"
|
|
|
|
|
: degradedReason === "-"
|
|
|
|
|
? "dashboard verification did not pass"
|
|
|
|
|
: `dashboard verification did not pass: ${degradedReason}`,
|
|
|
|
|
? "health endpoint validation did not pass"
|
|
|
|
|
: `health endpoint validation did not pass: ${degradedReason}`,
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (!withinBudget) {
|
|
|
|
|
return {
|
|
|
|
|
code: "sentinel-publish-current-over-budget",
|
|
|
|
|
reason: "runtime and dashboard converged, but the one-click CI/CD path exceeded the YAML end-to-end budget",
|
|
|
|
|
reason: "runtime and health endpoint converged, but the one-click CI/CD path exceeded the YAML end-to-end budget",
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
@@ -2886,7 +2895,7 @@ function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, prob
|
|
|
|
|
function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] {
|
|
|
|
|
if (applyOnly) return [];
|
|
|
|
|
const next = sentinelP5Next(state);
|
|
|
|
|
return [`targetValidation quick verify is deferred from control-plane confirm-wait to keep CI/CD wait under ${Math.round(budgetSeconds)}s; run ${next.quickVerify}.`];
|
|
|
|
|
return [`targetValidation quick verify is outside the CI/CD validation gate; run ${next.quickVerify} only as separate post-deploy evidence if needed.`];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
|
|
|
|
@@ -3559,15 +3568,12 @@ function renderPublishCurrentResult(result: Record<string, unknown>): string {
|
|
|
|
|
const argo = record(observed.argo);
|
|
|
|
|
const runtime = record(observed.runtime);
|
|
|
|
|
const runtimeDeployment = record(record(runtime.probe).deployment);
|
|
|
|
|
const dashboard = record(result.dashboard);
|
|
|
|
|
const dashboardPage = record(dashboard.page);
|
|
|
|
|
const dashboardDom = record(dashboardPage.dom);
|
|
|
|
|
const latestRunCounts = record(dashboardDom.latestRunCounts);
|
|
|
|
|
const checkScope = record(dashboardDom.checkScope);
|
|
|
|
|
const health = record(result.health);
|
|
|
|
|
const healthBody = record(record(health.health).bodyJson);
|
|
|
|
|
const timings = record(result.timings);
|
|
|
|
|
const budget = record(result.budget);
|
|
|
|
|
const stageBudgets = record(result.stageBudgets);
|
|
|
|
|
const dashboardPlan = record(result.dashboardPlan);
|
|
|
|
|
const validationPlan = record(result.validationPlan);
|
|
|
|
|
const blocker = record(result.blocker);
|
|
|
|
|
const next = record(result.next);
|
|
|
|
|
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
|
|
@@ -3593,27 +3599,27 @@ function renderPublishCurrentResult(result: Record<string, unknown>): string {
|
|
|
|
|
result.pipelineRun ?? publish.jobName ?? "-",
|
|
|
|
|
]]),
|
|
|
|
|
"",
|
|
|
|
|
table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "DASHBOARD"], [[
|
|
|
|
|
table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "HEALTH"], [[
|
|
|
|
|
short(gitops.revision),
|
|
|
|
|
short(argo.revision),
|
|
|
|
|
`${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`,
|
|
|
|
|
short(runtimeDeployment.image),
|
|
|
|
|
`${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`,
|
|
|
|
|
dashboard.ok === true ? "pass" : dashboard.skipped === true ? `skipped:${text(dashboard.reason)}` : Object.keys(dashboard).length === 0 ? "planned" : "blocked",
|
|
|
|
|
health.ok === true ? "pass" : health.skipped === true ? `skipped:${text(health.reason)}` : Object.keys(health).length === 0 ? "planned" : "blocked",
|
|
|
|
|
]]),
|
|
|
|
|
"",
|
|
|
|
|
table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "DASHBOARD_MS", "TOTAL_MS"], [[
|
|
|
|
|
table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "VALIDATION_MS", "TOTAL_MS"], [[
|
|
|
|
|
timings.sourceSyncMs ?? "-",
|
|
|
|
|
timings.sourceFetchMs ?? "-",
|
|
|
|
|
timings.monitorWebVerifyMs ?? "-",
|
|
|
|
|
timings.imageBuildMs ?? "-",
|
|
|
|
|
timings.gitopsMs ?? "-",
|
|
|
|
|
timings.argoRuntimeMs ?? "-",
|
|
|
|
|
timings.dashboardVerifyMs ?? "-",
|
|
|
|
|
timings.healthValidationMs ?? "-",
|
|
|
|
|
timings.totalMs ?? "-",
|
|
|
|
|
]]),
|
|
|
|
|
"",
|
|
|
|
|
table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "DASHBOARD"], [[
|
|
|
|
|
table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "VALIDATION"], [[
|
|
|
|
|
"YAML publishCurrent",
|
|
|
|
|
stageBudgets.sourceSyncSeconds ?? "-",
|
|
|
|
|
stageBudgets.sourceFetchSeconds ?? "-",
|
|
|
|
@@ -3641,16 +3647,15 @@ function renderPublishCurrentResult(result: Record<string, unknown>): string {
|
|
|
|
|
}
|
|
|
|
|
lines.push(
|
|
|
|
|
"",
|
|
|
|
|
Object.keys(dashboard).length === 0
|
|
|
|
|
? "DASHBOARD_VERIFY\n-"
|
|
|
|
|
: table(["URL", "HTTP", "LATEST_RUN", "CHECK_SCOPE", "CHECK_MATCH", "REQ_FAIL", "CONSOLE_ERR"], [[
|
|
|
|
|
dashboard.publicUrl ?? "-",
|
|
|
|
|
dashboardPage.httpStatus ?? "-",
|
|
|
|
|
latestRunCounts.runId ?? "-",
|
|
|
|
|
checkScope.scope ?? "-",
|
|
|
|
|
checkScope.matchesRunDetail ?? "-",
|
|
|
|
|
dashboardPage.requestFailureCount ?? "-",
|
|
|
|
|
dashboardPage.consoleErrorCount ?? "-",
|
|
|
|
|
Object.keys(health).length === 0
|
|
|
|
|
? "HEALTH_VALIDATION\n-"
|
|
|
|
|
: table(["ENDPOINT", "HTTP", "OK", "STATUS", "PUBLIC_URL", "INTERNAL_URL"], [[
|
|
|
|
|
health.endpoint ?? validationPlan.endpoint ?? "-",
|
|
|
|
|
health.httpStatus ?? "-",
|
|
|
|
|
healthBody.ok ?? "-",
|
|
|
|
|
healthBody.status ?? "-",
|
|
|
|
|
health.publicUrl ?? "-",
|
|
|
|
|
health.internalUrl ?? "-",
|
|
|
|
|
]]),
|
|
|
|
|
"",
|
|
|
|
|
slowStages.length === 0 ? "SLOW_STAGES\n-" : [
|
|
|
|
@@ -3665,12 +3670,13 @@ function renderPublishCurrentResult(result: Record<string, unknown>): string {
|
|
|
|
|
"NEXT",
|
|
|
|
|
` publish-current: ${next.publishCurrent ?? "-"}`,
|
|
|
|
|
` status: ${next.controlPlaneStatus ?? "-"}`,
|
|
|
|
|
` dashboard: ${next.dashboardVerify ?? "-"}`,
|
|
|
|
|
` post-deploy-dashboard: ${next.dashboardVerify ?? "-"}`,
|
|
|
|
|
` git-mirror: ${next.gitMirrorStatus ?? "-"}`,
|
|
|
|
|
` flush: ${next.gitMirrorFlush ?? "-"}`,
|
|
|
|
|
"",
|
|
|
|
|
"DISCLOSURE",
|
|
|
|
|
` end-to-end and stage budgets are read from ${Object.keys(dashboardPlan).length > 0 ? "publishCurrent YAML" : "YAML-required publishCurrent fields"}.`,
|
|
|
|
|
` end-to-end and stage budgets are read from ${Object.keys(validationPlan).length > 0 ? "publishCurrent YAML and runtime.healthPath" : "YAML-required publishCurrent fields"}.`,
|
|
|
|
|
" CI/CD validation only checks the configured health endpoint; web-probe, Playwright and browser dashboard checks are post-deploy evidence, not this gate.",
|
|
|
|
|
" image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.",
|
|
|
|
|
);
|
|
|
|
|
return lines.join("\n");
|
|
|
|
|
Codex