fix: add one-click web sentinel publish
This commit is contained in:
Codex
@@ -18,12 +18,13 @@ import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-c
|
||||
import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver";
|
||||
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
||||
import { probeSentinelDashboardBrowser, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
||||
import { remainingSeconds, runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
|
||||
export type WebProbeSentinelConfigAction = "plan" | "status";
|
||||
export type WebProbeSentinelImageAction = "status" | "build";
|
||||
export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current";
|
||||
export type WebProbeSentinelPublishAction = "publish-current";
|
||||
export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop";
|
||||
export type WebProbeSentinelDashboardAction = "verify" | "screenshot";
|
||||
export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary";
|
||||
@@ -59,6 +60,17 @@ export type WebProbeSentinelOptions =
|
||||
readonly wait: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
}
|
||||
| {
|
||||
readonly kind: "publish";
|
||||
readonly action: WebProbeSentinelPublishAction;
|
||||
readonly node: string;
|
||||
readonly lane: string;
|
||||
readonly sentinelId: string | null;
|
||||
readonly dryRun: boolean;
|
||||
readonly confirm: boolean;
|
||||
readonly wait: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
}
|
||||
| {
|
||||
readonly kind: "maintenance";
|
||||
readonly action: WebProbeSentinelMaintenanceAction;
|
||||
@@ -165,6 +177,7 @@ interface SentinelObservedStatus {
|
||||
readonly gitops: Record<string, unknown>;
|
||||
readonly argo: Record<string, unknown>;
|
||||
readonly runtime: Record<string, unknown>;
|
||||
readonly wait?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface SentinelObservedExpectation {
|
||||
@@ -209,6 +222,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
|
||||
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds);
|
||||
if (options.kind === "image") return runSentinelImage(state, options);
|
||||
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
|
||||
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
|
||||
if (options.kind === "maintenance") return runSentinelMaintenance(state, options);
|
||||
if (options.kind === "validate") return runSentinelValidate(state, options);
|
||||
if (options.kind === "dashboard") return runSentinelDashboard(state, options);
|
||||
@@ -311,6 +325,112 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
|
||||
return rendered(result.ok, command, renderControlPlaneResult(result));
|
||||
}
|
||||
|
||||
function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
|
||||
const command = "web-probe sentinel publish-current";
|
||||
if (options.confirm) {
|
||||
if (!options.wait) return renderAsyncSentinelJob(state, "publish", options.action, options.timeoutSeconds);
|
||||
return runSentinelPublishCurrentConfirmed(state, options);
|
||||
}
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
mode: "dry-run",
|
||||
mutation: false,
|
||||
specRef: SPEC_REF,
|
||||
source: state.sourceHead,
|
||||
image: state.image,
|
||||
pipelineRun: sentinelPipelineRunName(state),
|
||||
gitops: {
|
||||
path: stringAt(state.cicd, "gitopsPath"),
|
||||
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
||||
manifestSha256: state.manifestSha256,
|
||||
},
|
||||
argo: {
|
||||
namespace: stringAt(state.cicd, "argo.namespace"),
|
||||
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
||||
},
|
||||
budget: publishCurrentBudget(state),
|
||||
dashboardPlan: publishCurrentDashboardPlan(state),
|
||||
stageBudgets: publishCurrentStageBudgets(state),
|
||||
blocker: state.configReady && state.sourceHead.ok ? null : { code: "sentinel-publish-current-plan-blocked", reason: "sentinel config or source head is not ready" },
|
||||
next: publishCurrentNext(state),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok === true, command, renderPublishCurrentResult(result));
|
||||
}
|
||||
|
||||
function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "publish" }>): RenderedCliResult {
|
||||
const startedAt = Date.now();
|
||||
const command = "web-probe sentinel publish-current";
|
||||
const budget = publishCurrentBudget(state);
|
||||
const budgetSeconds = Math.min(options.timeoutSeconds, numberAt(budget, "maxSeconds"));
|
||||
const deadline = startedAt + budgetSeconds * 1000;
|
||||
const remainingBudgetSeconds = () => remainingSeconds(deadline, budgetSeconds);
|
||||
const controlResult = sentinelControlPlaneConfirmedResult(state, {
|
||||
kind: "control-plane",
|
||||
action: "trigger-current",
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
sentinelId: options.sentinelId,
|
||||
dryRun: false,
|
||||
confirm: true,
|
||||
wait: true,
|
||||
timeoutSeconds: remainingBudgetSeconds(),
|
||||
});
|
||||
const dashboardRequired = publishCurrentDashboardRequired(state);
|
||||
let dashboard: Record<string, unknown>;
|
||||
let dashboardElapsedMs: number | null = null;
|
||||
if (controlResult.ok !== true) {
|
||||
dashboard = { ok: false, skipped: true, reason: "control-plane-blocked", valuesRedacted: true };
|
||||
} else if (!publishCurrentDashboardEnabled(state)) {
|
||||
dashboard = { ok: !dashboardRequired, skipped: true, reason: "disabled-by-yaml", valuesRedacted: true };
|
||||
} else if (remainingBudgetSeconds() < 2) {
|
||||
dashboard = { ok: false, skipped: true, reason: "end-to-end-budget-exhausted-before-dashboard", valuesRedacted: true };
|
||||
} else {
|
||||
const dashboardStartedAt = Date.now();
|
||||
dashboard = probeSentinelDashboardBrowser(state, publishCurrentDashboardOptions(state, remainingBudgetSeconds()));
|
||||
dashboardElapsedMs = Date.now() - dashboardStartedAt;
|
||||
dashboard = { ...dashboard, elapsedMs: dashboardElapsedMs, valuesRedacted: true };
|
||||
}
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
const timings = publishCurrentStageTimings(controlResult, dashboard, elapsedMs);
|
||||
const slowStages = publishCurrentSlowStages(state, timings, budgetSeconds);
|
||||
const withinBudget = elapsedMs <= budgetSeconds * 1000;
|
||||
const dashboardOk = dashboardRequired ? dashboard.ok === true : dashboard.ok !== false;
|
||||
const ok = controlResult.ok === true && dashboardOk && withinBudget;
|
||||
const blocker = ok ? null : publishCurrentBlocker(controlResult, dashboard, withinBudget);
|
||||
const result = {
|
||||
ok,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
mode: "confirm-wait",
|
||||
mutation: true,
|
||||
specRef: SPEC_REF,
|
||||
source: state.sourceHead,
|
||||
image: state.image,
|
||||
pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state),
|
||||
controlPlane: controlResult,
|
||||
dashboard,
|
||||
budget,
|
||||
dashboardPlan: publishCurrentDashboardPlan(state),
|
||||
stageBudgets: publishCurrentStageBudgets(state),
|
||||
elapsedMs,
|
||||
withinBudget,
|
||||
timings,
|
||||
slowStages,
|
||||
warnings: mergeWarnings(controlResult.warnings, publishCurrentBudgetWarnings(slowStages, withinBudget, budgetSeconds, elapsedMs)),
|
||||
blocker,
|
||||
next: publishCurrentNext(state),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(ok, command, renderPublishCurrentResult(result));
|
||||
}
|
||||
|
||||
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState {
|
||||
const sentinel = resolveWebProbeSentinel(spec, sentinelId);
|
||||
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
|
||||
@@ -427,6 +547,168 @@ function monitorWebImageBuildNetworkMode(cicd: Record<string, unknown>): "defaul
|
||||
return value;
|
||||
}
|
||||
|
||||
function publishCurrentBudget(state: SentinelCicdState): Record<string, unknown> {
|
||||
const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget");
|
||||
return {
|
||||
maxSeconds: numberAt(budget, "maxSeconds"),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function publishCurrentStageBudgets(state: SentinelCicdState): Record<string, unknown> {
|
||||
const budgets = recordTarget(valueAtPath(state.cicd, "publishCurrent.stageBudgets"), "publishCurrent.stageBudgets");
|
||||
return {
|
||||
sourceSyncSeconds: numberAt(budgets, "sourceSyncSeconds"),
|
||||
sourceFetchSeconds: numberAt(budgets, "sourceFetchSeconds"),
|
||||
monitorWebVerifySeconds: numberAt(budgets, "monitorWebVerifySeconds"),
|
||||
imageBuildSeconds: numberAt(budgets, "imageBuildSeconds"),
|
||||
gitopsSeconds: numberAt(budgets, "gitopsSeconds"),
|
||||
argoRuntimeSeconds: numberAt(budgets, "argoRuntimeSeconds"),
|
||||
dashboardVerifySeconds: numberAt(budgets, "dashboardVerifySeconds"),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function publishCurrentDashboardPlan(state: SentinelCicdState): Record<string, unknown> {
|
||||
const dashboard = recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard");
|
||||
const viewport = stringAt(dashboard, "viewport");
|
||||
if (!/^[1-9][0-9]{1,4}x[1-9][0-9]{1,4}$/u.test(viewport)) throw new Error(`publishCurrent.dashboard.viewport must look like 1440x900, got ${viewport}`);
|
||||
return {
|
||||
enabled: booleanAt(dashboard, "enabled"),
|
||||
required: booleanAt(dashboard, "required"),
|
||||
viewport,
|
||||
timeoutMs: numberAt(dashboard, "timeoutMs"),
|
||||
waitTimeoutMs: numberAt(dashboard, "waitTimeoutMs"),
|
||||
commandTimeoutSeconds: numberAt(dashboard, "commandTimeoutSeconds"),
|
||||
fullPage: booleanAt(dashboard, "fullPage"),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function publishCurrentDashboardEnabled(state: SentinelCicdState): boolean {
|
||||
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "enabled");
|
||||
}
|
||||
|
||||
function publishCurrentDashboardRequired(state: SentinelCicdState): boolean {
|
||||
return booleanAt(recordTarget(valueAtPath(state.cicd, "publishCurrent.dashboard"), "publishCurrent.dashboard"), "required");
|
||||
}
|
||||
|
||||
function publishCurrentDashboardOptions(state: SentinelCicdState, timeoutSeconds: number): Extract<WebProbeSentinelOptions, { kind: "dashboard" }> {
|
||||
const dashboard = publishCurrentDashboardPlan(state);
|
||||
const remainingMs = Math.max(1000, Math.trunc(timeoutSeconds * 1000));
|
||||
const waitTimeoutMs = Math.min(numberAt(dashboard, "waitTimeoutMs"), remainingMs);
|
||||
const timeoutMs = Math.min(numberAt(dashboard, "timeoutMs"), waitTimeoutMs);
|
||||
return {
|
||||
kind: "dashboard",
|
||||
action: "verify",
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
viewport: stringAt(dashboard, "viewport"),
|
||||
localDir: "/tmp",
|
||||
name: null,
|
||||
timeoutMs,
|
||||
waitTimeoutMs,
|
||||
timeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
|
||||
commandTimeoutSeconds: Math.min(numberAt(dashboard, "commandTimeoutSeconds"), Math.max(1, Math.trunc(timeoutSeconds))),
|
||||
fullPage: booleanAt(dashboard, "fullPage"),
|
||||
raw: false,
|
||||
};
|
||||
}
|
||||
|
||||
function publishCurrentStageTimings(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, elapsedMs: number): Record<string, unknown> {
|
||||
const publish = record(controlResult.publish);
|
||||
const payload = record(publish.payload);
|
||||
const stageTimings = record(payload.stageTimings);
|
||||
const observedWait = record(record(controlResult.observed).wait);
|
||||
return {
|
||||
sourceSyncMs: finiteNumberOrNull(record(controlResult.sourceMirrorSync).elapsedMs),
|
||||
sourceFetchMs: finiteNumberOrNull(stageTimings.sourceFetchMs),
|
||||
monitorWebVerifyMs: finiteNumberOrNull(stageTimings.monitorWebVerifyMs),
|
||||
imageBuildMs: finiteNumberOrNull(stageTimings.imageBuildMs),
|
||||
gitopsMs: finiteNumberOrNull(stageTimings.gitopsMs),
|
||||
argoRuntimeMs: finiteNumberOrNull(observedWait.elapsedMs),
|
||||
dashboardVerifyMs: finiteNumberOrNull(dashboard.elapsedMs),
|
||||
totalMs: elapsedMs,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function publishCurrentSlowStages(state: SentinelCicdState, timings: Record<string, unknown>, budgetSeconds: number): Record<string, unknown>[] {
|
||||
const budgets = publishCurrentStageBudgets(state);
|
||||
const stageMap: Array<[string, string, string, string]> = [
|
||||
["source-sync", "sourceSyncMs", "sourceSyncSeconds", "check git-mirror pre-sync, node-local mirror health and SSH-over-proxy latency"],
|
||||
["source-fetch", "sourceFetchMs", "sourceFetchSeconds", "keep sparse checkout paths narrow and verify node-local git mirror object availability"],
|
||||
["monitor-web-verify", "monitorWebVerifyMs", "monitorWebVerifySeconds", "keep monitor-web verification copy-only and avoid rebuilding frontend assets during publish"],
|
||||
["image-build", "imageBuildMs", "imageBuildSeconds", "verify env reuse node_modules hit, BuildKit layer cache, copy-only Containerfile and image-build proxy route"],
|
||||
["gitops", "gitopsMs", "gitopsSeconds", "inspect GitOps mirror cache, commit/writeback latency and post-flush state"],
|
||||
["argo-runtime", "argoRuntimeMs", "argoRuntimeSeconds", "inspect Argo refresh, runtime Deployment readiness and image digest alignment probes"],
|
||||
["dashboard-verify", "dashboardVerifyMs", "dashboardVerifySeconds", "inspect remote browser startup, monitor-web public route and dashboard API latency"],
|
||||
];
|
||||
const slow = stageMap.flatMap(([stage, timingKey, budgetKey, suggestion]) => {
|
||||
const elapsed = finiteNumberOrNull(timings[timingKey]);
|
||||
const stageBudgetSeconds = finiteNumberOrNull(budgets[budgetKey]);
|
||||
if (elapsed === null || stageBudgetSeconds === null || elapsed <= stageBudgetSeconds * 1000) return [];
|
||||
return [{ stage, elapsedMs: elapsed, budgetSeconds: stageBudgetSeconds, suggestion, valuesRedacted: true }];
|
||||
});
|
||||
const total = finiteNumberOrNull(timings.totalMs);
|
||||
if (total !== null && total > budgetSeconds * 1000) {
|
||||
slow.push({
|
||||
stage: "total",
|
||||
elapsedMs: total,
|
||||
budgetSeconds,
|
||||
suggestion: "stop blind waiting; use the stage table to optimize the largest source sync, BuildKit/cache, GitOps/Argo or dashboard segment before rerun",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
return slow;
|
||||
}
|
||||
|
||||
function publishCurrentBudgetWarnings(slowStages: readonly Record<string, unknown>[], withinBudget: boolean, budgetSeconds: number, elapsedMs: number): string[] {
|
||||
const warnings = slowStages.map((stage) => `${text(stage.stage)} exceeded configured ${text(stage.budgetSeconds)}s budget (${Math.round((finiteNumberOrNull(stage.elapsedMs) ?? 0) / 1000)}s): ${text(stage.suggestion)}`);
|
||||
if (!withinBudget) warnings.unshift(`publish-current exceeded configured ${budgetSeconds}s end-to-end budget (${Math.round(elapsedMs / 1000)}s); no further blind wait was attempted.`);
|
||||
return warnings;
|
||||
}
|
||||
|
||||
function publishCurrentBlocker(controlResult: Record<string, unknown>, dashboard: Record<string, unknown>, withinBudget: boolean): Record<string, unknown> {
|
||||
if (controlResult.ok !== true) {
|
||||
const blocker = record(controlResult.blocker);
|
||||
return {
|
||||
code: blocker.code ?? "sentinel-publish-current-control-plane-blocked",
|
||||
reason: blocker.reason ?? "control-plane publish did not converge",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
if (dashboard.ok !== true) {
|
||||
return {
|
||||
code: dashboard.skipped === true ? text(dashboard.reason) : "sentinel-publish-current-dashboard-verify-failed",
|
||||
reason: dashboard.skipped === true ? "dashboard verification did not run" : "dashboard verification did not pass",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
if (!withinBudget) {
|
||||
return {
|
||||
code: "sentinel-publish-current-over-budget",
|
||||
reason: "runtime and dashboard converged, but the one-click CI/CD path exceeded the YAML end-to-end budget",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
return { code: "sentinel-publish-current-blocked", reason: "publish-current did not satisfy all checks", valuesRedacted: true };
|
||||
}
|
||||
|
||||
function publishCurrentNext(state: SentinelCicdState): Record<string, string> {
|
||||
const node = state.spec.nodeId;
|
||||
const lane = state.spec.lane;
|
||||
const suffix = sentinelCliSuffix(state);
|
||||
return {
|
||||
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${node} --lane ${lane}${suffix} --confirm --wait`,
|
||||
controlPlaneStatus: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`,
|
||||
dashboardVerify: `bun scripts/cli.ts web-probe sentinel dashboard verify --node ${node} --lane ${lane}${suffix}`,
|
||||
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`,
|
||||
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`,
|
||||
};
|
||||
}
|
||||
|
||||
function renderSentinelManifests(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
sentinelId: string,
|
||||
@@ -931,6 +1213,11 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra
|
||||
}
|
||||
|
||||
function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
||||
const result = sentinelControlPlaneConfirmedResult(state, options);
|
||||
return rendered(result.ok === true, String(result.command), renderControlPlaneResult(result));
|
||||
}
|
||||
|
||||
function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): Record<string, unknown> {
|
||||
const startedAt = Date.now();
|
||||
const command = `web-probe sentinel control-plane ${options.action}`;
|
||||
const applyOnly = options.action === "apply";
|
||||
@@ -953,25 +1240,35 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds());
|
||||
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false);
|
||||
const observedReady = sentinelObservedReady(observed);
|
||||
const publishReady = applyOnly || record(publish).ok === true || observedReady;
|
||||
const flushReady = applyOnly || record(flush).ok === true || observedReady;
|
||||
const targetValidation = null;
|
||||
const targetValidationBlocked = false;
|
||||
const ok = state.configReady
|
||||
&& state.sourceHead.ok
|
||||
&& sourceMirrorReady
|
||||
&& (applyOnly || record(publish).ok === true)
|
||||
&& (applyOnly || record(flush).ok === true)
|
||||
&& publishReady
|
||||
&& flushReady
|
||||
&& record(runtimeSecretsApply).ok === true
|
||||
&& record(publicExposureApply).ok === true
|
||||
&& record(argoApply).ok === true
|
||||
&& observedReady;
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
const blocker = ok ? null : {
|
||||
code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready",
|
||||
code: !sourceMirrorReady
|
||||
? "sentinel-source-mirror-sync-failed"
|
||||
: !publishReady
|
||||
? "sentinel-image-gitops-publish-not-ready"
|
||||
: record(runtimeSecretsApply).ok === false
|
||||
? "sentinel-runtime-secret-sync-failed"
|
||||
: "sentinel-control-plane-not-ready",
|
||||
reason: !sourceMirrorReady
|
||||
? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
|
||||
: record(runtimeSecretsApply).ok === false
|
||||
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
|
||||
: "one or more publish, publicExposure, Argo or runtime observation checks did not pass",
|
||||
: !publishReady
|
||||
? "Tekton publish did not finish and runtime observation has not proven the selected source/image/GitOps state yet"
|
||||
: record(runtimeSecretsApply).ok === false
|
||||
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
|
||||
: "one or more publicExposure, Argo or runtime observation checks did not pass",
|
||||
};
|
||||
const result = {
|
||||
ok,
|
||||
@@ -1022,7 +1319,8 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"),
|
||||
...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
|
||||
...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds),
|
||||
...asyncGitMirrorFlushWarnings(flush),
|
||||
...asyncGitMirrorFlushWarnings(flush, cicdWaitWarningSeconds),
|
||||
...publishSatisfiedByObservedWarnings(publish, flush, observedReady),
|
||||
...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync),
|
||||
...sentinelObservedWarnings(observed),
|
||||
...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
|
||||
@@ -1034,15 +1332,17 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
next: controlPlaneNext(state, options.action),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(ok, command, renderControlPlaneResult(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane", action: string, timeoutSeconds: number): RenderedCliResult {
|
||||
function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "control-plane" | "publish", action: string, timeoutSeconds: number): RenderedCliResult {
|
||||
const args = domain === "image"
|
||||
? ["web-probe", "sentinel", "image", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
|
||||
: ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
|
||||
: domain === "control-plane"
|
||||
? ["web-probe", "sentinel", "control-plane", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]
|
||||
: ["web-probe", "sentinel", action, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
|
||||
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${domain}_${action}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${domain} ${action} for node ${state.spec.nodeId}`);
|
||||
const command = `web-probe sentinel ${domain} ${action}`;
|
||||
const command = domain === "publish" ? `web-probe sentinel ${action}` : `web-probe sentinel ${domain} ${action}`;
|
||||
const result = {
|
||||
ok: true,
|
||||
command,
|
||||
@@ -1081,11 +1381,11 @@ function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record<stri
|
||||
};
|
||||
}
|
||||
|
||||
function asyncGitMirrorFlushWarnings(flush: unknown): string[] {
|
||||
function asyncGitMirrorFlushWarnings(flush: unknown, budgetSeconds: number): string[] {
|
||||
const item = record(flush);
|
||||
if (item.mode !== "async-job") return [];
|
||||
const next = record(item.next);
|
||||
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under 120s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
|
||||
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under configured ${Math.round(budgetSeconds)}s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
|
||||
}
|
||||
|
||||
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
|
||||
@@ -1111,11 +1411,23 @@ function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
||||
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
||||
let polls = 1;
|
||||
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
|
||||
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
|
||||
polls += 1;
|
||||
}
|
||||
return observed;
|
||||
return {
|
||||
...observed,
|
||||
wait: {
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
timeoutMs,
|
||||
ready: sentinelObservedReady(observed),
|
||||
includeGitMirror,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelObservedReady(value: Record<string, unknown> | SentinelObservedStatus): boolean {
|
||||
@@ -2238,11 +2550,18 @@ function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string
|
||||
return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`];
|
||||
}
|
||||
|
||||
function sentinelElapsedWarnings(value: unknown, subject = "sentinel confirmed operation", budgetSeconds = 120): string[] {
|
||||
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
||||
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
||||
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s timing budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, investigate wait-stage latency without treating timing alone as HWLAB business blockage.`];
|
||||
function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] {
|
||||
if (!observedReady) return [];
|
||||
const warnings: string[] = [];
|
||||
const publishRecord = record(publish);
|
||||
if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) {
|
||||
warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`);
|
||||
}
|
||||
const flushRecord = record(flush);
|
||||
if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) {
|
||||
warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout.");
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number {
|
||||
@@ -2944,6 +3263,134 @@ function renderPublishResult(publish: Record<string, unknown>): string {
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function renderPublishCurrentResult(result: Record<string, unknown>): string {
|
||||
const source = record(result.source);
|
||||
const image = record(result.image);
|
||||
const controlPlane = record(result.controlPlane);
|
||||
const publish = record(controlPlane.publish);
|
||||
const publishPayload = record(publish.payload);
|
||||
const observed = record(controlPlane.observed);
|
||||
const gitops = record(observed.gitops);
|
||||
const argo = record(observed.argo);
|
||||
const runtime = record(observed.runtime);
|
||||
const runtimeDeployment = record(record(runtime.probe).deployment);
|
||||
const dashboard = record(result.dashboard);
|
||||
const dashboardPage = record(dashboard.page);
|
||||
const dashboardDom = record(dashboardPage.dom);
|
||||
const latestRunCounts = record(dashboardDom.latestRunCounts);
|
||||
const checkScope = record(dashboardDom.checkScope);
|
||||
const timings = record(result.timings);
|
||||
const budget = record(result.budget);
|
||||
const stageBudgets = record(result.stageBudgets);
|
||||
const dashboardPlan = record(result.dashboardPlan);
|
||||
const blocker = record(result.blocker);
|
||||
const next = record(result.next);
|
||||
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
||||
const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : [];
|
||||
const lines = [
|
||||
String(result.command),
|
||||
"",
|
||||
table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[
|
||||
result.node,
|
||||
result.lane,
|
||||
result.sentinelId,
|
||||
result.ok === true ? "ok" : "blocked",
|
||||
result.mode,
|
||||
budget.maxSeconds ?? "-",
|
||||
finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000),
|
||||
]]),
|
||||
"",
|
||||
table(["SOURCE", "COMMIT", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
|
||||
`${source.repository ?? "-"}@${source.branch ?? "-"}`,
|
||||
short(source.commit),
|
||||
image.ref ?? "-",
|
||||
short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest),
|
||||
result.pipelineRun ?? publish.jobName ?? "-",
|
||||
]]),
|
||||
"",
|
||||
table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "DASHBOARD"], [[
|
||||
short(gitops.revision),
|
||||
short(argo.revision),
|
||||
`${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`,
|
||||
short(runtimeDeployment.image),
|
||||
`${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`,
|
||||
dashboard.ok === true ? "pass" : dashboard.skipped === true ? `skipped:${text(dashboard.reason)}` : Object.keys(dashboard).length === 0 ? "planned" : "blocked",
|
||||
]]),
|
||||
"",
|
||||
table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "DASHBOARD_MS", "TOTAL_MS"], [[
|
||||
timings.sourceSyncMs ?? "-",
|
||||
timings.sourceFetchMs ?? "-",
|
||||
timings.monitorWebVerifyMs ?? "-",
|
||||
timings.imageBuildMs ?? "-",
|
||||
timings.gitopsMs ?? "-",
|
||||
timings.argoRuntimeMs ?? "-",
|
||||
timings.dashboardVerifyMs ?? "-",
|
||||
timings.totalMs ?? "-",
|
||||
]]),
|
||||
"",
|
||||
table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "DASHBOARD"], [[
|
||||
"YAML publishCurrent",
|
||||
stageBudgets.sourceSyncSeconds ?? "-",
|
||||
stageBudgets.sourceFetchSeconds ?? "-",
|
||||
stageBudgets.monitorWebVerifySeconds ?? "-",
|
||||
stageBudgets.imageBuildSeconds ?? "-",
|
||||
stageBudgets.gitopsSeconds ?? "-",
|
||||
stageBudgets.argoRuntimeSeconds ?? "-",
|
||||
stageBudgets.dashboardVerifySeconds ?? "-",
|
||||
]]),
|
||||
];
|
||||
if (Object.keys(publish).length > 0) {
|
||||
const payloadImageBuild = record(publishPayload.imageBuild);
|
||||
const payloadEnvReuse = record(publishPayload.envReuse);
|
||||
lines.push(
|
||||
"",
|
||||
table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[
|
||||
payloadEnvReuse.dependencyReuse ?? "-",
|
||||
payloadEnvReuse.nodeDepsPresent ?? "-",
|
||||
payloadImageBuild.packageMode ?? "-",
|
||||
payloadImageBuild.networkMode ?? "-",
|
||||
payloadImageBuild.layerCache ?? "-",
|
||||
payloadImageBuild.cacheHitLines ?? "-",
|
||||
]]),
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
"",
|
||||
Object.keys(dashboard).length === 0
|
||||
? "DASHBOARD_VERIFY\n-"
|
||||
: table(["URL", "HTTP", "LATEST_RUN", "CHECK_SCOPE", "CHECK_MATCH", "REQ_FAIL", "CONSOLE_ERR"], [[
|
||||
dashboard.publicUrl ?? "-",
|
||||
dashboardPage.httpStatus ?? "-",
|
||||
latestRunCounts.runId ?? "-",
|
||||
checkScope.scope ?? "-",
|
||||
checkScope.matchesRunDetail ?? "-",
|
||||
dashboardPage.requestFailureCount ?? "-",
|
||||
dashboardPage.consoleErrorCount ?? "-",
|
||||
]]),
|
||||
"",
|
||||
slowStages.length === 0 ? "SLOW_STAGES\n-" : [
|
||||
"SLOW_STAGES",
|
||||
table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])),
|
||||
].join("\n"),
|
||||
"",
|
||||
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
||||
"",
|
||||
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
||||
"",
|
||||
"NEXT",
|
||||
` publish-current: ${next.publishCurrent ?? "-"}`,
|
||||
` status: ${next.controlPlaneStatus ?? "-"}`,
|
||||
` dashboard: ${next.dashboardVerify ?? "-"}`,
|
||||
` git-mirror: ${next.gitMirrorStatus ?? "-"}`,
|
||||
` flush: ${next.gitMirrorFlush ?? "-"}`,
|
||||
"",
|
||||
"DISCLOSURE",
|
||||
` end-to-end and stage budgets are read from ${Object.keys(dashboardPlan).length > 0 ? "publishCurrent YAML" : "YAML-required publishCurrent fields"}.`,
|
||||
" image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.",
|
||||
);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function renderImageResult(result: Record<string, unknown>): string {
|
||||
const source = record(result.source);
|
||||
const sourceMirror = record(result.sourceMirror);
|
||||
@@ -3197,6 +3644,16 @@ export function numberAt(value: unknown, path: string): number {
|
||||
return found;
|
||||
}
|
||||
|
||||
function booleanAt(value: unknown, path: string): boolean {
|
||||
const found = valueAtPath(value, path);
|
||||
if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function finiteNumberOrNull(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
export function arrayAt(value: unknown, path: string): unknown[] {
|
||||
const found = valueAtPath(value, path);
|
||||
if (!Array.isArray(found)) throw new Error(`${path} must be an array`);
|
||||
|
||||
Reference in New Issue
Block a user