Files
pikasTech-unidesk/scripts/src/hwlab-node-web-sentinel-p5.ts
T
2026-07-02 07:33:19 +00:00

1688 lines
92 KiB
TypeScript

// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
// Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands.
import { existsSync, readFileSync } from "node:fs";
import type { CommandResult } from "./command";
import { runCommand } from "./command";
import { repoRoot } from "./config";
import { startJob } from "./jobs";
import type { RenderedCliResult } from "./output";
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
import {
clipTail,
compactCommand,
compactSentinelServiceBodyJson,
mergeWarnings,
numberAt,
numberAtNullable,
parseJsonObject,
pickFields,
record,
rendered,
renderAsyncJobResult,
safeJobSegment,
sentinelCliSuffix,
shellQuote,
short,
stringAt,
stringAtNullable,
table,
targetValidationElapsedWarnings,
text,
withWarnings,
} from "./hwlab-node-web-sentinel-cicd";
import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55;
export function runSentinelMaintenance(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "maintenance" }>): RenderedCliResult {
const command = `web-probe sentinel maintenance ${options.action}`;
const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
if (options.action === "status") {
const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds);
const result = {
ok: serviceHealth.ok && maintenance.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
serviceHealth,
maintenance,
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderMaintenanceResult(result));
}
if (!options.confirm) {
const result = {
ok: serviceHealth.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "dry-run",
serviceHealth,
mutation: false,
planned: {
action: options.action,
releaseId: options.releaseId,
reason: options.reason,
quickVerify: options.action === "stop" && options.quickVerify,
},
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderMaintenanceResult(result));
}
if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify);
if (!serviceHealth.ok) {
const result = {
ok: false,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "confirm-wait",
mutation: false,
serviceHealth,
blocker: serviceUnavailableBlocker(state),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(false, command, renderMaintenanceResult(result));
}
const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true };
const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds);
const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok
? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds)
: null;
const result = {
ok: mutation.ok && (quickVerify === null || quickVerify.ok === true),
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "confirm-wait",
mutation: true,
serviceHealth,
maintenance: mutation,
quickVerify,
blocker: mutation.ok ? null : serviceUnavailableBlocker(state),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderMaintenanceResult(result));
}
export function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
const command = "web-probe sentinel validate";
const startedAt = Date.now();
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
let quickVerify: Record<string, unknown> | null = null;
if (options.quickVerify) {
if (!options.confirm) {
const result = {
ok: initialHealth.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "dry-run",
serviceHealth: initialHealth,
planned: { quickVerify: true, waitRequired: true },
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderValidateResult(result));
}
if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true);
if (!initialHealth.ok) {
const result = {
ok: false,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "confirm-wait",
serviceHealth: initialHealth,
blocker: serviceUnavailableBlocker(state),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(false, command, renderValidateResult(result));
}
quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds);
}
const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20);
const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds);
const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds);
const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds);
const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health");
const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds);
const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds);
const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds);
const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health;
const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics;
const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report;
const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds);
const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds);
if (quickVerify !== null) {
quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds"))));
}
const publicFallbackWarnings = [
...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []),
...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []),
...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []),
];
const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health");
const ok = effectiveHealth.ok
&& record(effectiveHealth.bodyJson).ok === true
&& effectiveMetricsOk
&& effectiveReport.ok
&& publicExposure.ok === true
&& publicDashboard.ok === true
&& (quickVerify === null || quickVerify.ok === true);
const result = {
ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: options.quickVerify ? "confirm-wait" : "status",
serviceHealth: effectiveHealth,
metrics: effectiveMetrics,
report: effectiveReport,
internalServiceHealth: health,
internalMetrics: metrics,
internalReport: report,
publicServiceHealth: publicHealth,
publicMetrics,
publicReport,
publicExposure,
publicDashboard,
quickVerify,
warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []),
blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(ok, command, renderValidateResult(result));
}
export function probeSentinelRuntimeHealthEndpoint(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const endpoint = stringAt(state.runtime, "healthPath");
const serviceProbeTimeoutSeconds = Math.min(timeoutSeconds, 20);
const health = callSentinelService(state, "GET", endpoint, null, serviceProbeTimeoutSeconds);
const publicHealth = health.ok ? null : probePublicSentinelService(state, endpoint, serviceProbeTimeoutSeconds);
const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health;
const bodyJson = record(effectiveHealth.bodyJson);
const ok = effectiveHealth.ok === true && bodyJson.ok === true;
return {
ok,
endpoint,
health: effectiveHealth,
internalHealth: health,
publicHealth,
httpStatus: effectiveHealth.httpStatus ?? null,
publicUrl: effectiveHealth.publicUrl ?? null,
internalUrl: effectiveHealth.internalUrl ?? null,
degradedReason: ok ? null : "sentinel-runtime-health-endpoint-failed",
valuesRedacted: true,
};
}
export function runSentinelReport(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "report" }>): RenderedCliResult {
const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`;
const query = new URLSearchParams({ view: options.view });
if (options.runId !== null) query.set("run", options.runId);
if (options.traceId !== null) query.set("traceId", options.traceId);
if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq));
const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds);
const body = record(report.bodyJson);
const rawPayload = Object.keys(body).length > 0 ? body : report;
if (options.full) return rendered(report.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2));
if (options.raw) {
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS));
return rendered(report.ok && body.ok !== false, command, JSON.stringify(compactSentinelReportRawPayload(state, body, report, artifactSummary), null, 2));
}
if (options.view === "summary") {
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS));
const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary);
return rendered(report.ok && body.ok !== false, command, renderSentinelReportSummary(payload, state));
}
if (options.view === "findings") {
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS));
const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary);
return rendered(report.ok && body.ok !== false, command, renderSentinelReportFindings(payload));
}
const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true });
return rendered(report.ok && body.ok !== false, command, renderedText);
}
function compactSentinelReportRawPayload(
state: SentinelCicdState,
body: Record<string, unknown>,
report: Record<string, unknown>,
artifactSummary: Record<string, unknown> | null,
): Record<string, unknown> {
const run = record(body.run);
const artifact = record(artifactSummary);
const findings = Array.isArray(body.findings) ? body.findings.map(record) : [];
const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : [];
const visibleFindings = mergeSentinelReportFindings(findings, artifactFindings);
const offlineReclassify = offlineQuickVerifyReclassify(state, run, visibleFindings);
const storedFindingCount = numberAtNullable(run, "finding_count") ?? numberAtNullable(run, "findingCount") ?? findings.length;
const artifactFindingCount = numberAtNullable(artifact, "findingCount") ?? artifactFindings.length;
const visibleFindingCount = Math.max(storedFindingCount, artifactFindingCount, visibleFindings.length);
const rootCauseSignalFindings = artifactFindings
.filter((item) => Object.keys(record(item.rootCauseSignals)).length > 0)
.slice(0, 8)
.map((item) => ({
id: stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"),
severity: stringAtNullable(item, "severity") ?? stringAtNullable(item, "level"),
summary: reportText(item.summary ?? item.message, 220),
rootCauseSignals: compactRootCauseSignals(item.rootCauseSignals),
valuesRedacted: true,
}));
return {
ok: body.ok !== false && report.ok !== false,
view: body.view ?? null,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
run: {
id: run.id ?? null,
scenarioId: run.scenario_id ?? run.scenarioId ?? null,
status: run.status ?? null,
observerId: run.observer_id ?? run.observerId ?? null,
stateDir: run.state_dir ?? run.stateDir ?? null,
reportJsonSha256: run.report_json_sha256 ?? run.reportJsonSha256 ?? artifact.reportJsonSha256 ?? null,
findingCount: visibleFindingCount,
storedFindingCount,
artifactFindingCount,
artifactCount: run.artifact_count ?? run.artifactCount ?? artifact.artifactCount ?? null,
durationMinutes: run.durationMinutes ?? run.runDurationMinutes ?? record(run.timing).durationMinutes ?? null,
runDurationMinutes: run.runDurationMinutes ?? run.durationMinutes ?? record(run.timing).durationMinutes ?? null,
durationSource: run.durationSource ?? record(run.timing).durationSource ?? null,
scenarioCadenceMinutes: run.scenarioCadenceMinutes ?? record(run.timing).scenarioCadenceMinutes ?? null,
scenarioMaxRunMinutes: run.scenarioMaxRunMinutes ?? record(run.timing).scenarioMaxRunMinutes ?? null,
schedulerIntervalMinutes: run.schedulerIntervalMinutes ?? record(run.timing).schedulerIntervalMinutes ?? null,
timing: pickFields(record(run.timing), ["startedAt", "finishedAt", "durationMs", "durationMinutes", "durationSource", "scenarioCadence", "scenarioCadenceMinutes", "scenarioMaxRunMinutes", "schedulerIntervalMinutes", "sourceOfTruth", "valuesRedacted"]),
updatedAt: run.updated_at ?? run.updatedAt ?? null,
valuesRedacted: true,
},
summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]),
findings: visibleFindings.slice(0, 12).map(compactSentinelReportFinding),
offlineReclassify,
artifactSummary: Object.keys(artifact).length === 0 ? null : {
ok: artifact.ok === true,
reason: artifact.reason ?? null,
reportReadWaitMs: artifact.reportReadWaitMs ?? null,
reportParseError: artifact.reportParseError ?? null,
reportOk: artifact.reportOk === true ? true : artifact.reportOk === false ? false : null,
reportJsonPath: artifact.reportJsonPath ?? null,
reportJsonSha256: artifact.reportJsonSha256 ?? null,
reportMdSha256: artifact.reportMdSha256 ?? null,
findingCount: artifactFindingCount,
findings: artifactFindings.slice(0, 12).map(compactSentinelReportFinding),
screenshot: record(artifact.screenshot),
counts: record(artifact.counts),
analysisWindow: compactSentinelAnalysisWindow(artifact.analysisWindow),
pagePerformanceSlowApi: Array.isArray(artifact.pagePerformanceSlowApi) ? artifact.pagePerformanceSlowApi.slice(0, 8).map(record) : [],
rootCauseSignalFindings,
valuesRedacted: true,
},
next: {
text: "Default report is bounded text; use --full for the full indexed service payload.",
report: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")}`,
full: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")} --full`,
},
valuesRedacted: true,
};
}
function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record<string, unknown>, findings: readonly Record<string, unknown>[]): Record<string, unknown> | null {
const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("quick-verify-no-business-turn"));
if (!hasLegacyQuickVerifyControl) return null;
const catalog = sentinelReportCheckCatalogById(state);
const result = reclassifyQuickVerifyControlFindings(state, {
runId: stringAtNullable(run, "id"),
scenarioId: stringAtNullable(run, "scenario_id") ?? stringAtNullable(run, "scenarioId"),
observerId: stringAtNullable(run, "observer_id") ?? stringAtNullable(run, "observerId"),
failure: stringAtNullable(record(run.summary), "failure"),
timeoutSeconds: SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS,
});
const resultFindings = Array.isArray(result.findings) ? result.findings.map(record) : [];
return {
...pickFields(result, ["ok", "reason", "runId", "scenarioId", "observerId", "promptIndex", "findingCount", "turnSummary", "traceFrame", "valuesRedacted"]),
source: "offline-existing-observe-artifact",
note: "This is a local CLI reclassification of existing turn-summary/trace-frame artifacts; it does not mutate the runner index.",
findings: resultFindings.slice(0, 8).map((item) => compactSentinelReportFinding(enrichSentinelReportFindingWithCatalog(item, catalog))),
valuesRedacted: true,
};
}
function sentinelReportCheckCatalogById(state: SentinelCicdState): Map<string, Record<string, unknown>> {
try {
const reportViews = record(readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.reportViews));
const catalogRef = stringAtNullable(reportViews, "checkCatalogRef");
if (catalogRef === null) return new Map();
const catalog = record(readWebProbeSentinelConfigRefTarget(state.spec, catalogRef));
const items = Array.isArray(catalog.items) ? catalog.items.map(record) : Array.isArray(catalog.checks) ? catalog.checks.map(record) : [];
return new Map(items.map((item) => [stringAtNullable(item, "id") ?? "", item]).filter((item): item is [string, Record<string, unknown>] => item[0].length > 0));
} catch {
return new Map();
}
}
function enrichSentinelReportFindingWithCatalog(item: Record<string, unknown>, catalog: ReadonlyMap<string, Record<string, unknown>>): Record<string, unknown> {
const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code");
const check = id === null ? null : catalog.get(id) ?? null;
if (check === null) return item;
return {
...item,
check,
checkCode: stringAtNullable(check, "code"),
checkTitleZh: stringAtNullable(check, "titleZh"),
valuesRedacted: true,
};
}
function mergeSentinelReportFindings(primary: readonly Record<string, unknown>[], artifact: readonly Record<string, unknown>[]): Record<string, unknown>[] {
const merged: Record<string, unknown>[] = [];
const seen = new Set<string>();
const aliases = sentinelReportFindingIdentityAliases(primary);
for (const item of [...primary, ...artifact]) {
const id = sentinelReportCanonicalFindingIdentity(item, aliases);
const key = id ?? `${JSON.stringify(item).slice(0, 160)}:${stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? ""}`;
if (seen.has(key)) continue;
seen.add(key);
merged.push(item);
}
return merged;
}
function sentinelReportFindingIdentityAliases(items: readonly Record<string, unknown>[]): Map<string, string> {
const aliases = new Map<string, string>();
for (const item of items) {
const canonical = sentinelReportFindingCanonicalCandidate(item) ?? sentinelReportFindingIdentity(item);
if (canonical === null) continue;
for (const candidate of sentinelReportFindingIdentityCandidates(item)) aliases.set(candidate, canonical);
}
return aliases;
}
function sentinelReportCanonicalFindingIdentity(item: Record<string, unknown>, aliases: ReadonlyMap<string, string>): string | null {
for (const candidate of sentinelReportFindingIdentityCandidates(item)) {
const canonical = aliases.get(candidate);
if (canonical !== undefined) return canonical;
}
return sentinelReportFindingCanonicalCandidate(item) ?? sentinelReportFindingIdentity(item);
}
function sentinelReportFindingCanonicalCandidate(item: Record<string, unknown>): string | null {
const check = record(item.check);
return stringAtNullable(item, "checkCode")
?? stringAtNullable(check, "code")
?? stringAtNullable(item, "checkId")
?? stringAtNullable(check, "id");
}
function sentinelReportFindingIdentityCandidates(item: Record<string, unknown>): readonly string[] {
const check = record(item.check);
const candidates = [
stringAtNullable(item, "finding_id"),
stringAtNullable(item, "findingId"),
stringAtNullable(item, "id"),
stringAtNullable(item, "kind"),
stringAtNullable(item, "code"),
stringAtNullable(item, "checkId"),
stringAtNullable(item, "checkCode"),
stringAtNullable(check, "id"),
stringAtNullable(check, "code"),
];
return [...new Set(candidates.filter((item): item is string => item !== null))];
}
function sentinelReportFindingIdentity(item: Record<string, unknown>): string | null {
return stringAtNullable(item, "finding_id")
?? stringAtNullable(item, "findingId")
?? stringAtNullable(item, "id")
?? stringAtNullable(item, "kind")
?? stringAtNullable(item, "code");
}
function compactSentinelReportFinding(value: Record<string, unknown>): Record<string, unknown> {
const result: Record<string, unknown> = {
id: stringAtNullable(value, "finding_id") ?? stringAtNullable(value, "findingId") ?? stringAtNullable(value, "id") ?? stringAtNullable(value, "kind") ?? stringAtNullable(value, "code"),
severity: stringAtNullable(value, "severity") ?? stringAtNullable(value, "level"),
count: value.count ?? null,
summary: reportText(value.summary ?? value.message, 220),
valuesRedacted: true,
};
const rootCause = stringAtNullable(value, "rootCause");
if (rootCause !== null) result.rootCause = rootCause;
const rootCauseStatus = stringAtNullable(value, "rootCauseStatus");
if (rootCauseStatus !== null) result.rootCauseStatus = rootCauseStatus;
const rootCauseConfidence = stringAtNullable(value, "rootCauseConfidence");
if (rootCauseConfidence !== null) result.rootCauseConfidence = rootCauseConfidence;
const nextAction = reportText(value.nextAction, 240);
if (nextAction !== null) result.nextAction = nextAction;
const evidenceSummary = reportText(value.evidenceSummary, 240);
if (evidenceSummary !== null) result.evidenceSummary = evidenceSummary;
const timingSourceOfTruth = stringAtNullable(value, "timingSourceOfTruth");
if (timingSourceOfTruth !== null) result.timingSourceOfTruth = timingSourceOfTruth;
const timingStatus = stringAtNullable(value, "timingStatus");
if (timingStatus !== null) result.timingStatus = timingStatus;
if (value.timingAlert === true) result.timingAlert = true;
const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals);
if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals;
const check = record(value.check);
const checkCodeFromValue = stringAtNullable(value, "checkCode");
const checkTitleFromValue = stringAtNullable(value, "checkTitleZh");
const checkCode = stringAtNullable(check, "code");
const checkTitle = stringAtNullable(check, "titleZh");
if (checkCodeFromValue !== null) result.checkCode = checkCodeFromValue;
if (checkTitleFromValue !== null) result.checkTitleZh = checkTitleFromValue;
if (checkCode !== null || checkTitle !== null) {
result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]);
}
return result;
}
function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> | null {
const run = record(body.run);
const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir");
if (stateDir === null || !isSafeSentinelReportStateDir(stateDir)) return null;
const waitMs = Math.max(0, Math.min(50_000, (Math.max(5, timeoutSeconds) * 1000) - 5000));
const script = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
`wait_ms=${waitMs}`,
"node - \"$state_dir\" \"$wait_ms\" <<'NODE'",
"const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');",
"const stateDir=process.argv[2]; const waitMs=Math.max(0, Math.min(60000, Number(process.argv[3]||0)||0)); const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');",
"const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}};",
"const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;",
"const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);",
"const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};",
"const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};",
"let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;",
"while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }",
"let artifactCount=0; let screenshot=null;",
"function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}",
"walk(stateDir);",
"const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});",
"const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});",
"console.log(JSON.stringify({ok:!!report,reason:report?null:(jsonBuf?'report-json-parse-failed':'report-json-missing'),reportReadWaitMs:waitMs,reportParseError:clip(reportParseError,220),reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function isSafeSentinelReportStateDir(value: string): boolean {
return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..") && !value.startsWith("/");
}
function renderSentinelReportSummary(payload: Record<string, unknown>, state: SentinelCicdState): string {
const run = record(payload.run);
const summary = record(payload.summary);
const artifactSummary = record(payload.artifactSummary);
const offlineReclassify = record(payload.offlineReclassify);
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
const findingCount = run.findingCount ?? findings.length;
const analyzerArtifactCount = numberAtNullable(artifactSummary, "artifactCount") ?? numberAtNullable(record(artifactSummary.counts), "artifacts");
const runArtifactCount = numberAtNullable(run, "artifactCount");
const artifactCount = runArtifactCount === null || runArtifactCount === 0 && analyzerArtifactCount !== null
? analyzerArtifactCount ?? "-"
: runArtifactCount;
const publicOrigin = stringAtNullable(state.publicExposure, "publicBaseUrl") ?? "-";
const findingRows = findings.length === 0
? "-"
: table(["ID", "SEVERITY", "COUNT", "SUMMARY"], findings.slice(0, 12).map((item) => [
stringAtNullable(item, "id") ?? "-",
stringAtNullable(item, "severity") ?? "-",
item.count ?? "-",
reportText(item.summary, 120) ?? "-",
]));
return [
"Web Probe Sentinel Quick Verify",
"=======================================================",
`run=${run.id ?? "-"} scenario=${run.scenarioId ?? "-"} observer=${run.observerId ?? "-"}`,
`status=${run.status ?? summary.status ?? "-"} reason=${summary.reason ?? "-"} failure=${summary.failure ?? "-"}`,
`report=${reportSha ?? "-"} artifacts=${artifactCount} findings=${findingCount}`,
`publicOrigin=${publicOrigin}`,
"",
"Findings",
findingRows,
offlineFindings.length === 0 ? "" : "",
offlineFindings.length === 0 ? "" : "Offline Reclassify",
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
].join("\n");
}
function renderSentinelReportFindings(payload: Record<string, unknown>): string {
const run = record(payload.run);
const artifactSummary = record(payload.artifactSummary);
const offlineReclassify = record(payload.offlineReclassify);
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
return [
"Web Probe Sentinel Findings",
"=======================================================",
`run=${run.id ?? "-"} report=${reportSha ?? "-"} findings=${run.findingCount ?? findings.length}`,
findings.length === 0 ? "-" : findings.map(formatSentinelReportFindingLine).join("\n"),
offlineFindings.length === 0 ? "" : "",
offlineFindings.length === 0 ? "" : "Offline Reclassify",
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
].join("\n");
}
function formatSentinelReportFindingLine(item: Record<string, unknown>): string {
const check = record(item.check);
const code = stringAtNullable(item, "checkCode") ?? stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-";
const title = stringAtNullable(item, "checkTitleZh") ?? stringAtNullable(check, "titleZh") ?? "";
const summary = reportText(item.summary, 180) ?? "";
const rootCause = reportText(item.rootCause, 180);
const evidence = reportText(item.evidenceSummary, 180);
const nextAction = reportText(item.nextAction, 180);
return [
`${item.severity ?? "-"} ${code} ${title} count=${item.count ?? "-"} ${summary}`.trim(),
rootCause === null ? null : `rootCause=${rootCause}`,
evidence === null ? null : `evidence=${evidence}`,
nextAction === null ? null : `next=${nextAction}`,
].filter((part): part is string => part !== null && part.length > 0).join(" | ");
}
function compactRootCauseSignals(value: unknown): Record<string, unknown> | null {
const item = record(value);
const keys = [
"sessionListReadCount",
"traceEventsReadCount",
"webPerformanceBeaconFailureCount",
"eventSourceFailureCount",
"requestFailedCount",
"httpErrorCount",
"consoleAlertCount",
"requestfailedTop",
"httpStatusTop",
];
const out: Record<string, unknown> = {};
for (const key of keys) {
const raw = item[key];
if (raw === null || raw === undefined) continue;
out[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
}
return Object.keys(out).length === 0 ? null : { ...out, valuesRedacted: true };
}
function compactSentinelAnalysisWindow(value: unknown): Record<string, unknown> | null {
const item = record(value);
if (Object.keys(item).length === 0) return null;
return pickFields(item, ["name", "windowMs", "samples", "control", "network", "console", "valuesRedacted"]);
}
function reportText(value: unknown, maxChars: number): string | null {
if (value === undefined || value === null || value === "") return null;
const raw = text(value);
return raw.length <= maxChars ? raw : `${raw.slice(0, Math.max(0, maxChars - 1))}…`;
}
export function runSentinelDashboard(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): RenderedCliResult {
const command = `web-probe sentinel dashboard ${options.action}${options.runId === null ? "" : ` --run ${options.runId}`}`;
const result = probeSentinelDashboardBrowser(state, options);
return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result));
}
export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const dashboardUrl = sentinelDashboardUrl(publicBaseUrl, options.runId);
const [widthRaw, heightRaw] = options.viewport.split("x");
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
const script = [
"set -eu",
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`,
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
`export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_TRIGGER=${shellQuote(options.action === "trigger" ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`,
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`,
`export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`,
`export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`,
"export PLAYWRIGHT_BROWSERS_PATH=0",
"if command -v chromium >/dev/null 2>&1; then",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)",
"elif command -v chromium-browser >/dev/null 2>&1; then",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)",
"elif command -v google-chrome >/dev/null 2>&1; then",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)",
"else",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=",
"fi",
"cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'",
sentinelDashboardBrowserModule(),
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
"node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
].join("\n");
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
const job = runWebProbeRemoteArtifactJob({
route,
localDir: options.localDir,
waitTimeoutMs: options.waitTimeoutMs,
commandTimeoutMs: options.commandTimeoutSeconds * 1000,
inactivityTimeoutMs: 30000,
runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`,
stdoutTailBytes: 32768,
}, script);
const result = job.result;
const transport = record(job.transport);
const remote = record(transport.remote);
const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : "");
const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : [];
const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null;
const browserOk = page?.ok === true;
const screenshotOk = options.action !== "screenshot" || screenshot !== null && screenshot.verified === true;
const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk;
return {
ok,
status: ok ? "pass" : "blocked",
command: `web-probe sentinel dashboard ${options.action}`,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
runId: options.runId,
publicUrl: `${publicBaseUrl}/`,
route,
viewport: options.viewport,
page,
screenshot,
artifacts,
artifactCount: artifacts.length,
remote: {
exitCode: remote.exitCode ?? null,
remoteDir: remote.remoteDir ?? null,
stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "",
stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "",
},
transport: {
ok: transport.ok ?? null,
runId: transport.runId ?? null,
artifactCount: transport.artifactCount ?? null,
expectedArtifactCount: transport.expectedArtifactCount ?? null,
downloadFailure: transport.downloadFailure ?? null,
pollFailure: transport.pollFailure ?? null,
},
result: compactCommand(result),
degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk),
valuesRedacted: true,
};
}
function sentinelDashboardUrl(publicBaseUrl: string, runId: string | null): string {
const url = new URL(`${publicBaseUrl.replace(/\/$/u, "")}/`);
if (runId !== null && runId.length > 0) {
url.searchParams.set("run", runId);
url.searchParams.set("focus", "memory");
}
return url.toString();
}
function sentinelDashboardBrowserModule(): string {
return String.raw`import { pathToFileURL } from "node:url";
const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || "";
const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright";
const { chromium } = await import(playwrightModuleSpecifier);
const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL;
const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || "";
const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1";
const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1";
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || "";
const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/";
const requestedRunId = process.env.UNIDESK_SENTINEL_DASHBOARD_RUN_ID || "";
if (!url) throw new Error("missing dashboard URL");
const consoleMessages = [];
const pageErrors = [];
const requestFailures = [];
const browser = await chromium.launch({
headless: true,
args: ["--disable-gpu", "--no-sandbox"],
...(executablePath ? { executablePath } : {}),
});
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
const page = await context.newPage();
page.on("console", (message) => {
if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
});
page.on("pageerror", (error) => {
if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) });
});
page.on("requestfailed", (request) => {
if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null });
});
let httpStatus = null;
let navigationError = null;
let navigationAttempts = 0;
const maxNavigationAttempts = 2;
const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts));
for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) {
navigationAttempts = attempt;
navigationError = null;
try {
const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" });
httpStatus = response?.status() ?? null;
await page.waitForFunction(() => {
const root = document.querySelector("#monitor-web-root");
if (!root) return false;
const ready = root.getAttribute("data-monitor-ready") === "true";
const error = document.querySelector("#monitor-web-error");
return ready || Boolean(error);
}, null, { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {});
await page.waitForTimeout(300);
const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false);
if (appReady || attempt === maxNavigationAttempts) break;
} catch (error) {
navigationError = String(error?.message || error).slice(0, 500);
if (attempt === maxNavigationAttempts) break;
}
await page.waitForTimeout(750 * attempt);
}
let requestedRunSelection = { requestedRunId, ok: requestedRunId.length === 0, reason: requestedRunId.length === 0 ? "not-requested" : "not-attempted" };
if (requestedRunId) {
requestedRunSelection = await page.evaluate(async (runId) => {
const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms));
const rowForRun = () => document.querySelector('[data-run-id="' + CSS.escape(runId) + '"]');
const row = rowForRun();
if (row instanceof HTMLElement) row.click();
for (let index = 0; index < 80; index += 1) {
const selected = rowForRun();
if (selected instanceof HTMLElement) return { requestedRunId: runId, ok: true, reason: "run-row-present" };
await wait(150);
}
return { requestedRunId: runId, ok: false, reason: "run-row-missing" };
}, requestedRunId).catch((error) => ({ requestedRunId, ok: false, reason: "selection-error", error: String(error?.message || error).slice(0, 300) }));
await page.waitForTimeout(150);
}
let manualTrigger = { requested: triggerManual, ok: !triggerManual, status: triggerManual ? "not-attempted" : "not-requested", jobName: null, statusText: "" };
if (triggerManual) {
for (let attempt = 1; attempt <= 2; attempt += 1) {
const buttonReady = await page.waitForSelector("[data-monitor-manual-trigger='true']", { timeout: Math.min(20000, timeout) }).then(() => true).catch(() => false);
if (buttonReady) break;
if (attempt === 1) {
await page.reload({ timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }).catch(() => null);
await page.waitForFunction(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true", null, { timeout: Math.min(12000, perAttemptTimeout) }).catch(() => {});
}
}
manualTrigger = await page.evaluate(async () => {
const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms));
const root = document.querySelector("[data-monitor-shell='true']");
const button = document.querySelector("[data-monitor-manual-trigger='true']");
if (!(button instanceof HTMLButtonElement)) return { requested: true, ok: false, status: "button-missing", jobName: null, statusText: "" };
if (button.disabled) return { requested: true, ok: false, status: "button-disabled", jobName: button.getAttribute("data-trigger-job") || null, statusText: "" };
button.click();
for (let index = 0; index < 160; index += 1) {
const state = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "";
const jobName = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null;
const statusText = String(document.querySelector("[data-monitor-manual-trigger-status='true']")?.textContent || "").replace(/\s+/g, " ").trim();
if (state === "triggered" || state === "already-active" || state === "failed") {
return { requested: true, ok: state === "triggered" || state === "already-active", status: state, jobName, statusText };
}
await wait(250);
}
const finalState = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "timeout";
const finalJob = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null;
return { requested: true, ok: false, status: finalState || "timeout", jobName: finalJob, statusText: "timeout" };
}).catch((error) => ({ requested: true, ok: false, status: "trigger-error", jobName: null, statusText: String(error?.message || error).slice(0, 300) }));
await page.waitForTimeout(300);
}
const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }) => {
const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0;
const objectOrNull = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null;
const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim();
const root = document.querySelector("#monitor-web-root");
const shell = document.querySelector("[data-monitor-shell='true']");
const error = document.querySelector("#monitor-web-error");
const trend = document.querySelector("[data-monitor-trend-curve]");
const trendErrorCurve = document.querySelector("[data-monitor-trend-error-curve='true']");
const trendWarningCurve = document.querySelector("[data-monitor-trend-warning-curve='true']");
const memoryChart = document.querySelector("[data-run-memory-chart='true']");
const manualTriggerButton = document.querySelector("[data-monitor-manual-trigger='true']");
const manualTriggerStatus = document.querySelector("[data-monitor-manual-trigger-status='true']");
const chartTiming = {
hasDurationCurve: Boolean(trend),
hasErrorCurve: Boolean(trendErrorCurve),
hasWarningCurve: Boolean(trendWarningCurve),
ok: false,
};
chartTiming.ok = chartTiming.hasDurationCurve === true
&& chartTiming.hasErrorCurve === true
&& chartTiming.hasWarningCurve === true;
const dataset = root ? {
node: root.getAttribute("data-node"),
lane: root.getAttribute("data-lane"),
sentinelId: root.getAttribute("data-sentinel-id"),
basePath: root.getAttribute("data-base-path"),
contractVersion: root.getAttribute("data-contract-version"),
ready: root.getAttribute("data-monitor-ready"),
} : {};
const apiUrl = (path) => {
const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || "";
const prefix = basePath.replace(/\/+$/u, "");
return (prefix + (path.startsWith("/") ? path : "/" + path)) || path;
};
const fetchJson = async (path) => {
try {
const response = await fetch(apiUrl(path), { cache: "no-store" });
return {
ok: response.ok,
httpStatus: response.status,
body: await response.json().catch(() => null),
};
} catch (error) {
return {
ok: false,
httpStatus: null,
body: null,
error: String(error?.message || error).slice(0, 300),
};
}
};
const [overviewResult, runsResult] = await Promise.all([
fetchJson("/api/overview"),
fetchJson("/api/runs?limit=30&sort=updated"),
]);
const overviewPayload = objectOrNull(overviewResult.body) || {};
const runsPayload = objectOrNull(runsResult.body) || {};
const runs = Array.isArray(runsPayload.runs) ? runsPayload.runs : Array.isArray(runsPayload.items) ? runsPayload.items : [];
const latestRun = runs[0] || null;
const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts)
? latestRun.severityCounts
: {};
const latestRunCounts = {
runId: latestRun?.id || latestRun?.runId || null,
typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count),
durationMinutes: numberValue(latestRun?.runDurationMinutes ?? latestRun?.durationMinutes ?? latestRun?.timing?.durationMinutes),
severityKeys: Object.keys(latestCounts).sort(),
};
const targetRunId = requestedRunId || latestRunCounts.runId;
const targetRun = targetRunId ? runs.find((run) => (run?.id || run?.runId) === targetRunId) || latestRun : latestRun;
const targetDetailResult = targetRunId === null ? { ok: false, httpStatus: null, body: null } : await fetchJson("/api/runs/" + encodeURIComponent(targetRunId));
const targetDetailPayload = objectOrNull(targetDetailResult.body) || {};
const targetMemory = objectOrNull(targetDetailPayload.memory) || {};
const targetPageSeries = Array.isArray(targetMemory.pageSeries) ? targetMemory.pageSeries : [];
const memorySummary = {
present: Boolean(memoryChart),
runId: memoryChart?.getAttribute("data-memory-run-id") || null,
targetRunId,
targetScenarioId: targetRun?.scenarioId || targetRun?.scenario_id || null,
matchesTargetRun: memoryChart?.getAttribute("data-memory-run-id") === targetRunId,
pageCount: numberValue(memoryChart?.getAttribute("data-memory-page-count")),
sampleCount: numberValue(memoryChart?.getAttribute("data-memory-sample-count")),
source: memoryChart?.getAttribute("data-memory-source") || null,
apiOk: targetDetailResult.ok === true && targetDetailPayload.ok !== false,
apiPageCount: targetPageSeries.length,
apiSampleCount: numberValue(targetMemory.sampleCount),
apiSource: targetMemory.source || null,
expectedFromApi: targetPageSeries.length > 0 || numberValue(targetMemory.sampleCount) > 0,
contractOk: false,
status: "unknown",
};
memorySummary.contractOk = memorySummary.apiOk === true
&& (memorySummary.expectedFromApi !== true || (
memorySummary.present === true
&& memorySummary.matchesTargetRun === true
&& memorySummary.pageCount === memorySummary.apiPageCount
));
memorySummary.status = memorySummary.apiOk !== true
? "api-unavailable"
: memorySummary.expectedFromApi === true
? memorySummary.contractOk === true ? "rendered" : "mismatch"
: "no-samples";
const datasetSentinelId = String(dataset.sentinelId || "");
const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/";
const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/";
const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/");
const sentinelBoundary = {
expectedSentinelId,
expectedRoutePrefix,
datasetSentinelId,
overviewSentinelId: overviewPayload.sentinelId || null,
runsSentinelId: runsPayload.sentinelId || null,
finalPath,
routePrefixMatches,
datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true,
overviewMatches: expectedSentinelId ? overviewPayload.sentinelId === expectedSentinelId : true,
runsPayloadMatches: expectedSentinelId ? runsPayload.sentinelId === expectedSentinelId : true,
runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true,
};
const doc = document.documentElement;
const body = document.body;
const viewport = { width: window.innerWidth, height: window.innerHeight };
const documentSize = {
width: Math.max(doc.scrollWidth, body?.scrollWidth || 0),
height: Math.max(doc.scrollHeight, body?.scrollHeight || 0),
};
const overflow = [];
let overflowCount = 0;
for (const element of Array.from(document.querySelectorAll("body *"))) {
const rect = element.getBoundingClientRect();
const overflowRight = rect.right - viewport.width;
const overflowLeft = -rect.left;
if (overflowRight > 1 || overflowLeft > 1) {
overflowCount += 1;
if (overflow.length < 5) {
overflow.push({
tag: element.tagName.toLowerCase(),
x: Math.round(rect.x),
y: Math.round(rect.y),
width: Math.round(rect.width),
height: Math.round(rect.height),
overflowRight: Math.max(0, Math.round(overflowRight)),
overflowLeft: Math.max(0, Math.round(overflowLeft)),
});
}
}
}
return {
shell: Boolean(root && shell),
ready: root?.getAttribute("data-monitor-ready") === "true",
dataset,
contract: {
htmlShell: Boolean(root && shell),
appReady: root?.getAttribute("data-monitor-ready") === "true",
apiOverview: overviewResult.ok === true && overviewPayload.ok !== false,
apiRuns: runsResult.ok === true && Array.isArray(runs),
runCount: runs.length,
latestRunId: latestRunCounts.runId,
latestFindingTypeCount: latestRunCounts.typeCount,
trendCurves: chartTiming.ok,
memoryContract: memorySummary.contractOk,
},
sentinelBoundary,
title: document.title,
finalUrl: window.location.href,
runRows: runs.length,
checkRows: document.querySelectorAll("[data-check-row='true']").length,
latestRunCounts,
targetRunCounts: {
runId: targetRunId,
scenarioId: targetRun?.scenarioId || targetRun?.scenario_id || null,
durationMinutes: numberValue(targetRun?.runDurationMinutes ?? targetRun?.durationMinutes ?? targetRun?.timing?.durationMinutes),
requested: Boolean(requestedRunId),
requestMatched: !requestedRunId || targetRunId === requestedRunId,
},
requestedRunSelection,
manualTriggerUi: {
requested: manualTrigger.requested,
ok: manualTrigger.ok,
status: manualTrigger.status,
jobName: manualTrigger.jobName,
buttonPresent: Boolean(manualTriggerButton),
buttonDisabled: manualTriggerButton instanceof HTMLButtonElement ? manualTriggerButton.disabled : null,
buttonState: manualTriggerButton?.getAttribute("data-trigger-state") || null,
statusText: String(manualTriggerStatus?.textContent || "").replace(/\s+/g, " ").trim(),
},
chartTiming,
memorySummary,
api: {
overview: { ok: overviewResult.ok, httpStatus: overviewResult.httpStatus },
runs: { ok: runsResult.ok, httpStatus: runsResult.httpStatus },
targetDetail: { ok: targetDetailResult.ok, httpStatus: targetDetailResult.httpStatus },
},
errorVisible: Boolean(error && !error.hidden),
errorText: error && !error.hidden ? String(error.textContent || "").replace(/\s+/g, " ").trim().slice(0, 500) : "",
layout: {
viewport,
documentSize,
horizontalOverflow: documentSize.width > viewport.width + 1,
overflowCount,
overflow: overflow.slice(0, 2),
},
};
}, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger });
if (captureScreenshot && screenshotPath) {
if (requestedRunId && dom.memorySummary?.present === true && dom.memorySummary?.matchesTargetRun === true) {
await page.evaluate(() => {
const chart = document.querySelector("[data-run-memory-chart='true']");
if (chart instanceof HTMLElement) chart.scrollIntoView({ block: "center", inline: "nearest", behavior: "auto" });
}).catch(() => null);
await page.waitForTimeout(150);
}
await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => {
pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) });
});
}
const consoleErrors = consoleMessages.filter((item) => item.type === "error");
const navigationOk = !navigationError || (dom.shell === true && dom.ready === true);
const ok = navigationOk
&& httpStatus !== null
&& httpStatus >= 200
&& httpStatus < 300
&& dom.shell === true
&& dom.contract?.apiOverview === true
&& dom.contract?.apiRuns === true
&& dom.sentinelBoundary?.datasetMatches === true
&& dom.sentinelBoundary?.overviewMatches === true
&& dom.sentinelBoundary?.runsPayloadMatches === true
&& dom.sentinelBoundary?.runRowsMatch === true
&& dom.sentinelBoundary?.routePrefixMatches === true
&& dom.errorVisible !== true
&& dom.requestedRunSelection?.ok === true
&& manualTrigger.ok === true
&& dom.chartTiming?.ok === true
&& dom.memorySummary?.contractOk === true
&& dom.layout?.horizontalOverflow !== true
&& pageErrors.length === 0;
console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
ok,
url,
httpStatus,
navigationError,
navigationAttempts,
executablePath: executablePath || null,
viewport: { width, height },
screenshotPath: captureScreenshot ? screenshotPath : null,
manualTrigger,
dom,
consoleCount: consoleMessages.length,
consoleErrorCount: consoleErrors.length,
pageErrorCount: pageErrors.length,
requestFailureCount: requestFailures.length,
consoleMessages: consoleMessages.slice(0, 8),
pageErrors: pageErrors.slice(0, 8),
requestFailures: requestFailures.slice(0, 8),
valuesRedacted: true,
}));
await context.close().catch(() => {});
await browser.close().catch(() => {});
`;
}
function parseDashboardBrowserPayload(textValue: string): Record<string, unknown> | null {
const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__";
const index = textValue.lastIndexOf(marker);
if (index < 0) return null;
try {
return record(JSON.parse(textValue.slice(index + marker.length).trim()));
} catch {
return null;
}
}
function dashboardScreenshotName(options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>, state: SentinelCicdState): string {
const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`;
const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120);
return safe.endsWith(".png") ? safe : `${safe}.png`;
}
function compactDashboardArtifact(artifact: Record<string, unknown>): Record<string, unknown> {
const transfer = record(artifact.transfer);
return {
remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null,
localPath: typeof artifact.localPath === "string" ? artifact.localPath : null,
bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null,
sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null,
verified: artifact.verified === true,
transfer: Object.keys(transfer).length === 0 ? null : {
strategy: transfer.strategy ?? null,
transport: transfer.transport ?? null,
chunks: transfer.chunks ?? null,
elapsedMs: transfer.elapsedMs ?? null,
throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null,
},
};
}
function dashboardDegradedReason(result: CommandResult, transport: Record<string, unknown>, page: Record<string, unknown> | null, screenshotOk: boolean): string {
if (result.timedOut) return "sentinel-dashboard-command-timeout";
if (transport.ok !== true) {
const pollFailure = record(transport.pollFailure);
if (pollFailure.phase === "timeout") return "sentinel-dashboard-transport-timeout";
if (Object.keys(pollFailure).length > 0) return "sentinel-dashboard-transport-failed";
return "sentinel-dashboard-transport-failed";
}
if (page === null) return "sentinel-dashboard-browser-output-missing";
const manualTrigger = record(page.manualTrigger);
if (manualTrigger.requested === true && manualTrigger.ok !== true) return "sentinel-dashboard-manual-trigger-failed";
if (page.ok !== true) return "sentinel-dashboard-render-failed";
if (!screenshotOk) return "sentinel-dashboard-screenshot-missing";
return "sentinel-dashboard-unknown";
}
function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult {
const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
if (releaseId !== null) args.push("--release-id", releaseId);
if (reason !== null) args.push("--reason", reason);
if (quickVerify) args.push("--quick-verify");
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`);
const command = `web-probe sentinel ${subcommand.join(" ")}`;
return rendered(true, command, renderAsyncJobResult({
ok: true,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "async-job",
mutation: true,
job,
next: {
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
},
valuesRedacted: true,
}));
}
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
const namespace = stringAt(state.runtime, "namespace");
const serviceName = stringAt(state.runtime, "serviceName");
const servicePort = numberAt(state.runtime, "servicePort");
const deploymentName = stringAt(state.runtime, "deploymentName");
const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`;
if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") {
return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url);
}
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
const postScript = [
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
"}).catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(23);",
"});",
].join(" ");
const script = method === "GET"
? `kubectl get --raw ${shellQuote(proxyPath)}`
: [
"set -eu",
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
].join("\n");
const maxAttempts = method === "GET" ? 3 : 1;
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
const attempts: Record<string, unknown>[] = [];
let result: CommandResult | null = null;
let parsed: Record<string, unknown> | null = null;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 });
parsed = parseJsonObject(result.stdout);
const recovered = parsed === null ? recoverTruncatedSshStdoutJson(result) : null;
if (recovered !== null) parsed = recovered.parsed;
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, parsedFromDump: recovered !== null, dumpPath: recovered?.dumpPath ?? null, valuesRedacted: true });
if (result.exitCode === 0) break;
}
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
return {
ok: result?.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
httpStatus: result?.exitCode === 0 ? 200 : null,
bodyJson: record(compactBodyJson),
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
proxyPath,
result: result === null ? null : compactCommand(result),
attempts,
valuesRedacted: true,
};
}
function recoverTruncatedSshStdoutJson(result: CommandResult): { parsed: Record<string, unknown>; dumpPath: string } | null {
const dumpPath = sshStdoutDumpPathFromStderr(result.stderr);
if (dumpPath === null || !existsSync(dumpPath)) return null;
const parsed = parseJsonObject(readFileSync(dumpPath, "utf8"));
return parsed === null ? null : { parsed, dumpPath };
}
function sshStdoutDumpPathFromStderr(value: string): string | null {
for (const rawLine of value.split(/\r?\n/u)) {
const line = rawLine.trim();
const prefix = "UNIDESK_SSH_STDOUT_TRUNCATED ";
if (!line.startsWith(prefix)) continue;
try {
const payload = JSON.parse(line.slice(prefix.length)) as unknown;
const dumpPath = stringAtNullable(record(payload), "dumpPath");
if (dumpPath !== null) return dumpPath;
} catch {
return null;
}
}
return null;
}
function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number, url: string): Record<string, unknown> {
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const fetchScript = [
"const method = process.env.SENTINEL_METHOD || 'GET';",
"const url = process.env.SENTINEL_URL || '';",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
"const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);",
"const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);",
"const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;",
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
"(async () => {",
" let lastError = null;",
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
" try {",
" const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
" process.exit(0);",
" } catch (error) {",
" lastError = error;",
" console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));",
" if (attempt < attempts && delayMs > 0) await sleep(delayMs);",
" }",
" }",
" console.error(lastError && lastError.stack ? lastError.stack : String(lastError));",
" process.exit(23);",
"})().catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(24);",
"});",
].join(" ");
const attempts = method === "GET" ? 6 : 3;
const retryDelayMs = 1000;
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70));
const result = runCommand(["node", "-e", fetchScript], repoRoot, {
timeoutMs: attemptTimeoutSeconds * 1000,
env: {
...process.env,
SENTINEL_METHOD: method,
SENTINEL_URL: url,
SENTINEL_BODY_B64: bodyB64,
SENTINEL_ATTEMPTS: String(attempts),
SENTINEL_RETRY_DELAY_MS: String(retryDelayMs),
},
});
const parsed = parseJsonObject(result.stdout);
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
return {
ok: result.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: url,
httpStatus: result.exitCode === 0 ? 200 : null,
bodyJson: record(compactBodyJson),
bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "",
bodyBytes: Buffer.byteLength(result.stdout),
error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000),
proxyPath: null,
result: compactCommand(result),
attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }],
transport: "direct-service",
valuesRedacted: true,
};
}
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
if (value === null || typeof value.renderedText !== "string") return value;
return {
...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]),
run: pickFields(record(value.run), ["id", "runId", "scenario_id", "scenarioId", "status", "node", "lane", "observer_id", "observerId", "state_dir", "stateDir", "report_json_sha256", "reportJsonSha256", "finding_count", "findingCount", "artifact_count", "artifactCount", "maintenance", "created_at", "createdAt", "updated_at", "updatedAt", "startedAt", "finishedAt", "durationMs", "runDurationMs", "durationMinutes", "runDurationMinutes", "durationSource", "scenarioCadence", "scenarioCadenceMinutes", "scenarioMaxRunMinutes", "schedulerIntervalMinutes", "timing"]),
summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]),
findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [],
renderedText: value.renderedText,
valuesRedacted: true,
};
}
function pickFields(value: Record<string, unknown>, keys: readonly string[]): Record<string, unknown> {
const picked: Record<string, unknown> = {};
for (const key of keys) {
if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key];
}
return picked;
}
function clipTail(value: string, maxChars: number): string {
return value.length <= maxChars ? value : value.slice(-maxChars);
}
function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`;
const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000));
const js = [
"const url=process.env.REQ_URL||'';",
"const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);",
"let out;",
"try{",
" const controller=new AbortController();",
" const timer=setTimeout(()=>controller.abort(), timeoutMs);",
" const started=Date.now();",
" const res=await fetch(url,{signal:controller.signal});",
" const text=await res.text();",
" clearTimeout(timer);",
" let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}",
" out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};",
"}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}",
"console.log(JSON.stringify(out));",
].join("");
const result = runCommand(["bun", "-e", js], repoRoot, {
timeoutMs: timeoutMs + 2000,
env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) },
});
const parsed = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && parsed?.ok === true,
method: "GET",
path: pathWithQuery,
publicUrl: url,
httpStatus: parsed?.httpStatus ?? null,
bodyJson: record(parsed?.bodyJson),
bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "",
bodyBytes: parsed?.bodyBytes ?? null,
error: parsed?.error ?? null,
result: compactCommand(result),
valuesRedacted: true,
};
}
function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl");
const hostname = stringAt(state.publicExposure, "hostname");
const expectedA = stringAt(state.publicExposure, "expectedA");
const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`;
const script = [
"set +e",
`host=${shellQuote(hostname)}`,
`expected=${shellQuote(expectedA)}`,
`url=${shellQuote(probeUrl)}`,
"dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)",
"headers=$(mktemp)",
"body=$(mktemp)",
"writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)",
"curl_rc=$?",
"body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')",
"node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'",
"const fs=require('node:fs');",
"const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);",
"const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);",
"const status=Number(statusRaw||0);",
"const ssl=Number(sslRaw||-1);",
"const addrs=dns?dns.split(',').filter(Boolean):[];",
"const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();",
"const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');",
"let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}",
"const authCovered=status===401||status===403||status>=200&&status<300;",
"const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;",
"const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));",
"const dnsMatches=addrs.includes(expected);",
"console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const rootUrl = `${publicBaseUrl}/`;
const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`;
const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`;
const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`;
const script = [
"set +e",
`root_url=${shellQuote(rootUrl)}`,
`css_url=${shellQuote(cssUrl)}`,
`js_url=${shellQuote(jsUrl)}`,
`vue_url=${shellQuote(vueUrl)}`,
"root_body=$(mktemp)",
"css_body=$(mktemp)",
"js_body=$(mktemp)",
"vue_body=$(mktemp)",
"root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?",
"css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?",
"js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?",
"vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?",
"node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'",
"const fs=require('node:fs');",
"const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);",
"function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}",
"const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);",
"const contractMatch=/data-contract-version=\"([^\"]+)\"/u.exec(root);",
"const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&Boolean(contractMatch);",
"const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.length>1000;",
"const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.length>1000;",
"const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.length>80000;",
"console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contractVersion:contractMatch?contractMatch[1]:null},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css)},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js)},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue)},valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function renderMaintenanceResult(result: Record<string, unknown>): string {
const serviceHealth = record(result.serviceHealth);
const maintenance = record(result.maintenance);
const quickVerify = record(result.quickVerify);
const quickVerifyBusiness = record(quickVerify.businessStatus);
const planned = record(result.planned);
const blocker = record(result.blocker);
const next = record(result.next);
const maintenanceBody = record(maintenance.bodyJson);
const state = record(maintenanceBody.maintenance);
return [
String(result.command),
"",
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]),
"",
table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]),
"",
Object.keys(state).length > 0
? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]])
: Object.keys(planned).length > 0
? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]])
: "MAINTENANCE\n-",
"",
Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]),
"",
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
"",
"NEXT",
` validate: ${next.validate ?? "-"}`,
` report: ${next.report ?? "-"}`,
` maintenance-stop: ${next.maintenanceStop ?? "-"}`,
"",
"DISCLOSURE",
" maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.",
].join("\n");
}
function renderValidateResult(result: Record<string, unknown>): string {
const health = record(result.serviceHealth);
const metrics = record(result.metrics);
const report = record(result.report);
const publicExposure = record(result.publicExposure);
const publicDashboard = record(result.publicDashboard);
const quickVerify = record(result.quickVerify);
const blocker = record(result.blocker);
const next = record(result.next);
const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []);
return [
String(result.command),
"",
table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]),
"",
table(["CHECK", "OK", "DETAIL"], [
["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`],
["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`],
["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`],
["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`],
["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`],
["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`],
]),
"",
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
"",
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]),
"",
"NEXT",
` quick-verify: ${next.quickVerify ?? "-"}`,
` report: ${next.report ?? "-"}`,
` maintenance-start: ${next.maintenanceStart ?? "-"}`,
"",
"DISCLOSURE",
" validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.",
].join("\n");
}
function renderDashboardResult(result: Record<string, unknown>): string {
const page = record(result.page);
const dom = record(page.dom);
const dataset = record(dom.dataset);
const contract = record(dom.contract);
const sentinelBoundary = record(dom.sentinelBoundary);
const api = record(dom.api);
const apiOverview = record(api.overview);
const apiRuns = record(api.runs);
const layout = record(dom.layout);
const latestRunCounts = record(dom.latestRunCounts);
const targetRunCounts = record(dom.targetRunCounts);
const chartTiming = record(dom.chartTiming);
const memorySummary = record(dom.memorySummary);
const requestedRunSelection = record(dom.requestedRunSelection);
const manualTrigger = record(page.manualTrigger);
const manualTriggerUi = record(dom.manualTriggerUi);
const screenshot = record(result.screenshot);
const remote = record(result.remote);
const transport = record(result.transport);
const pollFailure = record(transport.pollFailure);
const degradedReason = result.degradedReason ?? null;
return [
String(result.command),
"",
table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]),
"",
table(["HTTP", "SHELL", "READY", "API_OVERVIEW", "API_RUNS", "RUN_ROWS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[
page.httpStatus ?? "-",
dom.shell,
dom.ready,
`${apiOverview.ok ?? "-"}/${apiOverview.httpStatus ?? "-"}`,
`${apiRuns.ok ?? "-"}/${apiRuns.httpStatus ?? "-"}`,
dom.runRows,
page.pageErrorCount,
page.consoleErrorCount,
page.requestFailureCount,
]]),
"",
table(["CONTRACT", "BASE_PATH", "DATASET_SENTINEL", "OVERVIEW_SENTINEL", "RUNS_SENTINEL", "ROUTE_MATCH"], [[
dataset.contractVersion ?? "-",
dataset.basePath ?? "-",
sentinelBoundary.datasetSentinelId ?? "-",
sentinelBoundary.overviewSentinelId ?? "-",
sentinelBoundary.runsSentinelId ?? "-",
sentinelBoundary.routePrefixMatches ?? "-",
]]),
"",
table(["HTML_SHELL", "APP_READY", "API_OVERVIEW_OK", "API_RUNS_OK", "LATEST_RUN", "TYPE_COUNT"], [[
contract.htmlShell ?? "-",
contract.appReady ?? "-",
contract.apiOverview ?? "-",
contract.apiRuns ?? "-",
latestRunCounts.runId ?? "-",
latestRunCounts.typeCount ?? "-",
]]),
"",
table(["TARGET_RUN", "REQUESTED", "SELECTED", "TREND_OK", "ERR_CURVE", "WARN_CURVE"], [[
targetRunCounts.runId ?? "-",
targetRunCounts.requested ?? "-",
requestedRunSelection.ok ?? "-",
chartTiming.ok ?? "-",
chartTiming.hasErrorCurve ?? "-",
chartTiming.hasWarningCurve ?? "-",
]]),
"",
manualTrigger.requested === true
? table(["TRIGGER", "STATUS", "JOB", "UI_STATE", "UI_TEXT"], [[
manualTrigger.ok ?? "-",
manualTrigger.status ?? "-",
manualTrigger.jobName ?? "-",
manualTriggerUi.buttonState ?? "-",
manualTriggerUi.statusText ?? "-",
]])
: "TRIGGER\n-",
"",
table(["MEMORY_CHART", "MEMORY_RUN", "MEMORY_MATCH", "MEMORY_PAGES", "MEMORY_SAMPLES", "API_PAGES", "API_SAMPLES", "CONTRACT", "STATUS", "MEMORY_SOURCE"], [[
memorySummary.present ?? "-",
memorySummary.runId ?? "-",
memorySummary.matchesTargetRun ?? "-",
memorySummary.pageCount ?? "-",
memorySummary.sampleCount ?? "-",
memorySummary.apiPageCount ?? "-",
memorySummary.apiSampleCount ?? "-",
memorySummary.contractOk ?? "-",
memorySummary.status ?? "-",
memorySummary.source ?? memorySummary.apiSource ?? "-",
]]),
"",
table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[
result.viewport,
`${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`,
layout.horizontalOverflow,
layout.overflowCount,
]]),
"",
Object.keys(screenshot).length === 0
? "SCREENSHOT\n-"
: table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]),
"",
degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT", "POLL_PHASE"], [[degradedReason, remote.exitCode, transport.ok, pollFailure.phase ?? "-"]]),
"",
"NEXT",
` trigger: bun scripts/cli.ts web-probe sentinel dashboard trigger --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
"",
"DISCLOSURE",
" dashboard verify uses YAML publicExposure, remote browser execution and API/data-contract checks; it does not assert UI copy or CSS class names.",
].join("\n");
}
function renderReportResult(result: Record<string, unknown>): string {
const report = record(result.report);
const body = record(report.bodyJson);
const run = record(body.run);
return [
String(result.command),
"",
table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]),
"",
table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]),
"",
"DISCLOSURE",
" report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.",
].join("\n");
}