1688 lines
92 KiB
TypeScript
1688 lines
92 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
|
|
// Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands.
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import type { CommandResult } from "./command";
|
|
import { runCommand } from "./command";
|
|
import { repoRoot } from "./config";
|
|
import { startJob } from "./jobs";
|
|
import type { RenderedCliResult } from "./output";
|
|
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
|
|
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
|
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
|
|
import {
|
|
clipTail,
|
|
compactCommand,
|
|
compactSentinelServiceBodyJson,
|
|
mergeWarnings,
|
|
numberAt,
|
|
numberAtNullable,
|
|
parseJsonObject,
|
|
pickFields,
|
|
record,
|
|
rendered,
|
|
renderAsyncJobResult,
|
|
safeJobSegment,
|
|
sentinelCliSuffix,
|
|
shellQuote,
|
|
short,
|
|
stringAt,
|
|
stringAtNullable,
|
|
table,
|
|
targetValidationElapsedWarnings,
|
|
text,
|
|
withWarnings,
|
|
} from "./hwlab-node-web-sentinel-cicd";
|
|
import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
|
|
|
|
const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55;
|
|
|
|
export function runSentinelMaintenance(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "maintenance" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel maintenance ${options.action}`;
|
|
const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
|
|
if (options.action === "status") {
|
|
const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds);
|
|
const result = {
|
|
ok: serviceHealth.ok && maintenance.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
serviceHealth,
|
|
maintenance,
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderMaintenanceResult(result));
|
|
}
|
|
if (!options.confirm) {
|
|
const result = {
|
|
ok: serviceHealth.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "dry-run",
|
|
serviceHealth,
|
|
mutation: false,
|
|
planned: {
|
|
action: options.action,
|
|
releaseId: options.releaseId,
|
|
reason: options.reason,
|
|
quickVerify: options.action === "stop" && options.quickVerify,
|
|
},
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderMaintenanceResult(result));
|
|
}
|
|
if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify);
|
|
if (!serviceHealth.ok) {
|
|
const result = {
|
|
ok: false,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: false,
|
|
serviceHealth,
|
|
blocker: serviceUnavailableBlocker(state),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(false, command, renderMaintenanceResult(result));
|
|
}
|
|
const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true };
|
|
const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds);
|
|
const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok
|
|
? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds)
|
|
: null;
|
|
const result = {
|
|
ok: mutation.ok && (quickVerify === null || quickVerify.ok === true),
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
mutation: true,
|
|
serviceHealth,
|
|
maintenance: mutation,
|
|
quickVerify,
|
|
blocker: mutation.ok ? null : serviceUnavailableBlocker(state),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderMaintenanceResult(result));
|
|
}
|
|
|
|
export function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
|
|
const command = "web-probe sentinel validate";
|
|
const startedAt = Date.now();
|
|
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
|
|
let quickVerify: Record<string, unknown> | null = null;
|
|
if (options.quickVerify) {
|
|
if (!options.confirm) {
|
|
const result = {
|
|
ok: initialHealth.ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "dry-run",
|
|
serviceHealth: initialHealth,
|
|
planned: { quickVerify: true, waitRequired: true },
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(result.ok, command, renderValidateResult(result));
|
|
}
|
|
if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true);
|
|
if (!initialHealth.ok) {
|
|
const result = {
|
|
ok: false,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "confirm-wait",
|
|
serviceHealth: initialHealth,
|
|
blocker: serviceUnavailableBlocker(state),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(false, command, renderValidateResult(result));
|
|
}
|
|
quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds);
|
|
}
|
|
const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20);
|
|
const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds);
|
|
const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds);
|
|
const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds);
|
|
const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health");
|
|
const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds);
|
|
const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds);
|
|
const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds);
|
|
const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health;
|
|
const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics;
|
|
const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report;
|
|
const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds);
|
|
const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds);
|
|
if (quickVerify !== null) {
|
|
quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds"))));
|
|
}
|
|
const publicFallbackWarnings = [
|
|
...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []),
|
|
...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []),
|
|
...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []),
|
|
];
|
|
const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health");
|
|
const ok = effectiveHealth.ok
|
|
&& record(effectiveHealth.bodyJson).ok === true
|
|
&& effectiveMetricsOk
|
|
&& effectiveReport.ok
|
|
&& publicExposure.ok === true
|
|
&& publicDashboard.ok === true
|
|
&& (quickVerify === null || quickVerify.ok === true);
|
|
const result = {
|
|
ok,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: options.quickVerify ? "confirm-wait" : "status",
|
|
serviceHealth: effectiveHealth,
|
|
metrics: effectiveMetrics,
|
|
report: effectiveReport,
|
|
internalServiceHealth: health,
|
|
internalMetrics: metrics,
|
|
internalReport: report,
|
|
publicServiceHealth: publicHealth,
|
|
publicMetrics,
|
|
publicReport,
|
|
publicExposure,
|
|
publicDashboard,
|
|
quickVerify,
|
|
warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []),
|
|
blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify),
|
|
next: sentinelP5Next(state),
|
|
valuesRedacted: true,
|
|
};
|
|
return rendered(ok, command, renderValidateResult(result));
|
|
}
|
|
|
|
export function probeSentinelRuntimeHealthEndpoint(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const endpoint = stringAt(state.runtime, "healthPath");
|
|
const serviceProbeTimeoutSeconds = Math.min(timeoutSeconds, 20);
|
|
const health = callSentinelService(state, "GET", endpoint, null, serviceProbeTimeoutSeconds);
|
|
const publicHealth = health.ok ? null : probePublicSentinelService(state, endpoint, serviceProbeTimeoutSeconds);
|
|
const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health;
|
|
const bodyJson = record(effectiveHealth.bodyJson);
|
|
const ok = effectiveHealth.ok === true && bodyJson.ok === true;
|
|
return {
|
|
ok,
|
|
endpoint,
|
|
health: effectiveHealth,
|
|
internalHealth: health,
|
|
publicHealth,
|
|
httpStatus: effectiveHealth.httpStatus ?? null,
|
|
publicUrl: effectiveHealth.publicUrl ?? null,
|
|
internalUrl: effectiveHealth.internalUrl ?? null,
|
|
degradedReason: ok ? null : "sentinel-runtime-health-endpoint-failed",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
export function runSentinelReport(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "report" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`;
|
|
const query = new URLSearchParams({ view: options.view });
|
|
if (options.runId !== null) query.set("run", options.runId);
|
|
if (options.traceId !== null) query.set("traceId", options.traceId);
|
|
if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq));
|
|
const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds);
|
|
const body = record(report.bodyJson);
|
|
const rawPayload = Object.keys(body).length > 0 ? body : report;
|
|
if (options.full) return rendered(report.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2));
|
|
if (options.raw) {
|
|
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS));
|
|
return rendered(report.ok && body.ok !== false, command, JSON.stringify(compactSentinelReportRawPayload(state, body, report, artifactSummary), null, 2));
|
|
}
|
|
if (options.view === "summary") {
|
|
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS));
|
|
const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary);
|
|
return rendered(report.ok && body.ok !== false, command, renderSentinelReportSummary(payload, state));
|
|
}
|
|
if (options.view === "findings") {
|
|
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS));
|
|
const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary);
|
|
return rendered(report.ok && body.ok !== false, command, renderSentinelReportFindings(payload));
|
|
}
|
|
const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true });
|
|
return rendered(report.ok && body.ok !== false, command, renderedText);
|
|
}
|
|
|
|
function compactSentinelReportRawPayload(
|
|
state: SentinelCicdState,
|
|
body: Record<string, unknown>,
|
|
report: Record<string, unknown>,
|
|
artifactSummary: Record<string, unknown> | null,
|
|
): Record<string, unknown> {
|
|
const run = record(body.run);
|
|
const artifact = record(artifactSummary);
|
|
const findings = Array.isArray(body.findings) ? body.findings.map(record) : [];
|
|
const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : [];
|
|
const visibleFindings = mergeSentinelReportFindings(findings, artifactFindings);
|
|
const offlineReclassify = offlineQuickVerifyReclassify(state, run, visibleFindings);
|
|
const storedFindingCount = numberAtNullable(run, "finding_count") ?? numberAtNullable(run, "findingCount") ?? findings.length;
|
|
const artifactFindingCount = numberAtNullable(artifact, "findingCount") ?? artifactFindings.length;
|
|
const visibleFindingCount = Math.max(storedFindingCount, artifactFindingCount, visibleFindings.length);
|
|
const rootCauseSignalFindings = artifactFindings
|
|
.filter((item) => Object.keys(record(item.rootCauseSignals)).length > 0)
|
|
.slice(0, 8)
|
|
.map((item) => ({
|
|
id: stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"),
|
|
severity: stringAtNullable(item, "severity") ?? stringAtNullable(item, "level"),
|
|
summary: reportText(item.summary ?? item.message, 220),
|
|
rootCauseSignals: compactRootCauseSignals(item.rootCauseSignals),
|
|
valuesRedacted: true,
|
|
}));
|
|
return {
|
|
ok: body.ok !== false && report.ok !== false,
|
|
view: body.view ?? null,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
run: {
|
|
id: run.id ?? null,
|
|
scenarioId: run.scenario_id ?? run.scenarioId ?? null,
|
|
status: run.status ?? null,
|
|
observerId: run.observer_id ?? run.observerId ?? null,
|
|
stateDir: run.state_dir ?? run.stateDir ?? null,
|
|
reportJsonSha256: run.report_json_sha256 ?? run.reportJsonSha256 ?? artifact.reportJsonSha256 ?? null,
|
|
findingCount: visibleFindingCount,
|
|
storedFindingCount,
|
|
artifactFindingCount,
|
|
artifactCount: run.artifact_count ?? run.artifactCount ?? artifact.artifactCount ?? null,
|
|
durationMinutes: run.durationMinutes ?? run.runDurationMinutes ?? record(run.timing).durationMinutes ?? null,
|
|
runDurationMinutes: run.runDurationMinutes ?? run.durationMinutes ?? record(run.timing).durationMinutes ?? null,
|
|
durationSource: run.durationSource ?? record(run.timing).durationSource ?? null,
|
|
scenarioCadenceMinutes: run.scenarioCadenceMinutes ?? record(run.timing).scenarioCadenceMinutes ?? null,
|
|
scenarioMaxRunMinutes: run.scenarioMaxRunMinutes ?? record(run.timing).scenarioMaxRunMinutes ?? null,
|
|
schedulerIntervalMinutes: run.schedulerIntervalMinutes ?? record(run.timing).schedulerIntervalMinutes ?? null,
|
|
timing: pickFields(record(run.timing), ["startedAt", "finishedAt", "durationMs", "durationMinutes", "durationSource", "scenarioCadence", "scenarioCadenceMinutes", "scenarioMaxRunMinutes", "schedulerIntervalMinutes", "sourceOfTruth", "valuesRedacted"]),
|
|
updatedAt: run.updated_at ?? run.updatedAt ?? null,
|
|
valuesRedacted: true,
|
|
},
|
|
summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]),
|
|
findings: visibleFindings.slice(0, 12).map(compactSentinelReportFinding),
|
|
offlineReclassify,
|
|
artifactSummary: Object.keys(artifact).length === 0 ? null : {
|
|
ok: artifact.ok === true,
|
|
reason: artifact.reason ?? null,
|
|
reportReadWaitMs: artifact.reportReadWaitMs ?? null,
|
|
reportParseError: artifact.reportParseError ?? null,
|
|
reportOk: artifact.reportOk === true ? true : artifact.reportOk === false ? false : null,
|
|
reportJsonPath: artifact.reportJsonPath ?? null,
|
|
reportJsonSha256: artifact.reportJsonSha256 ?? null,
|
|
reportMdSha256: artifact.reportMdSha256 ?? null,
|
|
findingCount: artifactFindingCount,
|
|
findings: artifactFindings.slice(0, 12).map(compactSentinelReportFinding),
|
|
screenshot: record(artifact.screenshot),
|
|
counts: record(artifact.counts),
|
|
analysisWindow: compactSentinelAnalysisWindow(artifact.analysisWindow),
|
|
pagePerformanceSlowApi: Array.isArray(artifact.pagePerformanceSlowApi) ? artifact.pagePerformanceSlowApi.slice(0, 8).map(record) : [],
|
|
rootCauseSignalFindings,
|
|
valuesRedacted: true,
|
|
},
|
|
next: {
|
|
text: "Default report is bounded text; use --full for the full indexed service payload.",
|
|
report: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")}`,
|
|
full: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")} --full`,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record<string, unknown>, findings: readonly Record<string, unknown>[]): Record<string, unknown> | null {
|
|
const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("quick-verify-no-business-turn"));
|
|
if (!hasLegacyQuickVerifyControl) return null;
|
|
const catalog = sentinelReportCheckCatalogById(state);
|
|
const result = reclassifyQuickVerifyControlFindings(state, {
|
|
runId: stringAtNullable(run, "id"),
|
|
scenarioId: stringAtNullable(run, "scenario_id") ?? stringAtNullable(run, "scenarioId"),
|
|
observerId: stringAtNullable(run, "observer_id") ?? stringAtNullable(run, "observerId"),
|
|
failure: stringAtNullable(record(run.summary), "failure"),
|
|
timeoutSeconds: SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS,
|
|
});
|
|
const resultFindings = Array.isArray(result.findings) ? result.findings.map(record) : [];
|
|
return {
|
|
...pickFields(result, ["ok", "reason", "runId", "scenarioId", "observerId", "promptIndex", "findingCount", "turnSummary", "traceFrame", "valuesRedacted"]),
|
|
source: "offline-existing-observe-artifact",
|
|
note: "This is a local CLI reclassification of existing turn-summary/trace-frame artifacts; it does not mutate the runner index.",
|
|
findings: resultFindings.slice(0, 8).map((item) => compactSentinelReportFinding(enrichSentinelReportFindingWithCatalog(item, catalog))),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelReportCheckCatalogById(state: SentinelCicdState): Map<string, Record<string, unknown>> {
|
|
try {
|
|
const reportViews = record(readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.reportViews));
|
|
const catalogRef = stringAtNullable(reportViews, "checkCatalogRef");
|
|
if (catalogRef === null) return new Map();
|
|
const catalog = record(readWebProbeSentinelConfigRefTarget(state.spec, catalogRef));
|
|
const items = Array.isArray(catalog.items) ? catalog.items.map(record) : Array.isArray(catalog.checks) ? catalog.checks.map(record) : [];
|
|
return new Map(items.map((item) => [stringAtNullable(item, "id") ?? "", item]).filter((item): item is [string, Record<string, unknown>] => item[0].length > 0));
|
|
} catch {
|
|
return new Map();
|
|
}
|
|
}
|
|
|
|
function enrichSentinelReportFindingWithCatalog(item: Record<string, unknown>, catalog: ReadonlyMap<string, Record<string, unknown>>): Record<string, unknown> {
|
|
const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code");
|
|
const check = id === null ? null : catalog.get(id) ?? null;
|
|
if (check === null) return item;
|
|
return {
|
|
...item,
|
|
check,
|
|
checkCode: stringAtNullable(check, "code"),
|
|
checkTitleZh: stringAtNullable(check, "titleZh"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function mergeSentinelReportFindings(primary: readonly Record<string, unknown>[], artifact: readonly Record<string, unknown>[]): Record<string, unknown>[] {
|
|
const merged: Record<string, unknown>[] = [];
|
|
const seen = new Set<string>();
|
|
const aliases = sentinelReportFindingIdentityAliases(primary);
|
|
for (const item of [...primary, ...artifact]) {
|
|
const id = sentinelReportCanonicalFindingIdentity(item, aliases);
|
|
const key = id ?? `${JSON.stringify(item).slice(0, 160)}:${stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? ""}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
merged.push(item);
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
function sentinelReportFindingIdentityAliases(items: readonly Record<string, unknown>[]): Map<string, string> {
|
|
const aliases = new Map<string, string>();
|
|
for (const item of items) {
|
|
const canonical = sentinelReportFindingCanonicalCandidate(item) ?? sentinelReportFindingIdentity(item);
|
|
if (canonical === null) continue;
|
|
for (const candidate of sentinelReportFindingIdentityCandidates(item)) aliases.set(candidate, canonical);
|
|
}
|
|
return aliases;
|
|
}
|
|
|
|
function sentinelReportCanonicalFindingIdentity(item: Record<string, unknown>, aliases: ReadonlyMap<string, string>): string | null {
|
|
for (const candidate of sentinelReportFindingIdentityCandidates(item)) {
|
|
const canonical = aliases.get(candidate);
|
|
if (canonical !== undefined) return canonical;
|
|
}
|
|
return sentinelReportFindingCanonicalCandidate(item) ?? sentinelReportFindingIdentity(item);
|
|
}
|
|
|
|
function sentinelReportFindingCanonicalCandidate(item: Record<string, unknown>): string | null {
|
|
const check = record(item.check);
|
|
return stringAtNullable(item, "checkCode")
|
|
?? stringAtNullable(check, "code")
|
|
?? stringAtNullable(item, "checkId")
|
|
?? stringAtNullable(check, "id");
|
|
}
|
|
|
|
function sentinelReportFindingIdentityCandidates(item: Record<string, unknown>): readonly string[] {
|
|
const check = record(item.check);
|
|
const candidates = [
|
|
stringAtNullable(item, "finding_id"),
|
|
stringAtNullable(item, "findingId"),
|
|
stringAtNullable(item, "id"),
|
|
stringAtNullable(item, "kind"),
|
|
stringAtNullable(item, "code"),
|
|
stringAtNullable(item, "checkId"),
|
|
stringAtNullable(item, "checkCode"),
|
|
stringAtNullable(check, "id"),
|
|
stringAtNullable(check, "code"),
|
|
];
|
|
return [...new Set(candidates.filter((item): item is string => item !== null))];
|
|
}
|
|
|
|
function sentinelReportFindingIdentity(item: Record<string, unknown>): string | null {
|
|
return stringAtNullable(item, "finding_id")
|
|
?? stringAtNullable(item, "findingId")
|
|
?? stringAtNullable(item, "id")
|
|
?? stringAtNullable(item, "kind")
|
|
?? stringAtNullable(item, "code");
|
|
}
|
|
|
|
function compactSentinelReportFinding(value: Record<string, unknown>): Record<string, unknown> {
|
|
const result: Record<string, unknown> = {
|
|
id: stringAtNullable(value, "finding_id") ?? stringAtNullable(value, "findingId") ?? stringAtNullable(value, "id") ?? stringAtNullable(value, "kind") ?? stringAtNullable(value, "code"),
|
|
severity: stringAtNullable(value, "severity") ?? stringAtNullable(value, "level"),
|
|
count: value.count ?? null,
|
|
summary: reportText(value.summary ?? value.message, 220),
|
|
valuesRedacted: true,
|
|
};
|
|
const rootCause = stringAtNullable(value, "rootCause");
|
|
if (rootCause !== null) result.rootCause = rootCause;
|
|
const rootCauseStatus = stringAtNullable(value, "rootCauseStatus");
|
|
if (rootCauseStatus !== null) result.rootCauseStatus = rootCauseStatus;
|
|
const rootCauseConfidence = stringAtNullable(value, "rootCauseConfidence");
|
|
if (rootCauseConfidence !== null) result.rootCauseConfidence = rootCauseConfidence;
|
|
const nextAction = reportText(value.nextAction, 240);
|
|
if (nextAction !== null) result.nextAction = nextAction;
|
|
const evidenceSummary = reportText(value.evidenceSummary, 240);
|
|
if (evidenceSummary !== null) result.evidenceSummary = evidenceSummary;
|
|
const timingSourceOfTruth = stringAtNullable(value, "timingSourceOfTruth");
|
|
if (timingSourceOfTruth !== null) result.timingSourceOfTruth = timingSourceOfTruth;
|
|
const timingStatus = stringAtNullable(value, "timingStatus");
|
|
if (timingStatus !== null) result.timingStatus = timingStatus;
|
|
if (value.timingAlert === true) result.timingAlert = true;
|
|
const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals);
|
|
if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals;
|
|
const check = record(value.check);
|
|
const checkCodeFromValue = stringAtNullable(value, "checkCode");
|
|
const checkTitleFromValue = stringAtNullable(value, "checkTitleZh");
|
|
const checkCode = stringAtNullable(check, "code");
|
|
const checkTitle = stringAtNullable(check, "titleZh");
|
|
if (checkCodeFromValue !== null) result.checkCode = checkCodeFromValue;
|
|
if (checkTitleFromValue !== null) result.checkTitleZh = checkTitleFromValue;
|
|
if (checkCode !== null || checkTitle !== null) {
|
|
result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> | null {
|
|
const run = record(body.run);
|
|
const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir");
|
|
if (stateDir === null || !isSafeSentinelReportStateDir(stateDir)) return null;
|
|
const waitMs = Math.max(0, Math.min(50_000, (Math.max(5, timeoutSeconds) * 1000) - 5000));
|
|
const script = [
|
|
"set -eu",
|
|
`state_dir=${shellQuote(stateDir)}`,
|
|
`wait_ms=${waitMs}`,
|
|
"node - \"$state_dir\" \"$wait_ms\" <<'NODE'",
|
|
"const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');",
|
|
"const stateDir=process.argv[2]; const waitMs=Math.max(0, Math.min(60000, Number(process.argv[3]||0)||0)); const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');",
|
|
"const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}};",
|
|
"const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;",
|
|
"const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);",
|
|
"const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};",
|
|
"const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};",
|
|
"let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;",
|
|
"while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }",
|
|
"let artifactCount=0; let screenshot=null;",
|
|
"function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}",
|
|
"walk(stateDir);",
|
|
"const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});",
|
|
"const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});",
|
|
"console.log(JSON.stringify({ok:!!report,reason:report?null:(jsonBuf?'report-json-parse-failed':'report-json-missing'),reportReadWaitMs:waitMs,reportParseError:clip(reportParseError,220),reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function isSafeSentinelReportStateDir(value: string): boolean {
|
|
return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..") && !value.startsWith("/");
|
|
}
|
|
|
|
function renderSentinelReportSummary(payload: Record<string, unknown>, state: SentinelCicdState): string {
|
|
const run = record(payload.run);
|
|
const summary = record(payload.summary);
|
|
const artifactSummary = record(payload.artifactSummary);
|
|
const offlineReclassify = record(payload.offlineReclassify);
|
|
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
|
|
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
|
|
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
|
|
const findingCount = run.findingCount ?? findings.length;
|
|
const analyzerArtifactCount = numberAtNullable(artifactSummary, "artifactCount") ?? numberAtNullable(record(artifactSummary.counts), "artifacts");
|
|
const runArtifactCount = numberAtNullable(run, "artifactCount");
|
|
const artifactCount = runArtifactCount === null || runArtifactCount === 0 && analyzerArtifactCount !== null
|
|
? analyzerArtifactCount ?? "-"
|
|
: runArtifactCount;
|
|
const publicOrigin = stringAtNullable(state.publicExposure, "publicBaseUrl") ?? "-";
|
|
const findingRows = findings.length === 0
|
|
? "-"
|
|
: table(["ID", "SEVERITY", "COUNT", "SUMMARY"], findings.slice(0, 12).map((item) => [
|
|
stringAtNullable(item, "id") ?? "-",
|
|
stringAtNullable(item, "severity") ?? "-",
|
|
item.count ?? "-",
|
|
reportText(item.summary, 120) ?? "-",
|
|
]));
|
|
return [
|
|
"Web Probe Sentinel Quick Verify",
|
|
"=======================================================",
|
|
`run=${run.id ?? "-"} scenario=${run.scenarioId ?? "-"} observer=${run.observerId ?? "-"}`,
|
|
`status=${run.status ?? summary.status ?? "-"} reason=${summary.reason ?? "-"} failure=${summary.failure ?? "-"}`,
|
|
`report=${reportSha ?? "-"} artifacts=${artifactCount} findings=${findingCount}`,
|
|
`publicOrigin=${publicOrigin}`,
|
|
"",
|
|
"Findings",
|
|
findingRows,
|
|
offlineFindings.length === 0 ? "" : "",
|
|
offlineFindings.length === 0 ? "" : "Offline Reclassify",
|
|
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
|
|
].join("\n");
|
|
}
|
|
|
|
function renderSentinelReportFindings(payload: Record<string, unknown>): string {
|
|
const run = record(payload.run);
|
|
const artifactSummary = record(payload.artifactSummary);
|
|
const offlineReclassify = record(payload.offlineReclassify);
|
|
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
|
|
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
|
|
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
|
|
return [
|
|
"Web Probe Sentinel Findings",
|
|
"=======================================================",
|
|
`run=${run.id ?? "-"} report=${reportSha ?? "-"} findings=${run.findingCount ?? findings.length}`,
|
|
findings.length === 0 ? "-" : findings.map(formatSentinelReportFindingLine).join("\n"),
|
|
offlineFindings.length === 0 ? "" : "",
|
|
offlineFindings.length === 0 ? "" : "Offline Reclassify",
|
|
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
|
|
].join("\n");
|
|
}
|
|
|
|
function formatSentinelReportFindingLine(item: Record<string, unknown>): string {
|
|
const check = record(item.check);
|
|
const code = stringAtNullable(item, "checkCode") ?? stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-";
|
|
const title = stringAtNullable(item, "checkTitleZh") ?? stringAtNullable(check, "titleZh") ?? "";
|
|
const summary = reportText(item.summary, 180) ?? "";
|
|
const rootCause = reportText(item.rootCause, 180);
|
|
const evidence = reportText(item.evidenceSummary, 180);
|
|
const nextAction = reportText(item.nextAction, 180);
|
|
return [
|
|
`${item.severity ?? "-"} ${code} ${title} count=${item.count ?? "-"} ${summary}`.trim(),
|
|
rootCause === null ? null : `rootCause=${rootCause}`,
|
|
evidence === null ? null : `evidence=${evidence}`,
|
|
nextAction === null ? null : `next=${nextAction}`,
|
|
].filter((part): part is string => part !== null && part.length > 0).join(" | ");
|
|
}
|
|
|
|
function compactRootCauseSignals(value: unknown): Record<string, unknown> | null {
|
|
const item = record(value);
|
|
const keys = [
|
|
"sessionListReadCount",
|
|
"traceEventsReadCount",
|
|
"webPerformanceBeaconFailureCount",
|
|
"eventSourceFailureCount",
|
|
"requestFailedCount",
|
|
"httpErrorCount",
|
|
"consoleAlertCount",
|
|
"requestfailedTop",
|
|
"httpStatusTop",
|
|
];
|
|
const out: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
const raw = item[key];
|
|
if (raw === null || raw === undefined) continue;
|
|
out[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
|
|
}
|
|
return Object.keys(out).length === 0 ? null : { ...out, valuesRedacted: true };
|
|
}
|
|
|
|
function compactSentinelAnalysisWindow(value: unknown): Record<string, unknown> | null {
|
|
const item = record(value);
|
|
if (Object.keys(item).length === 0) return null;
|
|
return pickFields(item, ["name", "windowMs", "samples", "control", "network", "console", "valuesRedacted"]);
|
|
}
|
|
|
|
function reportText(value: unknown, maxChars: number): string | null {
|
|
if (value === undefined || value === null || value === "") return null;
|
|
const raw = text(value);
|
|
return raw.length <= maxChars ? raw : `${raw.slice(0, Math.max(0, maxChars - 1))}…`;
|
|
}
|
|
|
|
export function runSentinelDashboard(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): RenderedCliResult {
|
|
const command = `web-probe sentinel dashboard ${options.action}${options.runId === null ? "" : ` --run ${options.runId}`}`;
|
|
const result = probeSentinelDashboardBrowser(state, options);
|
|
return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result));
|
|
}
|
|
|
|
export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
|
const dashboardUrl = sentinelDashboardUrl(publicBaseUrl, options.runId);
|
|
const [widthRaw, heightRaw] = options.viewport.split("x");
|
|
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
|
|
const script = [
|
|
"set -eu",
|
|
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_TRIGGER=${shellQuote(options.action === "trigger" ? "1" : "0")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`,
|
|
`export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`,
|
|
"export PLAYWRIGHT_BROWSERS_PATH=0",
|
|
"if command -v chromium >/dev/null 2>&1; then",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)",
|
|
"elif command -v chromium-browser >/dev/null 2>&1; then",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)",
|
|
"elif command -v google-chrome >/dev/null 2>&1; then",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)",
|
|
"else",
|
|
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=",
|
|
"fi",
|
|
"cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'",
|
|
sentinelDashboardBrowserModule(),
|
|
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
|
|
"node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
|
|
].join("\n");
|
|
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
|
|
const job = runWebProbeRemoteArtifactJob({
|
|
route,
|
|
localDir: options.localDir,
|
|
waitTimeoutMs: options.waitTimeoutMs,
|
|
commandTimeoutMs: options.commandTimeoutSeconds * 1000,
|
|
inactivityTimeoutMs: 30000,
|
|
runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`,
|
|
stdoutTailBytes: 32768,
|
|
}, script);
|
|
const result = job.result;
|
|
const transport = record(job.transport);
|
|
const remote = record(transport.remote);
|
|
const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : "");
|
|
const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : [];
|
|
const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null;
|
|
const browserOk = page?.ok === true;
|
|
const screenshotOk = options.action !== "screenshot" || screenshot !== null && screenshot.verified === true;
|
|
const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk;
|
|
return {
|
|
ok,
|
|
status: ok ? "pass" : "blocked",
|
|
command: `web-probe sentinel dashboard ${options.action}`,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
sentinelId: state.sentinelId,
|
|
runId: options.runId,
|
|
publicUrl: `${publicBaseUrl}/`,
|
|
route,
|
|
viewport: options.viewport,
|
|
page,
|
|
screenshot,
|
|
artifacts,
|
|
artifactCount: artifacts.length,
|
|
remote: {
|
|
exitCode: remote.exitCode ?? null,
|
|
remoteDir: remote.remoteDir ?? null,
|
|
stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "",
|
|
stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "",
|
|
},
|
|
transport: {
|
|
ok: transport.ok ?? null,
|
|
runId: transport.runId ?? null,
|
|
artifactCount: transport.artifactCount ?? null,
|
|
expectedArtifactCount: transport.expectedArtifactCount ?? null,
|
|
downloadFailure: transport.downloadFailure ?? null,
|
|
pollFailure: transport.pollFailure ?? null,
|
|
},
|
|
result: compactCommand(result),
|
|
degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelDashboardUrl(publicBaseUrl: string, runId: string | null): string {
|
|
const url = new URL(`${publicBaseUrl.replace(/\/$/u, "")}/`);
|
|
if (runId !== null && runId.length > 0) {
|
|
url.searchParams.set("run", runId);
|
|
url.searchParams.set("focus", "memory");
|
|
}
|
|
return url.toString();
|
|
}
|
|
|
|
function sentinelDashboardBrowserModule(): string {
|
|
return String.raw`import { pathToFileURL } from "node:url";
|
|
|
|
const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || "";
|
|
const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright";
|
|
const { chromium } = await import(playwrightModuleSpecifier);
|
|
|
|
const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL;
|
|
const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || "";
|
|
const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1";
|
|
const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1";
|
|
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
|
|
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
|
|
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
|
|
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
|
|
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
|
|
const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || "";
|
|
const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/";
|
|
const requestedRunId = process.env.UNIDESK_SENTINEL_DASHBOARD_RUN_ID || "";
|
|
|
|
if (!url) throw new Error("missing dashboard URL");
|
|
|
|
const consoleMessages = [];
|
|
const pageErrors = [];
|
|
const requestFailures = [];
|
|
const browser = await chromium.launch({
|
|
headless: true,
|
|
args: ["--disable-gpu", "--no-sandbox"],
|
|
...(executablePath ? { executablePath } : {}),
|
|
});
|
|
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
|
|
const page = await context.newPage();
|
|
page.on("console", (message) => {
|
|
if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
|
|
});
|
|
page.on("pageerror", (error) => {
|
|
if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) });
|
|
});
|
|
page.on("requestfailed", (request) => {
|
|
if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null });
|
|
});
|
|
|
|
let httpStatus = null;
|
|
let navigationError = null;
|
|
let navigationAttempts = 0;
|
|
const maxNavigationAttempts = 2;
|
|
const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts));
|
|
for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) {
|
|
navigationAttempts = attempt;
|
|
navigationError = null;
|
|
try {
|
|
const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" });
|
|
httpStatus = response?.status() ?? null;
|
|
await page.waitForFunction(() => {
|
|
const root = document.querySelector("#monitor-web-root");
|
|
if (!root) return false;
|
|
const ready = root.getAttribute("data-monitor-ready") === "true";
|
|
const error = document.querySelector("#monitor-web-error");
|
|
return ready || Boolean(error);
|
|
}, null, { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {});
|
|
await page.waitForTimeout(300);
|
|
const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false);
|
|
if (appReady || attempt === maxNavigationAttempts) break;
|
|
} catch (error) {
|
|
navigationError = String(error?.message || error).slice(0, 500);
|
|
if (attempt === maxNavigationAttempts) break;
|
|
}
|
|
await page.waitForTimeout(750 * attempt);
|
|
}
|
|
|
|
let requestedRunSelection = { requestedRunId, ok: requestedRunId.length === 0, reason: requestedRunId.length === 0 ? "not-requested" : "not-attempted" };
|
|
if (requestedRunId) {
|
|
requestedRunSelection = await page.evaluate(async (runId) => {
|
|
const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms));
|
|
const rowForRun = () => document.querySelector('[data-run-id="' + CSS.escape(runId) + '"]');
|
|
const row = rowForRun();
|
|
if (row instanceof HTMLElement) row.click();
|
|
for (let index = 0; index < 80; index += 1) {
|
|
const selected = rowForRun();
|
|
if (selected instanceof HTMLElement) return { requestedRunId: runId, ok: true, reason: "run-row-present" };
|
|
await wait(150);
|
|
}
|
|
return { requestedRunId: runId, ok: false, reason: "run-row-missing" };
|
|
}, requestedRunId).catch((error) => ({ requestedRunId, ok: false, reason: "selection-error", error: String(error?.message || error).slice(0, 300) }));
|
|
await page.waitForTimeout(150);
|
|
}
|
|
|
|
let manualTrigger = { requested: triggerManual, ok: !triggerManual, status: triggerManual ? "not-attempted" : "not-requested", jobName: null, statusText: "" };
|
|
if (triggerManual) {
|
|
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
const buttonReady = await page.waitForSelector("[data-monitor-manual-trigger='true']", { timeout: Math.min(20000, timeout) }).then(() => true).catch(() => false);
|
|
if (buttonReady) break;
|
|
if (attempt === 1) {
|
|
await page.reload({ timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }).catch(() => null);
|
|
await page.waitForFunction(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true", null, { timeout: Math.min(12000, perAttemptTimeout) }).catch(() => {});
|
|
}
|
|
}
|
|
manualTrigger = await page.evaluate(async () => {
|
|
const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms));
|
|
const root = document.querySelector("[data-monitor-shell='true']");
|
|
const button = document.querySelector("[data-monitor-manual-trigger='true']");
|
|
if (!(button instanceof HTMLButtonElement)) return { requested: true, ok: false, status: "button-missing", jobName: null, statusText: "" };
|
|
if (button.disabled) return { requested: true, ok: false, status: "button-disabled", jobName: button.getAttribute("data-trigger-job") || null, statusText: "" };
|
|
button.click();
|
|
for (let index = 0; index < 160; index += 1) {
|
|
const state = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "";
|
|
const jobName = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null;
|
|
const statusText = String(document.querySelector("[data-monitor-manual-trigger-status='true']")?.textContent || "").replace(/\s+/g, " ").trim();
|
|
if (state === "triggered" || state === "already-active" || state === "failed") {
|
|
return { requested: true, ok: state === "triggered" || state === "already-active", status: state, jobName, statusText };
|
|
}
|
|
await wait(250);
|
|
}
|
|
const finalState = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "timeout";
|
|
const finalJob = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null;
|
|
return { requested: true, ok: false, status: finalState || "timeout", jobName: finalJob, statusText: "timeout" };
|
|
}).catch((error) => ({ requested: true, ok: false, status: "trigger-error", jobName: null, statusText: String(error?.message || error).slice(0, 300) }));
|
|
await page.waitForTimeout(300);
|
|
}
|
|
|
|
const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }) => {
|
|
const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0;
|
|
const objectOrNull = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim();
|
|
const root = document.querySelector("#monitor-web-root");
|
|
const shell = document.querySelector("[data-monitor-shell='true']");
|
|
const error = document.querySelector("#monitor-web-error");
|
|
const trend = document.querySelector("[data-monitor-trend-curve]");
|
|
const trendErrorCurve = document.querySelector("[data-monitor-trend-error-curve='true']");
|
|
const trendWarningCurve = document.querySelector("[data-monitor-trend-warning-curve='true']");
|
|
const memoryChart = document.querySelector("[data-run-memory-chart='true']");
|
|
const manualTriggerButton = document.querySelector("[data-monitor-manual-trigger='true']");
|
|
const manualTriggerStatus = document.querySelector("[data-monitor-manual-trigger-status='true']");
|
|
const chartTiming = {
|
|
hasDurationCurve: Boolean(trend),
|
|
hasErrorCurve: Boolean(trendErrorCurve),
|
|
hasWarningCurve: Boolean(trendWarningCurve),
|
|
ok: false,
|
|
};
|
|
chartTiming.ok = chartTiming.hasDurationCurve === true
|
|
&& chartTiming.hasErrorCurve === true
|
|
&& chartTiming.hasWarningCurve === true;
|
|
const dataset = root ? {
|
|
node: root.getAttribute("data-node"),
|
|
lane: root.getAttribute("data-lane"),
|
|
sentinelId: root.getAttribute("data-sentinel-id"),
|
|
basePath: root.getAttribute("data-base-path"),
|
|
contractVersion: root.getAttribute("data-contract-version"),
|
|
ready: root.getAttribute("data-monitor-ready"),
|
|
} : {};
|
|
const apiUrl = (path) => {
|
|
const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || "";
|
|
const prefix = basePath.replace(/\/+$/u, "");
|
|
return (prefix + (path.startsWith("/") ? path : "/" + path)) || path;
|
|
};
|
|
const fetchJson = async (path) => {
|
|
try {
|
|
const response = await fetch(apiUrl(path), { cache: "no-store" });
|
|
return {
|
|
ok: response.ok,
|
|
httpStatus: response.status,
|
|
body: await response.json().catch(() => null),
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
httpStatus: null,
|
|
body: null,
|
|
error: String(error?.message || error).slice(0, 300),
|
|
};
|
|
}
|
|
};
|
|
const [overviewResult, runsResult] = await Promise.all([
|
|
fetchJson("/api/overview"),
|
|
fetchJson("/api/runs?limit=30&sort=updated"),
|
|
]);
|
|
const overviewPayload = objectOrNull(overviewResult.body) || {};
|
|
const runsPayload = objectOrNull(runsResult.body) || {};
|
|
const runs = Array.isArray(runsPayload.runs) ? runsPayload.runs : Array.isArray(runsPayload.items) ? runsPayload.items : [];
|
|
const latestRun = runs[0] || null;
|
|
const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts)
|
|
? latestRun.severityCounts
|
|
: {};
|
|
const latestRunCounts = {
|
|
runId: latestRun?.id || latestRun?.runId || null,
|
|
typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count),
|
|
durationMinutes: numberValue(latestRun?.runDurationMinutes ?? latestRun?.durationMinutes ?? latestRun?.timing?.durationMinutes),
|
|
severityKeys: Object.keys(latestCounts).sort(),
|
|
};
|
|
const targetRunId = requestedRunId || latestRunCounts.runId;
|
|
const targetRun = targetRunId ? runs.find((run) => (run?.id || run?.runId) === targetRunId) || latestRun : latestRun;
|
|
const targetDetailResult = targetRunId === null ? { ok: false, httpStatus: null, body: null } : await fetchJson("/api/runs/" + encodeURIComponent(targetRunId));
|
|
const targetDetailPayload = objectOrNull(targetDetailResult.body) || {};
|
|
const targetMemory = objectOrNull(targetDetailPayload.memory) || {};
|
|
const targetPageSeries = Array.isArray(targetMemory.pageSeries) ? targetMemory.pageSeries : [];
|
|
const memorySummary = {
|
|
present: Boolean(memoryChart),
|
|
runId: memoryChart?.getAttribute("data-memory-run-id") || null,
|
|
targetRunId,
|
|
targetScenarioId: targetRun?.scenarioId || targetRun?.scenario_id || null,
|
|
matchesTargetRun: memoryChart?.getAttribute("data-memory-run-id") === targetRunId,
|
|
pageCount: numberValue(memoryChart?.getAttribute("data-memory-page-count")),
|
|
sampleCount: numberValue(memoryChart?.getAttribute("data-memory-sample-count")),
|
|
source: memoryChart?.getAttribute("data-memory-source") || null,
|
|
apiOk: targetDetailResult.ok === true && targetDetailPayload.ok !== false,
|
|
apiPageCount: targetPageSeries.length,
|
|
apiSampleCount: numberValue(targetMemory.sampleCount),
|
|
apiSource: targetMemory.source || null,
|
|
expectedFromApi: targetPageSeries.length > 0 || numberValue(targetMemory.sampleCount) > 0,
|
|
contractOk: false,
|
|
status: "unknown",
|
|
};
|
|
memorySummary.contractOk = memorySummary.apiOk === true
|
|
&& (memorySummary.expectedFromApi !== true || (
|
|
memorySummary.present === true
|
|
&& memorySummary.matchesTargetRun === true
|
|
&& memorySummary.pageCount === memorySummary.apiPageCount
|
|
));
|
|
memorySummary.status = memorySummary.apiOk !== true
|
|
? "api-unavailable"
|
|
: memorySummary.expectedFromApi === true
|
|
? memorySummary.contractOk === true ? "rendered" : "mismatch"
|
|
: "no-samples";
|
|
const datasetSentinelId = String(dataset.sentinelId || "");
|
|
const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/";
|
|
const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/";
|
|
const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/");
|
|
const sentinelBoundary = {
|
|
expectedSentinelId,
|
|
expectedRoutePrefix,
|
|
datasetSentinelId,
|
|
overviewSentinelId: overviewPayload.sentinelId || null,
|
|
runsSentinelId: runsPayload.sentinelId || null,
|
|
finalPath,
|
|
routePrefixMatches,
|
|
datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true,
|
|
overviewMatches: expectedSentinelId ? overviewPayload.sentinelId === expectedSentinelId : true,
|
|
runsPayloadMatches: expectedSentinelId ? runsPayload.sentinelId === expectedSentinelId : true,
|
|
runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true,
|
|
};
|
|
const doc = document.documentElement;
|
|
const body = document.body;
|
|
const viewport = { width: window.innerWidth, height: window.innerHeight };
|
|
const documentSize = {
|
|
width: Math.max(doc.scrollWidth, body?.scrollWidth || 0),
|
|
height: Math.max(doc.scrollHeight, body?.scrollHeight || 0),
|
|
};
|
|
const overflow = [];
|
|
let overflowCount = 0;
|
|
for (const element of Array.from(document.querySelectorAll("body *"))) {
|
|
const rect = element.getBoundingClientRect();
|
|
const overflowRight = rect.right - viewport.width;
|
|
const overflowLeft = -rect.left;
|
|
if (overflowRight > 1 || overflowLeft > 1) {
|
|
overflowCount += 1;
|
|
if (overflow.length < 5) {
|
|
overflow.push({
|
|
tag: element.tagName.toLowerCase(),
|
|
x: Math.round(rect.x),
|
|
y: Math.round(rect.y),
|
|
width: Math.round(rect.width),
|
|
height: Math.round(rect.height),
|
|
overflowRight: Math.max(0, Math.round(overflowRight)),
|
|
overflowLeft: Math.max(0, Math.round(overflowLeft)),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
shell: Boolean(root && shell),
|
|
ready: root?.getAttribute("data-monitor-ready") === "true",
|
|
dataset,
|
|
contract: {
|
|
htmlShell: Boolean(root && shell),
|
|
appReady: root?.getAttribute("data-monitor-ready") === "true",
|
|
apiOverview: overviewResult.ok === true && overviewPayload.ok !== false,
|
|
apiRuns: runsResult.ok === true && Array.isArray(runs),
|
|
runCount: runs.length,
|
|
latestRunId: latestRunCounts.runId,
|
|
latestFindingTypeCount: latestRunCounts.typeCount,
|
|
trendCurves: chartTiming.ok,
|
|
memoryContract: memorySummary.contractOk,
|
|
},
|
|
sentinelBoundary,
|
|
title: document.title,
|
|
finalUrl: window.location.href,
|
|
runRows: runs.length,
|
|
checkRows: document.querySelectorAll("[data-check-row='true']").length,
|
|
latestRunCounts,
|
|
targetRunCounts: {
|
|
runId: targetRunId,
|
|
scenarioId: targetRun?.scenarioId || targetRun?.scenario_id || null,
|
|
durationMinutes: numberValue(targetRun?.runDurationMinutes ?? targetRun?.durationMinutes ?? targetRun?.timing?.durationMinutes),
|
|
requested: Boolean(requestedRunId),
|
|
requestMatched: !requestedRunId || targetRunId === requestedRunId,
|
|
},
|
|
requestedRunSelection,
|
|
manualTriggerUi: {
|
|
requested: manualTrigger.requested,
|
|
ok: manualTrigger.ok,
|
|
status: manualTrigger.status,
|
|
jobName: manualTrigger.jobName,
|
|
buttonPresent: Boolean(manualTriggerButton),
|
|
buttonDisabled: manualTriggerButton instanceof HTMLButtonElement ? manualTriggerButton.disabled : null,
|
|
buttonState: manualTriggerButton?.getAttribute("data-trigger-state") || null,
|
|
statusText: String(manualTriggerStatus?.textContent || "").replace(/\s+/g, " ").trim(),
|
|
},
|
|
chartTiming,
|
|
memorySummary,
|
|
api: {
|
|
overview: { ok: overviewResult.ok, httpStatus: overviewResult.httpStatus },
|
|
runs: { ok: runsResult.ok, httpStatus: runsResult.httpStatus },
|
|
targetDetail: { ok: targetDetailResult.ok, httpStatus: targetDetailResult.httpStatus },
|
|
},
|
|
errorVisible: Boolean(error && !error.hidden),
|
|
errorText: error && !error.hidden ? String(error.textContent || "").replace(/\s+/g, " ").trim().slice(0, 500) : "",
|
|
layout: {
|
|
viewport,
|
|
documentSize,
|
|
horizontalOverflow: documentSize.width > viewport.width + 1,
|
|
overflowCount,
|
|
overflow: overflow.slice(0, 2),
|
|
},
|
|
};
|
|
}, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger });
|
|
|
|
if (captureScreenshot && screenshotPath) {
|
|
if (requestedRunId && dom.memorySummary?.present === true && dom.memorySummary?.matchesTargetRun === true) {
|
|
await page.evaluate(() => {
|
|
const chart = document.querySelector("[data-run-memory-chart='true']");
|
|
if (chart instanceof HTMLElement) chart.scrollIntoView({ block: "center", inline: "nearest", behavior: "auto" });
|
|
}).catch(() => null);
|
|
await page.waitForTimeout(150);
|
|
}
|
|
await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => {
|
|
pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) });
|
|
});
|
|
}
|
|
|
|
const consoleErrors = consoleMessages.filter((item) => item.type === "error");
|
|
const navigationOk = !navigationError || (dom.shell === true && dom.ready === true);
|
|
const ok = navigationOk
|
|
&& httpStatus !== null
|
|
&& httpStatus >= 200
|
|
&& httpStatus < 300
|
|
&& dom.shell === true
|
|
&& dom.contract?.apiOverview === true
|
|
&& dom.contract?.apiRuns === true
|
|
&& dom.sentinelBoundary?.datasetMatches === true
|
|
&& dom.sentinelBoundary?.overviewMatches === true
|
|
&& dom.sentinelBoundary?.runsPayloadMatches === true
|
|
&& dom.sentinelBoundary?.runRowsMatch === true
|
|
&& dom.sentinelBoundary?.routePrefixMatches === true
|
|
&& dom.errorVisible !== true
|
|
&& dom.requestedRunSelection?.ok === true
|
|
&& manualTrigger.ok === true
|
|
&& dom.chartTiming?.ok === true
|
|
&& dom.memorySummary?.contractOk === true
|
|
&& dom.layout?.horizontalOverflow !== true
|
|
&& pageErrors.length === 0;
|
|
|
|
console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
|
|
ok,
|
|
url,
|
|
httpStatus,
|
|
navigationError,
|
|
navigationAttempts,
|
|
executablePath: executablePath || null,
|
|
viewport: { width, height },
|
|
screenshotPath: captureScreenshot ? screenshotPath : null,
|
|
manualTrigger,
|
|
dom,
|
|
consoleCount: consoleMessages.length,
|
|
consoleErrorCount: consoleErrors.length,
|
|
pageErrorCount: pageErrors.length,
|
|
requestFailureCount: requestFailures.length,
|
|
consoleMessages: consoleMessages.slice(0, 8),
|
|
pageErrors: pageErrors.slice(0, 8),
|
|
requestFailures: requestFailures.slice(0, 8),
|
|
valuesRedacted: true,
|
|
}));
|
|
|
|
await context.close().catch(() => {});
|
|
await browser.close().catch(() => {});
|
|
`;
|
|
}
|
|
|
|
function parseDashboardBrowserPayload(textValue: string): Record<string, unknown> | null {
|
|
const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__";
|
|
const index = textValue.lastIndexOf(marker);
|
|
if (index < 0) return null;
|
|
try {
|
|
return record(JSON.parse(textValue.slice(index + marker.length).trim()));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function dashboardScreenshotName(options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>, state: SentinelCicdState): string {
|
|
const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`;
|
|
const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120);
|
|
return safe.endsWith(".png") ? safe : `${safe}.png`;
|
|
}
|
|
|
|
function compactDashboardArtifact(artifact: Record<string, unknown>): Record<string, unknown> {
|
|
const transfer = record(artifact.transfer);
|
|
return {
|
|
remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null,
|
|
localPath: typeof artifact.localPath === "string" ? artifact.localPath : null,
|
|
bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null,
|
|
sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null,
|
|
verified: artifact.verified === true,
|
|
transfer: Object.keys(transfer).length === 0 ? null : {
|
|
strategy: transfer.strategy ?? null,
|
|
transport: transfer.transport ?? null,
|
|
chunks: transfer.chunks ?? null,
|
|
elapsedMs: transfer.elapsedMs ?? null,
|
|
throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
function dashboardDegradedReason(result: CommandResult, transport: Record<string, unknown>, page: Record<string, unknown> | null, screenshotOk: boolean): string {
|
|
if (result.timedOut) return "sentinel-dashboard-command-timeout";
|
|
if (transport.ok !== true) {
|
|
const pollFailure = record(transport.pollFailure);
|
|
if (pollFailure.phase === "timeout") return "sentinel-dashboard-transport-timeout";
|
|
if (Object.keys(pollFailure).length > 0) return "sentinel-dashboard-transport-failed";
|
|
return "sentinel-dashboard-transport-failed";
|
|
}
|
|
if (page === null) return "sentinel-dashboard-browser-output-missing";
|
|
const manualTrigger = record(page.manualTrigger);
|
|
if (manualTrigger.requested === true && manualTrigger.ok !== true) return "sentinel-dashboard-manual-trigger-failed";
|
|
if (page.ok !== true) return "sentinel-dashboard-render-failed";
|
|
if (!screenshotOk) return "sentinel-dashboard-screenshot-missing";
|
|
return "sentinel-dashboard-unknown";
|
|
}
|
|
|
|
function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult {
|
|
const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
|
|
if (releaseId !== null) args.push("--release-id", releaseId);
|
|
if (reason !== null) args.push("--reason", reason);
|
|
if (quickVerify) args.push("--quick-verify");
|
|
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`);
|
|
const command = `web-probe sentinel ${subcommand.join(" ")}`;
|
|
return rendered(true, command, renderAsyncJobResult({
|
|
ok: true,
|
|
command,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
mode: "async-job",
|
|
mutation: true,
|
|
job,
|
|
next: {
|
|
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
|
|
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
|
|
},
|
|
valuesRedacted: true,
|
|
}));
|
|
}
|
|
|
|
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
|
|
const namespace = stringAt(state.runtime, "namespace");
|
|
const serviceName = stringAt(state.runtime, "serviceName");
|
|
const servicePort = numberAt(state.runtime, "servicePort");
|
|
const deploymentName = stringAt(state.runtime, "deploymentName");
|
|
const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`;
|
|
if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") {
|
|
return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url);
|
|
}
|
|
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
|
|
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
|
|
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
|
|
const postScript = [
|
|
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
|
|
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
|
|
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
|
|
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
|
|
" const text = await response.text();",
|
|
" process.stdout.write(text);",
|
|
" if (!response.ok) process.exit(22);",
|
|
"}).catch((error) => {",
|
|
" console.error(error && error.stack ? error.stack : String(error));",
|
|
" process.exit(23);",
|
|
"});",
|
|
].join(" ");
|
|
const script = method === "GET"
|
|
? `kubectl get --raw ${shellQuote(proxyPath)}`
|
|
: [
|
|
"set -eu",
|
|
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
|
|
].join("\n");
|
|
const maxAttempts = method === "GET" ? 3 : 1;
|
|
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
|
|
const attempts: Record<string, unknown>[] = [];
|
|
let result: CommandResult | null = null;
|
|
let parsed: Record<string, unknown> | null = null;
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 });
|
|
parsed = parseJsonObject(result.stdout);
|
|
const recovered = parsed === null ? recoverTruncatedSshStdoutJson(result) : null;
|
|
if (recovered !== null) parsed = recovered.parsed;
|
|
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, parsedFromDump: recovered !== null, dumpPath: recovered?.dumpPath ?? null, valuesRedacted: true });
|
|
if (result.exitCode === 0) break;
|
|
}
|
|
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
|
|
return {
|
|
ok: result?.exitCode === 0,
|
|
method,
|
|
path: pathWithQuery,
|
|
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
|
|
httpStatus: result?.exitCode === 0 ? 200 : null,
|
|
bodyJson: record(compactBodyJson),
|
|
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
|
|
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
|
|
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
|
|
proxyPath,
|
|
result: result === null ? null : compactCommand(result),
|
|
attempts,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function recoverTruncatedSshStdoutJson(result: CommandResult): { parsed: Record<string, unknown>; dumpPath: string } | null {
|
|
const dumpPath = sshStdoutDumpPathFromStderr(result.stderr);
|
|
if (dumpPath === null || !existsSync(dumpPath)) return null;
|
|
const parsed = parseJsonObject(readFileSync(dumpPath, "utf8"));
|
|
return parsed === null ? null : { parsed, dumpPath };
|
|
}
|
|
|
|
function sshStdoutDumpPathFromStderr(value: string): string | null {
|
|
for (const rawLine of value.split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
const prefix = "UNIDESK_SSH_STDOUT_TRUNCATED ";
|
|
if (!line.startsWith(prefix)) continue;
|
|
try {
|
|
const payload = JSON.parse(line.slice(prefix.length)) as unknown;
|
|
const dumpPath = stringAtNullable(record(payload), "dumpPath");
|
|
if (dumpPath !== null) return dumpPath;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number, url: string): Record<string, unknown> {
|
|
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
|
|
const fetchScript = [
|
|
"const method = process.env.SENTINEL_METHOD || 'GET';",
|
|
"const url = process.env.SENTINEL_URL || '';",
|
|
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
|
|
"const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);",
|
|
"const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);",
|
|
"const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;",
|
|
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
|
|
"(async () => {",
|
|
" let lastError = null;",
|
|
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
|
|
" try {",
|
|
" const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });",
|
|
" const text = await response.text();",
|
|
" process.stdout.write(text);",
|
|
" if (!response.ok) process.exit(22);",
|
|
" process.exit(0);",
|
|
" } catch (error) {",
|
|
" lastError = error;",
|
|
" console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));",
|
|
" if (attempt < attempts && delayMs > 0) await sleep(delayMs);",
|
|
" }",
|
|
" }",
|
|
" console.error(lastError && lastError.stack ? lastError.stack : String(lastError));",
|
|
" process.exit(23);",
|
|
"})().catch((error) => {",
|
|
" console.error(error && error.stack ? error.stack : String(error));",
|
|
" process.exit(24);",
|
|
"});",
|
|
].join(" ");
|
|
const attempts = method === "GET" ? 6 : 3;
|
|
const retryDelayMs = 1000;
|
|
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70));
|
|
const result = runCommand(["node", "-e", fetchScript], repoRoot, {
|
|
timeoutMs: attemptTimeoutSeconds * 1000,
|
|
env: {
|
|
...process.env,
|
|
SENTINEL_METHOD: method,
|
|
SENTINEL_URL: url,
|
|
SENTINEL_BODY_B64: bodyB64,
|
|
SENTINEL_ATTEMPTS: String(attempts),
|
|
SENTINEL_RETRY_DELAY_MS: String(retryDelayMs),
|
|
},
|
|
});
|
|
const parsed = parseJsonObject(result.stdout);
|
|
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
method,
|
|
path: pathWithQuery,
|
|
internalUrl: url,
|
|
httpStatus: result.exitCode === 0 ? 200 : null,
|
|
bodyJson: record(compactBodyJson),
|
|
bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "",
|
|
bodyBytes: Buffer.byteLength(result.stdout),
|
|
error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000),
|
|
proxyPath: null,
|
|
result: compactCommand(result),
|
|
attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }],
|
|
transport: "direct-service",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
|
|
if (value === null || typeof value.renderedText !== "string") return value;
|
|
return {
|
|
...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]),
|
|
run: pickFields(record(value.run), ["id", "runId", "scenario_id", "scenarioId", "status", "node", "lane", "observer_id", "observerId", "state_dir", "stateDir", "report_json_sha256", "reportJsonSha256", "finding_count", "findingCount", "artifact_count", "artifactCount", "maintenance", "created_at", "createdAt", "updated_at", "updatedAt", "startedAt", "finishedAt", "durationMs", "runDurationMs", "durationMinutes", "runDurationMinutes", "durationSource", "scenarioCadence", "scenarioCadenceMinutes", "scenarioMaxRunMinutes", "schedulerIntervalMinutes", "timing"]),
|
|
summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]),
|
|
findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [],
|
|
renderedText: value.renderedText,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function pickFields(value: Record<string, unknown>, keys: readonly string[]): Record<string, unknown> {
|
|
const picked: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key];
|
|
}
|
|
return picked;
|
|
}
|
|
|
|
function clipTail(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : value.slice(-maxChars);
|
|
}
|
|
|
|
function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
|
const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`;
|
|
const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000));
|
|
const js = [
|
|
"const url=process.env.REQ_URL||'';",
|
|
"const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);",
|
|
"let out;",
|
|
"try{",
|
|
" const controller=new AbortController();",
|
|
" const timer=setTimeout(()=>controller.abort(), timeoutMs);",
|
|
" const started=Date.now();",
|
|
" const res=await fetch(url,{signal:controller.signal});",
|
|
" const text=await res.text();",
|
|
" clearTimeout(timer);",
|
|
" let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}",
|
|
" out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};",
|
|
"}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}",
|
|
"console.log(JSON.stringify(out));",
|
|
].join("");
|
|
const result = runCommand(["bun", "-e", js], repoRoot, {
|
|
timeoutMs: timeoutMs + 2000,
|
|
env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) },
|
|
});
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return {
|
|
ok: result.exitCode === 0 && parsed?.ok === true,
|
|
method: "GET",
|
|
path: pathWithQuery,
|
|
publicUrl: url,
|
|
httpStatus: parsed?.httpStatus ?? null,
|
|
bodyJson: record(parsed?.bodyJson),
|
|
bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "",
|
|
bodyBytes: parsed?.bodyBytes ?? null,
|
|
error: parsed?.error ?? null,
|
|
result: compactCommand(result),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl");
|
|
const hostname = stringAt(state.publicExposure, "hostname");
|
|
const expectedA = stringAt(state.publicExposure, "expectedA");
|
|
const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`;
|
|
const script = [
|
|
"set +e",
|
|
`host=${shellQuote(hostname)}`,
|
|
`expected=${shellQuote(expectedA)}`,
|
|
`url=${shellQuote(probeUrl)}`,
|
|
"dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)",
|
|
"headers=$(mktemp)",
|
|
"body=$(mktemp)",
|
|
"writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)",
|
|
"curl_rc=$?",
|
|
"body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')",
|
|
"node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'",
|
|
"const fs=require('node:fs');",
|
|
"const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);",
|
|
"const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);",
|
|
"const status=Number(statusRaw||0);",
|
|
"const ssl=Number(sslRaw||-1);",
|
|
"const addrs=dns?dns.split(',').filter(Boolean):[];",
|
|
"const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();",
|
|
"const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');",
|
|
"let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}",
|
|
"const authCovered=status===401||status===403||status>=200&&status<300;",
|
|
"const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;",
|
|
"const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));",
|
|
"const dnsMatches=addrs.includes(expected);",
|
|
"console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
|
|
const rootUrl = `${publicBaseUrl}/`;
|
|
const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`;
|
|
const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`;
|
|
const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`;
|
|
const script = [
|
|
"set +e",
|
|
`root_url=${shellQuote(rootUrl)}`,
|
|
`css_url=${shellQuote(cssUrl)}`,
|
|
`js_url=${shellQuote(jsUrl)}`,
|
|
`vue_url=${shellQuote(vueUrl)}`,
|
|
"root_body=$(mktemp)",
|
|
"css_body=$(mktemp)",
|
|
"js_body=$(mktemp)",
|
|
"vue_body=$(mktemp)",
|
|
"root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?",
|
|
"css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?",
|
|
"js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?",
|
|
"vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?",
|
|
"node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'",
|
|
"const fs=require('node:fs');",
|
|
"const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);",
|
|
"function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}",
|
|
"const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);",
|
|
"const contractMatch=/data-contract-version=\"([^\"]+)\"/u.exec(root);",
|
|
"const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&Boolean(contractMatch);",
|
|
"const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.length>1000;",
|
|
"const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.length>1000;",
|
|
"const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.length>80000;",
|
|
"console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contractVersion:contractMatch?contractMatch[1]:null},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css)},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js)},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue)},valuesRedacted:true}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
|
|
const parsed = parseJsonObject(result.stdout);
|
|
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
function renderMaintenanceResult(result: Record<string, unknown>): string {
|
|
const serviceHealth = record(result.serviceHealth);
|
|
const maintenance = record(result.maintenance);
|
|
const quickVerify = record(result.quickVerify);
|
|
const quickVerifyBusiness = record(quickVerify.businessStatus);
|
|
const planned = record(result.planned);
|
|
const blocker = record(result.blocker);
|
|
const next = record(result.next);
|
|
const maintenanceBody = record(maintenance.bodyJson);
|
|
const state = record(maintenanceBody.maintenance);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]),
|
|
"",
|
|
table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]),
|
|
"",
|
|
Object.keys(state).length > 0
|
|
? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]])
|
|
: Object.keys(planned).length > 0
|
|
? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]])
|
|
: "MAINTENANCE\n-",
|
|
"",
|
|
Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
|
"",
|
|
"NEXT",
|
|
` validate: ${next.validate ?? "-"}`,
|
|
` report: ${next.report ?? "-"}`,
|
|
` maintenance-stop: ${next.maintenanceStop ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderValidateResult(result: Record<string, unknown>): string {
|
|
const health = record(result.serviceHealth);
|
|
const metrics = record(result.metrics);
|
|
const report = record(result.report);
|
|
const publicExposure = record(result.publicExposure);
|
|
const publicDashboard = record(result.publicDashboard);
|
|
const quickVerify = record(result.quickVerify);
|
|
const blocker = record(result.blocker);
|
|
const next = record(result.next);
|
|
const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]),
|
|
"",
|
|
table(["CHECK", "OK", "DETAIL"], [
|
|
["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`],
|
|
["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`],
|
|
["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`],
|
|
["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`],
|
|
["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`],
|
|
["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`],
|
|
]),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
|
"",
|
|
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]),
|
|
"",
|
|
"NEXT",
|
|
` quick-verify: ${next.quickVerify ?? "-"}`,
|
|
` report: ${next.report ?? "-"}`,
|
|
` maintenance-start: ${next.maintenanceStart ?? "-"}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderDashboardResult(result: Record<string, unknown>): string {
|
|
const page = record(result.page);
|
|
const dom = record(page.dom);
|
|
const dataset = record(dom.dataset);
|
|
const contract = record(dom.contract);
|
|
const sentinelBoundary = record(dom.sentinelBoundary);
|
|
const api = record(dom.api);
|
|
const apiOverview = record(api.overview);
|
|
const apiRuns = record(api.runs);
|
|
const layout = record(dom.layout);
|
|
const latestRunCounts = record(dom.latestRunCounts);
|
|
const targetRunCounts = record(dom.targetRunCounts);
|
|
const chartTiming = record(dom.chartTiming);
|
|
const memorySummary = record(dom.memorySummary);
|
|
const requestedRunSelection = record(dom.requestedRunSelection);
|
|
const manualTrigger = record(page.manualTrigger);
|
|
const manualTriggerUi = record(dom.manualTriggerUi);
|
|
const screenshot = record(result.screenshot);
|
|
const remote = record(result.remote);
|
|
const transport = record(result.transport);
|
|
const pollFailure = record(transport.pollFailure);
|
|
const degradedReason = result.degradedReason ?? null;
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]),
|
|
"",
|
|
table(["HTTP", "SHELL", "READY", "API_OVERVIEW", "API_RUNS", "RUN_ROWS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[
|
|
page.httpStatus ?? "-",
|
|
dom.shell,
|
|
dom.ready,
|
|
`${apiOverview.ok ?? "-"}/${apiOverview.httpStatus ?? "-"}`,
|
|
`${apiRuns.ok ?? "-"}/${apiRuns.httpStatus ?? "-"}`,
|
|
dom.runRows,
|
|
page.pageErrorCount,
|
|
page.consoleErrorCount,
|
|
page.requestFailureCount,
|
|
]]),
|
|
"",
|
|
table(["CONTRACT", "BASE_PATH", "DATASET_SENTINEL", "OVERVIEW_SENTINEL", "RUNS_SENTINEL", "ROUTE_MATCH"], [[
|
|
dataset.contractVersion ?? "-",
|
|
dataset.basePath ?? "-",
|
|
sentinelBoundary.datasetSentinelId ?? "-",
|
|
sentinelBoundary.overviewSentinelId ?? "-",
|
|
sentinelBoundary.runsSentinelId ?? "-",
|
|
sentinelBoundary.routePrefixMatches ?? "-",
|
|
]]),
|
|
"",
|
|
table(["HTML_SHELL", "APP_READY", "API_OVERVIEW_OK", "API_RUNS_OK", "LATEST_RUN", "TYPE_COUNT"], [[
|
|
contract.htmlShell ?? "-",
|
|
contract.appReady ?? "-",
|
|
contract.apiOverview ?? "-",
|
|
contract.apiRuns ?? "-",
|
|
latestRunCounts.runId ?? "-",
|
|
latestRunCounts.typeCount ?? "-",
|
|
]]),
|
|
"",
|
|
table(["TARGET_RUN", "REQUESTED", "SELECTED", "TREND_OK", "ERR_CURVE", "WARN_CURVE"], [[
|
|
targetRunCounts.runId ?? "-",
|
|
targetRunCounts.requested ?? "-",
|
|
requestedRunSelection.ok ?? "-",
|
|
chartTiming.ok ?? "-",
|
|
chartTiming.hasErrorCurve ?? "-",
|
|
chartTiming.hasWarningCurve ?? "-",
|
|
]]),
|
|
"",
|
|
manualTrigger.requested === true
|
|
? table(["TRIGGER", "STATUS", "JOB", "UI_STATE", "UI_TEXT"], [[
|
|
manualTrigger.ok ?? "-",
|
|
manualTrigger.status ?? "-",
|
|
manualTrigger.jobName ?? "-",
|
|
manualTriggerUi.buttonState ?? "-",
|
|
manualTriggerUi.statusText ?? "-",
|
|
]])
|
|
: "TRIGGER\n-",
|
|
"",
|
|
table(["MEMORY_CHART", "MEMORY_RUN", "MEMORY_MATCH", "MEMORY_PAGES", "MEMORY_SAMPLES", "API_PAGES", "API_SAMPLES", "CONTRACT", "STATUS", "MEMORY_SOURCE"], [[
|
|
memorySummary.present ?? "-",
|
|
memorySummary.runId ?? "-",
|
|
memorySummary.matchesTargetRun ?? "-",
|
|
memorySummary.pageCount ?? "-",
|
|
memorySummary.sampleCount ?? "-",
|
|
memorySummary.apiPageCount ?? "-",
|
|
memorySummary.apiSampleCount ?? "-",
|
|
memorySummary.contractOk ?? "-",
|
|
memorySummary.status ?? "-",
|
|
memorySummary.source ?? memorySummary.apiSource ?? "-",
|
|
]]),
|
|
"",
|
|
table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[
|
|
result.viewport,
|
|
`${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`,
|
|
layout.horizontalOverflow,
|
|
layout.overflowCount,
|
|
]]),
|
|
"",
|
|
Object.keys(screenshot).length === 0
|
|
? "SCREENSHOT\n-"
|
|
: table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]),
|
|
"",
|
|
degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT", "POLL_PHASE"], [[degradedReason, remote.exitCode, transport.ok, pollFailure.phase ?? "-"]]),
|
|
"",
|
|
"NEXT",
|
|
` trigger: bun scripts/cli.ts web-probe sentinel dashboard trigger --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
|
` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
|
` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
|
|
"",
|
|
"DISCLOSURE",
|
|
" dashboard verify uses YAML publicExposure, remote browser execution and API/data-contract checks; it does not assert UI copy or CSS class names.",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderReportResult(result: Record<string, unknown>): string {
|
|
const report = record(result.report);
|
|
const body = record(report.bodyJson);
|
|
const run = record(body.run);
|
|
return [
|
|
String(result.command),
|
|
"",
|
|
table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]),
|
|
"",
|
|
table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]),
|
|
"",
|
|
"DISCLOSURE",
|
|
" report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.",
|
|
].join("\n");
|
|
}
|