mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
88fa7db59f0b8d20cbbea033216802d6add45fd3
pikasTech-unidesk/scripts/src/hwlab-node-web-sentinel-p5.ts
T
Codex 88fa7db59f fix: bound sentinel report evidence output
2026-06-30 08:28:14 +00:00

1660 lines
90 KiB
TypeScript
Raw Blame History

// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
// Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands.
import type { CommandResult } from "./command";
import { runCommand } from "./command";
import { repoRoot } from "./config";
import { startJob } from "./jobs";
import type { RenderedCliResult } from "./output";
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
import {
clipTail,
compactCommand,
compactSentinelServiceBodyJson,
mergeWarnings,
numberAt,
parseJsonObject,
pickFields,
record,
rendered,
renderAsyncJobResult,
safeJobSegment,
sentinelCliSuffix,
shellQuote,
short,
stringAt,
stringAtNullable,
table,
targetValidationElapsedWarnings,
text,
withWarnings,
} from "./hwlab-node-web-sentinel-cicd";
import { metricNames, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
export function runSentinelMaintenance(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "maintenance" }>): RenderedCliResult {
const command = `web-probe sentinel maintenance ${options.action}`;
const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
if (options.action === "status") {
const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds);
const result = {
ok: serviceHealth.ok && maintenance.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
serviceHealth,
maintenance,
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderMaintenanceResult(result));
}
if (!options.confirm) {
const result = {
ok: serviceHealth.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "dry-run",
serviceHealth,
mutation: false,
planned: {
action: options.action,
releaseId: options.releaseId,
reason: options.reason,
quickVerify: options.action === "stop" && options.quickVerify,
},
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderMaintenanceResult(result));
}
if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify);
if (!serviceHealth.ok) {
const result = {
ok: false,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "confirm-wait",
mutation: false,
serviceHealth,
blocker: serviceUnavailableBlocker(state),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(false, command, renderMaintenanceResult(result));
}
const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true };
const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds);
const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok
? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds)
: null;
const result = {
ok: mutation.ok && (quickVerify === null || quickVerify.ok === true),
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "confirm-wait",
mutation: true,
serviceHealth,
maintenance: mutation,
quickVerify,
blocker: mutation.ok ? null : serviceUnavailableBlocker(state),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderMaintenanceResult(result));
}
export function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
const command = "web-probe sentinel validate";
const startedAt = Date.now();
const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
let quickVerify: Record<string, unknown> | null = null;
if (options.quickVerify) {
if (!options.confirm) {
const result = {
ok: initialHealth.ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "dry-run",
serviceHealth: initialHealth,
planned: { quickVerify: true, waitRequired: true },
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(result.ok, command, renderValidateResult(result));
}
if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true);
if (!initialHealth.ok) {
const result = {
ok: false,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "confirm-wait",
serviceHealth: initialHealth,
blocker: serviceUnavailableBlocker(state),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(false, command, renderValidateResult(result));
}
quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds);
}
const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20);
const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds);
const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds);
const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds);
const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health");
const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds);
const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds);
const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds);
const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health;
const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics;
const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report;
const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds);
const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds);
if (quickVerify !== null) {
quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds"))));
}
const publicFallbackWarnings = [
...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []),
...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []),
...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []),
];
const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health");
const ok = effectiveHealth.ok
&& record(effectiveHealth.bodyJson).ok === true
&& effectiveMetricsOk
&& effectiveReport.ok
&& publicExposure.ok === true
&& publicDashboard.ok === true
&& (quickVerify === null || quickVerify.ok === true);
const result = {
ok,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: options.quickVerify ? "confirm-wait" : "status",
serviceHealth: effectiveHealth,
metrics: effectiveMetrics,
report: effectiveReport,
internalServiceHealth: health,
internalMetrics: metrics,
internalReport: report,
publicServiceHealth: publicHealth,
publicMetrics,
publicReport,
publicExposure,
publicDashboard,
quickVerify,
warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []),
blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify),
next: sentinelP5Next(state),
valuesRedacted: true,
};
return rendered(ok, command, renderValidateResult(result));
}
export function runSentinelReport(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "report" }>): RenderedCliResult {
const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`;
const query = new URLSearchParams({ view: options.view });
if (options.runId !== null) query.set("run", options.runId);
if (options.traceId !== null) query.set("traceId", options.traceId);
if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq));
const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds);
const body = record(report.bodyJson);
const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true });
const rawPayload = Object.keys(body).length > 0 ? body : report;
if (options.full) return rendered(report.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2));
if (options.raw) {
const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, 55));
return rendered(report.ok && body.ok !== false, command, JSON.stringify(compactSentinelReportRawPayload(state, body, report, artifactSummary), null, 2));
}
return rendered(report.ok && body.ok !== false, command, renderedText);
}
function compactSentinelReportRawPayload(
state: SentinelCicdState,
body: Record<string, unknown>,
report: Record<string, unknown>,
artifactSummary: Record<string, unknown> | null,
): Record<string, unknown> {
const run = record(body.run);
const artifact = record(artifactSummary);
const findings = Array.isArray(body.findings) ? body.findings.map(record) : [];
const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : [];
const rootCauseSignalFindings = artifactFindings
.filter((item) => Object.keys(record(item.rootCauseSignals)).length > 0)
.slice(0, 8)
.map((item) => ({
id: stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"),
severity: stringAtNullable(item, "severity") ?? stringAtNullable(item, "level"),
summary: reportText(item.summary ?? item.message, 220),
rootCauseSignals: compactRootCauseSignals(item.rootCauseSignals),
valuesRedacted: true,
}));
return {
ok: body.ok !== false && report.ok !== false,
view: body.view ?? null,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
run: {
id: run.id ?? null,
scenarioId: run.scenario_id ?? run.scenarioId ?? null,
status: run.status ?? null,
observerId: run.observer_id ?? run.observerId ?? null,
stateDir: run.state_dir ?? run.stateDir ?? null,
reportJsonSha256: run.report_json_sha256 ?? run.reportJsonSha256 ?? artifact.reportJsonSha256 ?? null,
findingCount: run.finding_count ?? run.findingCount ?? findings.length,
artifactCount: run.artifact_count ?? run.artifactCount ?? artifact.artifactCount ?? null,
updatedAt: run.updated_at ?? run.updatedAt ?? null,
valuesRedacted: true,
},
summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]),
findings: findings.slice(0, 12).map(compactSentinelReportFinding),
artifactSummary: Object.keys(artifact).length === 0 ? null : {
ok: artifact.ok === true,
reportOk: artifact.reportOk === true ? true : artifact.reportOk === false ? false : null,
reportJsonPath: artifact.reportJsonPath ?? null,
reportJsonSha256: artifact.reportJsonSha256 ?? null,
reportMdSha256: artifact.reportMdSha256 ?? null,
screenshot: record(artifact.screenshot),
counts: record(artifact.counts),
analysisWindow: compactSentinelAnalysisWindow(artifact.analysisWindow),
pagePerformanceSlowApi: Array.isArray(artifact.pagePerformanceSlowApi) ? artifact.pagePerformanceSlowApi.slice(0, 8).map(record) : [],
rootCauseSignalFindings,
valuesRedacted: true,
},
next: {
text: "Default report is bounded text; use --full for the full indexed service payload.",
report: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")}`,
full: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")} --full`,
},
valuesRedacted: true,
};
}
function compactSentinelReportFinding(value: Record<string, unknown>): Record<string, unknown> {
const result: Record<string, unknown> = {
id: stringAtNullable(value, "finding_id") ?? stringAtNullable(value, "findingId") ?? stringAtNullable(value, "id") ?? stringAtNullable(value, "kind") ?? stringAtNullable(value, "code"),
severity: stringAtNullable(value, "severity") ?? stringAtNullable(value, "level"),
count: value.count ?? null,
summary: reportText(value.summary ?? value.message, 220),
valuesRedacted: true,
};
const rootCause = stringAtNullable(value, "rootCause");
if (rootCause !== null) result.rootCause = rootCause;
const rootCauseStatus = stringAtNullable(value, "rootCauseStatus");
if (rootCauseStatus !== null) result.rootCauseStatus = rootCauseStatus;
const rootCauseConfidence = stringAtNullable(value, "rootCauseConfidence");
if (rootCauseConfidence !== null) result.rootCauseConfidence = rootCauseConfidence;
const nextAction = reportText(value.nextAction, 240);
if (nextAction !== null) result.nextAction = nextAction;
const evidenceSummary = reportText(value.evidenceSummary, 240);
if (evidenceSummary !== null) result.evidenceSummary = evidenceSummary;
const timingSourceOfTruth = stringAtNullable(value, "timingSourceOfTruth");
if (timingSourceOfTruth !== null) result.timingSourceOfTruth = timingSourceOfTruth;
const timingStatus = stringAtNullable(value, "timingStatus");
if (timingStatus !== null) result.timingStatus = timingStatus;
if (value.timingAlert === true) result.timingAlert = true;
const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals);
if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals;
const check = record(value.check);
const checkCode = stringAtNullable(check, "code");
const checkTitle = stringAtNullable(check, "titleZh");
if (checkCode !== null || checkTitle !== null) {
result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]);
}
return result;
}
function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Record<string, unknown>, timeoutSeconds: number): Record<string, unknown> | null {
const run = record(body.run);
const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir");
if (stateDir === null || !isSafeSentinelReportStateDir(stateDir)) return null;
const script = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
"node - \"$state_dir\" <<'NODE'",
"const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');",
"const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');",
"const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);",
"const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;",
"const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);",
"const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};",
"let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}",
"let artifactCount=0; let screenshot=null;",
"function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}",
"walk(stateDir);",
"const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});",
"const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});",
"console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function isSafeSentinelReportStateDir(value: string): boolean {
return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..") && !value.startsWith("/");
}
function compactRootCauseSignals(value: unknown): Record<string, unknown> | null {
const item = record(value);
const keys = [
"sessionListReadCount",
"traceEventsReadCount",
"webPerformanceBeaconFailureCount",
"eventSourceFailureCount",
"requestFailedCount",
"httpErrorCount",
"consoleAlertCount",
"requestfailedTop",
"httpStatusTop",
];
const out: Record<string, unknown> = {};
for (const key of keys) {
const raw = item[key];
if (raw === null || raw === undefined) continue;
out[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
}
return Object.keys(out).length === 0 ? null : { ...out, valuesRedacted: true };
}
function compactSentinelAnalysisWindow(value: unknown): Record<string, unknown> | null {
const item = record(value);
if (Object.keys(item).length === 0) return null;
return pickFields(item, ["name", "windowMs", "samples", "control", "network", "console", "valuesRedacted"]);
}
function reportText(value: unknown, maxChars: number): string | null {
if (value === undefined || value === null || value === "") return null;
const raw = text(value);
return raw.length <= maxChars ? raw : `${raw.slice(0, Math.max(0, maxChars - 1))}…`;
}
export function runSentinelDashboard(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): RenderedCliResult {
const command = `web-probe sentinel dashboard ${options.action}`;
const result = probeSentinelDashboardBrowser(state, options);
return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result));
}
function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const [widthRaw, heightRaw] = options.viewport.split("x");
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
const script = [
"set -eu",
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(`${publicBaseUrl}/`)}`,
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
`export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`,
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`,
`export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`,
"export PLAYWRIGHT_BROWSERS_PATH=0",
"if command -v chromium >/dev/null 2>&1; then",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)",
"elif command -v chromium-browser >/dev/null 2>&1; then",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)",
"elif command -v google-chrome >/dev/null 2>&1; then",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)",
"else",
" export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=",
"fi",
"cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'",
sentinelDashboardBrowserModule(),
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
"node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
].join("\n");
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
const job = runWebProbeRemoteArtifactJob({
route,
localDir: options.localDir,
waitTimeoutMs: options.waitTimeoutMs,
commandTimeoutMs: options.commandTimeoutSeconds * 1000,
inactivityTimeoutMs: 30000,
runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`,
stdoutTailBytes: 32768,
}, script);
const result = job.result;
const transport = record(job.transport);
const remote = record(transport.remote);
const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : "");
const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : [];
const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null;
const browserOk = page?.ok === true;
const screenshotOk = options.action === "verify" || screenshot !== null && screenshot.verified === true;
const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk;
return {
ok,
status: ok ? "pass" : "blocked",
command: `web-probe sentinel dashboard ${options.action}`,
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
publicUrl: `${publicBaseUrl}/`,
route,
viewport: options.viewport,
page,
screenshot,
artifacts,
artifactCount: artifacts.length,
remote: {
exitCode: remote.exitCode ?? null,
remoteDir: remote.remoteDir ?? null,
stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "",
stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "",
},
transport: {
ok: transport.ok ?? null,
runId: transport.runId ?? null,
artifactCount: transport.artifactCount ?? null,
expectedArtifactCount: transport.expectedArtifactCount ?? null,
downloadFailure: transport.downloadFailure ?? null,
},
result: compactCommand(result),
degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk),
valuesRedacted: true,
};
}
function sentinelDashboardBrowserModule(): string {
return String.raw`import { pathToFileURL } from "node:url";
const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || "";
const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright";
const { chromium } = await import(playwrightModuleSpecifier);
const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL;
const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || "";
const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1";
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || "";
const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/";
if (!url) throw new Error("missing dashboard URL");
const consoleMessages = [];
const pageErrors = [];
const requestFailures = [];
const browser = await chromium.launch({
headless: true,
args: ["--disable-gpu", "--no-sandbox"],
...(executablePath ? { executablePath } : {}),
});
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
const page = await context.newPage();
page.on("console", (message) => {
if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
});
page.on("pageerror", (error) => {
if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) });
});
page.on("requestfailed", (request) => {
if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null });
});
let httpStatus = null;
let navigationError = null;
let navigationAttempts = 0;
const maxNavigationAttempts = 3;
const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts));
for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) {
navigationAttempts = attempt;
navigationError = null;
try {
const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" });
httpStatus = response?.status() ?? null;
await page.waitForLoadState("networkidle", { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {});
await page.waitForFunction(() => {
const root = document.querySelector("#monitor-web-root");
if (!root) return false;
const ready = root.getAttribute("data-monitor-ready") === "true";
const error = document.querySelector("#monitor-web-error");
const runs = document.querySelectorAll(".run-list .run-row").length;
const trend = document.querySelector("[data-monitor-trend-curve]");
return ready && (error || runs > 0 || trend);
}, null, { timeout: Math.min(15000, perAttemptTimeout) }).catch(() => {});
await page.waitForTimeout(500);
const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false);
if (appReady || attempt === maxNavigationAttempts) break;
} catch (error) {
navigationError = String(error?.message || error).slice(0, 500);
if (attempt === maxNavigationAttempts) break;
}
await page.waitForTimeout(750 * attempt);
}
await page.evaluate(() => {
const detailPane = document.querySelector(".workspace-grid .pane-detail");
if (detailPane instanceof HTMLElement) detailPane.scrollTop = Math.min(96, Math.max(0, detailPane.scrollHeight - detailPane.clientHeight));
}).catch(() => {});
await page.waitForTimeout(150);
const trendHoverPoint = await page.evaluate(() => {
const target = document.querySelector(".trend-dot-hit .trend-dot-red") || document.querySelector(".trend-dot-hit .trend-dot-warning");
if (!(target instanceof SVGElement)) return null;
const rect = target.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return null;
return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
}).catch(() => null);
if (trendHoverPoint) {
await page.mouse.move(trendHoverPoint.x, trendHoverPoint.y);
await page.waitForTimeout(250);
}
if (captureScreenshot && screenshotPath) {
await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => {
pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) });
});
}
const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId }) => {
const visible = (element) => Boolean(element && !element.hidden);
const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim();
const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0;
const errorSampleCount = (counts) => numberValue(counts?.red) + numberValue(counts?.critical) + numberValue(counts?.error);
const warningSampleCount = (counts) => numberValue(counts?.warning) + numberValue(counts?.warn) + numberValue(counts?.amber);
const allSampleCount = (counts) => Object.values(counts || {}).reduce((sum, value) => sum + numberValue(value), 0);
const root = document.querySelector("#monitor-web-root");
const shell = document.querySelector("[data-monitor-shell='true']");
const error = document.querySelector("#monitor-web-error");
const trend = document.querySelector("[data-monitor-trend-curve]");
const trendTooltip = document.querySelector("[data-monitor-trend-tooltip='true']");
const timeline = document.querySelector("[data-monitor-timeline='true']");
const workspace = document.querySelector("[data-monitor-independent-scroll='true']");
const checksPanel = document.querySelector("[data-monitor-checks='true']");
const panes = Array.from(document.querySelectorAll(".workspace-grid .pane, [data-monitor-checks='true']"));
const detailPane = document.querySelector(".workspace-grid .pane-detail");
const detailHeader = detailPane?.querySelector(".pane-header");
const checksHeader = checksPanel?.querySelector(".pane-header");
const internalTextPattern = /水合|投影|Trace|trace|Shell|API|DOM|Console|console|Runner|runner|JSONL|steer|facts|分页|HTTP|http|requestfailed|pageerror|Final Response|Code Agent|web-probe|observe|analyzer|终态/u;
const checkRows = Array.from(document.querySelectorAll("[data-check-row='true']"));
const cards = checkRows.slice(0, 8).map((row) => ({
code: String(row.querySelector(".check-code")?.textContent || "").trim(),
title: String(row.querySelector(".check-title-cell strong")?.textContent || row.querySelector("strong")?.textContent || "").trim(),
body: String(row.textContent || "").replace(/\s+/g, " ").trim().slice(0, 180),
}));
const badCardTitles = cards.filter((card) => internalTextPattern.test(card.title));
const badCardBodies = cards.filter((card) => internalTextPattern.test(card.body));
const legendTexts = Array.from(document.querySelectorAll(".trend-legend .legend-item")).map((item) => String(item.textContent || "").replace(/\s+/g, " ").trim());
const legendNumber = (label) => {
const row = legendTexts.find((item) => item.includes(label)) || "";
const match = /(\d+)/u.exec(row);
return match ? Number(match[1]) : null;
};
const chartCounts = {
error: legendNumber("最新点错误"),
warning: legendNumber("最新点告警"),
total: legendNumber("错误+告警合计"),
};
chartCounts.ok = typeof chartCounts.error === "number" && typeof chartCounts.warning === "number" && typeof chartCounts.total === "number"
? chartCounts.total === chartCounts.error + chartCounts.warning
: false;
const apiUrl = (path) => {
const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || "";
const prefix = basePath.replace(/\/+$/u, "");
return (prefix + (path.startsWith("/") ? path : "/" + path)) || path;
};
const runsPayload = await fetch(apiUrl("/api/runs?limit=30&sort=updated"), { cache: "no-store" }).then((item) => item.json()).catch(() => null);
const overviewPayload = await fetch(apiUrl("/api/overview"), { cache: "no-store" }).then((item) => item.json()).catch(() => null);
const runs = Array.isArray(runsPayload?.runs) ? runsPayload.runs : Array.isArray(runsPayload?.items) ? runsPayload.items : [];
const latestRun = runs[0] || null;
const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts)
? latestRun.severityCounts
: {};
const latestRunCounts = {
runId: latestRun?.id || latestRun?.runId || null,
typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count),
error: 0,
warning: 0,
total: 0,
all: 0,
errorSamples: errorSampleCount(latestCounts),
warningSamples: warningSampleCount(latestCounts),
alertSamples: errorSampleCount(latestCounts) + warningSampleCount(latestCounts),
allSamples: allSampleCount(latestCounts),
};
const latestDetailPayload = latestRunCounts.runId
? await fetch(apiUrl("/api/runs/" + encodeURIComponent(latestRunCounts.runId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null)
: null;
const latestDetailRows = Array.isArray(latestDetailPayload?.findings) ? latestDetailPayload.findings : [];
const rowSeverity = (row) => {
const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase();
if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red";
if (["warning", "warn", "amber"].includes(raw)) return "warning";
if (["info", "notice"].includes(raw)) return "info";
return "healthy";
};
const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1;
const summarizeRows = (rows) => {
const errorRows = rows.filter((row) => rowSeverity(row) === "red");
const warningRows = rows.filter((row) => rowSeverity(row) === "warning");
const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0);
return {
typeCount: rows.length,
errorTypeCount: errorRows.length,
warningTypeCount: warningRows.length,
alertTypeCount: errorRows.length + warningRows.length,
errorSamples: sum(errorRows),
warningSamples: sum(warningRows),
alertSamples: sum(errorRows) + sum(warningRows),
};
};
const latestDetailSummary = summarizeRows(latestDetailRows);
const expectsAlertRows = latestDetailSummary.alertTypeCount > 0;
latestRunCounts.typeCount = latestDetailSummary.typeCount;
latestRunCounts.error = latestDetailSummary.errorTypeCount;
latestRunCounts.warning = latestDetailSummary.warningTypeCount;
latestRunCounts.total = latestDetailSummary.alertTypeCount;
latestRunCounts.all = latestDetailSummary.typeCount;
const workspaceRect = workspace?.getBoundingClientRect();
const checksRect = checksPanel?.getBoundingClientRect();
const heightSummary = (rect) => {
const viewportHeight = window.innerHeight || 1;
const heightPx = rect ? Math.round(rect.height) : null;
const ratio = rect ? Math.round((rect.height / viewportHeight) * 1000) / 1000 : null;
return {
present: Boolean(rect),
heightPx,
ratio,
targetPx: Math.round(viewportHeight * 0.8),
bounded80: Boolean(rect && rect.height <= viewportHeight * 0.82 + 1),
near80: Boolean(rect && rect.height >= viewportHeight * 0.74 && rect.height <= viewportHeight * 0.82 + 1),
};
};
const workspacePaneHeights = Array.from(document.querySelectorAll(".workspace-grid > .pane")).map((pane) => heightSummary(pane.getBoundingClientRect()));
const stackedWorkspace = window.matchMedia("(max-width: 1120px)").matches;
const panelHeights = {
viewportHeight: window.innerHeight,
stackedWorkspace,
workspace: heightSummary(workspaceRect),
checks: heightSummary(checksRect),
workspacePanes: workspacePaneHeights,
workspacePaneBounded: workspacePaneHeights.length >= 2 && workspacePaneHeights.every((item) => item.bounded80 === true),
workspaceOk: false,
checksOk: false,
};
panelHeights.workspaceOk = stackedWorkspace
? panelHeights.workspacePaneBounded === true
: panelHeights.workspace.near80 === true && panelHeights.workspacePaneBounded === true;
panelHeights.checksOk = panelHeights.checks.near80 === true;
const checkScope = {
present: Boolean(checksPanel),
scope: checksPanel?.getAttribute("data-check-scope") || null,
runId: checksPanel?.getAttribute("data-check-run-id") || null,
typeCount: numberValue(checksPanel?.getAttribute("data-check-type-count")),
errorTypeCount: numberValue(checksPanel?.getAttribute("data-check-error-type-count")),
warningTypeCount: numberValue(checksPanel?.getAttribute("data-check-warning-type-count")),
alertTypeCount: numberValue(checksPanel?.getAttribute("data-check-alert-type-count")),
errorSamples: numberValue(checksPanel?.getAttribute("data-check-error-samples")),
warningSamples: numberValue(checksPanel?.getAttribute("data-check-warning-samples")),
alertSamples: numberValue(checksPanel?.getAttribute("data-check-alert-samples")),
visibleRowCount: document.querySelectorAll("[data-check-row='true']").length,
visibleAlertSamples: numberValue(checksPanel?.getAttribute("data-visible-check-alert-samples")),
expectsAlertRows,
matchesLatestRun: false,
matchesRunDetail: false,
belowWorkspace: Boolean(workspaceRect && checksRect && checksRect.top >= workspaceRect.bottom - 2),
fullWidth: Boolean(workspaceRect && checksRect && checksRect.width >= workspaceRect.width - 2),
};
const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row"))
.find((row) => row.getAttribute("data-run-id") === runId) || null;
const selectedRunRow = runListRowFor(latestRunCounts.runId) || document.querySelector(".run-list .run-row.selected");
const selectedRunTags = {
error: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]),
warning: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]),
errorVisible: Boolean(selectedRunRow?.querySelector("[data-run-error-tag='true']")),
warningVisible: Boolean(selectedRunRow?.querySelector("[data-run-warning-tag='true']")),
rowSelected: Boolean(selectedRunRow?.classList.contains("selected")),
matchesRunDetail: false,
};
selectedRunTags.matchesRunDetail = selectedRunTags.error === latestDetailSummary.errorTypeCount
&& selectedRunTags.warning === latestDetailSummary.warningTypeCount
&& selectedRunTags.errorVisible === (latestDetailSummary.errorTypeCount > 0)
&& selectedRunTags.warningVisible === (latestDetailSummary.warningTypeCount > 0)
&& selectedRunTags.rowSelected === true;
checkScope.matchesLatestRun = checkScope.present === true
&& checkScope.scope === "run"
&& checkScope.runId === latestRunCounts.runId
&& checkScope.errorTypeCount === latestRunCounts.error
&& checkScope.warningTypeCount === latestRunCounts.warning
&& checkScope.alertTypeCount === latestRunCounts.total;
checkScope.matchesRunDetail = checkScope.present === true
&& checkScope.typeCount === latestDetailSummary.typeCount
&& checkScope.errorTypeCount === latestDetailSummary.errorTypeCount
&& checkScope.warningTypeCount === latestDetailSummary.warningTypeCount
&& checkScope.alertTypeCount === latestDetailSummary.alertTypeCount
&& checkScope.errorSamples === latestDetailSummary.errorSamples
&& checkScope.warningSamples === latestDetailSummary.warningSamples
&& checkScope.alertSamples === latestDetailSummary.alertSamples
&& checkScope.visibleRowCount === latestDetailSummary.alertTypeCount
&& checkScope.visibleAlertSamples === latestDetailSummary.alertSamples;
const overviewCounts = overviewPayload?.severityCounts && typeof overviewPayload.severityCounts === "object" && !Array.isArray(overviewPayload.severityCounts)
? overviewPayload.severityCounts
: {};
const overviewSamples = {
error: errorSampleCount(overviewCounts),
warning: warningSampleCount(overviewCounts),
total: errorSampleCount(overviewCounts) + warningSampleCount(overviewCounts),
allSamples: allSampleCount(overviewCounts),
};
chartCounts.matchesLatestRun = chartCounts.ok === true
&& chartCounts.error === latestRunCounts.error
&& chartCounts.warning === latestRunCounts.warning
&& chartCounts.total === latestRunCounts.total;
const trendPanel = document.querySelector(".trend-panel");
const trendLegend = document.querySelector(".trend-panel .trend-legend");
const trendPanelRect = trendPanel?.getBoundingClientRect();
const trendLegendRect = trendLegend?.getBoundingClientRect();
const trendPanelCompact = {
present: Boolean(trendPanelRect && trendLegendRect),
bottomSlackPx: trendPanelRect && trendLegendRect ? Math.round(trendPanelRect.bottom - trendLegendRect.bottom) : null,
ok: Boolean(trendPanelRect && trendLegendRect && trendPanelRect.bottom - trendLegendRect.bottom <= 28),
};
const firstCheckRow = document.querySelector("[data-check-row='true']");
let checkDialog = { opened: false, title: "", width: null, height: null, large: false };
if (firstCheckRow instanceof HTMLElement) {
firstCheckRow.click();
await new Promise((resolve) => window.setTimeout(resolve, 80));
const dialog = document.querySelector("[data-check-dialog='true'] .check-dialog");
const rect = dialog?.getBoundingClientRect();
checkDialog = {
opened: Boolean(dialog),
title: String(dialog?.querySelector("#check-dialog-title")?.textContent || "").trim(),
width: rect ? Math.round(rect.width) : null,
height: rect ? Math.round(rect.height) : null,
large: Boolean(rect && rect.width >= Math.min(900, window.innerWidth * 0.7) && rect.height >= Math.min(460, window.innerHeight * 0.5)),
};
const close = dialog?.querySelector("button[aria-label='关闭监测项详情']");
if (close instanceof HTMLElement) close.click();
}
const datasetSentinelId = root?.getAttribute("data-sentinel-id") || "";
const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/";
const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/";
const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/");
const sentinelBoundary = {
expectedSentinelId,
expectedRoutePrefix,
datasetSentinelId,
overviewSentinelId: overviewPayload?.sentinelId || null,
runsSentinelId: runsPayload?.sentinelId || null,
finalPath,
routePrefixMatches,
datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true,
overviewMatches: expectedSentinelId ? overviewPayload?.sentinelId === expectedSentinelId : true,
runsPayloadMatches: expectedSentinelId ? runsPayload?.sentinelId === expectedSentinelId : true,
runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true,
};
const statusText = text(".status-strip");
const doc = document.documentElement;
const body = document.body;
const viewport = { width: window.innerWidth, height: window.innerHeight };
const documentSize = {
width: Math.max(doc.scrollWidth, body?.scrollWidth || 0),
height: Math.max(doc.scrollHeight, body?.scrollHeight || 0),
};
const overflow = [];
let overflowCount = 0;
for (const element of Array.from(document.querySelectorAll("body *"))) {
const rect = element.getBoundingClientRect();
const overflowRight = rect.right - viewport.width;
const overflowLeft = -rect.left;
if (overflowRight > 1 || overflowLeft > 1) {
overflowCount += 1;
if (overflow.length < 5) {
overflow.push({
tag: element.tagName.toLowerCase(),
className: String(element.className || "").slice(0, 40),
x: Math.round(rect.x),
y: Math.round(rect.y),
width: Math.round(rect.width),
height: Math.round(rect.height),
overflowRight: Math.max(0, Math.round(overflowRight)),
overflowLeft: Math.max(0, Math.round(overflowLeft)),
});
}
}
}
return {
shell: Boolean(root && shell),
ready: root?.getAttribute("data-monitor-ready") === "true",
dataset: root ? {
node: root.getAttribute("data-node"),
lane: root.getAttribute("data-lane"),
sentinelId: datasetSentinelId,
basePath: root.getAttribute("data-base-path"),
contractVersion: root.getAttribute("data-contract-version"),
} : {},
sentinelBoundary,
title: document.title,
finalUrl: window.location.href,
statusText: text(".topbar .pill"),
subtitle: text(".subtitle"),
summaryText: text(".status-strip"),
runRows: document.querySelectorAll(".run-list .run-row").length,
checkRows: document.querySelectorAll("[data-check-row='true']").length,
badCardTitleCount: badCardTitles.length,
badCardBodyCount: badCardBodies.length,
trendCurve: Boolean(trend),
trendDotCount: document.querySelectorAll(".trend-dot-hit").length,
trendTooltip: tooltipSummary(trendTooltip),
trendPanelText: text("#trend-heading"),
chartCounts,
latestRunCounts,
latestDetailSummary,
checkScope,
selectedRunTags,
trendPanelCompact,
checkDialog,
overviewSamples,
panelHeights,
scopeLabels: {
latestPointLegend: legendTexts.some((item) => item.includes("最新点")),
historicalSamples: legendTexts.some((item) => item.includes("历史样本累计")) || statusText.includes("历史错误样本"),
},
timelineItems: document.querySelectorAll(".timeline-list .timeline-item").length,
timelineVisible: Boolean(timeline),
errorVisible: visible(error),
errorText: visible(error) ? text("#monitor-web-error").slice(0, 500) : "",
scrollModel: {
workspace: Boolean(workspace),
paneCount: panes.length,
independentScroll: panes.length >= 3 && panes.every((pane) => {
const style = window.getComputedStyle(pane);
return style.overflowY === "auto" || style.overflowY === "scroll";
}),
stickyHeader: stickyHeaderSummary(detailPane, detailHeader),
stickyChecksHeader: stickyHeaderSummary(checksPanel, checksHeader),
},
layout: {
viewport,
documentSize,
horizontalOverflow: documentSize.width > viewport.width + 1,
overflowCount,
overflow: overflow.slice(0, 2),
},
};
function tooltipSummary(element) {
const body = String(element?.textContent || "").replace(/\s+/g, " ").trim();
return {
visible: Boolean(element && body.length > 0),
text: body.slice(0, 240),
hasValues: /错误\s+\d+/u.test(body) && /告警\s+\d+/u.test(body) && /合计\s+\d+/u.test(body),
hasTime: /UTC/u.test(body) || /\d{4}-\d{2}-\d{2}/u.test(body),
};
}
function stickyHeaderSummary(pane, header) {
if (!(pane instanceof HTMLElement) || !(header instanceof HTMLElement)) {
return { present: false, coversScroll: false, backgroundOpaque: false, detailScrollTop: null };
}
const rect = header.getBoundingClientRect();
const style = window.getComputedStyle(header);
const sampleX = Math.round(rect.left + Math.min(32, Math.max(2, rect.width / 2)));
const sampleY = Math.round(rect.top + Math.min(12, Math.max(2, rect.height / 2)));
const topElement = document.elementFromPoint(sampleX, sampleY);
return {
present: true,
detailScrollTop: pane.scrollTop,
headerTop: Math.round(rect.top),
headerBottom: Math.round(rect.bottom),
zIndex: style.zIndex,
backgroundColor: style.backgroundColor,
coversScroll: Boolean(topElement && header.contains(topElement)),
backgroundOpaque: backgroundIsOpaque(style.backgroundColor),
topElementClass: String(topElement?.className || "").slice(0, 80),
};
}
function backgroundIsOpaque(value) {
const rgba = /rgba?\(([^)]+)\)/u.exec(value);
if (rgba === null) return value.length > 0 && value !== "transparent";
const parts = rgba[1].split(",").map((part) => part.trim());
if (parts.length < 4) return true;
return Number(parts[3]) >= 0.99;
}
}, { expectedRoutePrefix, expectedSentinelId });
const runFilterProbe = await page.evaluate(async ({ expectedRoutePrefix }) => {
const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0;
const root = document.querySelector("#monitor-web-root");
const apiUrl = (path) => {
const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || "";
const prefix = basePath.replace(/\/+$/u, "");
return (prefix + (path.startsWith("/") ? path : "/" + path)) || path;
};
const rowSeverity = (row) => {
const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase();
if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red";
if (["warning", "warn", "amber"].includes(raw)) return "warning";
if (["info", "notice"].includes(raw)) return "info";
return "healthy";
};
const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1;
const summarizeRows = (rows) => {
const errorRows = rows.filter((row) => rowSeverity(row) === "red");
const warningRows = rows.filter((row) => rowSeverity(row) === "warning");
const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0);
return {
typeCount: rows.length,
errorTypeCount: errorRows.length,
warningTypeCount: warningRows.length,
alertTypeCount: errorRows.length + warningRows.length,
errorSamples: sum(errorRows),
warningSamples: sum(warningRows),
alertSamples: sum(errorRows) + sum(warningRows),
};
};
const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row"))
.find((row) => row.getAttribute("data-run-id") === runId) || null;
const panelCounts = () => {
const panel = document.querySelector("[data-monitor-checks='true']");
const panelRunId = panel?.getAttribute("data-check-run-id") || null;
const selectedRunRow = runListRowFor(panelRunId) || document.querySelector(".run-list .run-row.selected");
return {
present: Boolean(panel),
runId: panelRunId,
typeCount: numberValue(panel?.getAttribute("data-check-type-count")),
errorTypeCount: numberValue(panel?.getAttribute("data-check-error-type-count")),
warningTypeCount: numberValue(panel?.getAttribute("data-check-warning-type-count")),
alertTypeCount: numberValue(panel?.getAttribute("data-check-alert-type-count")),
errorSamples: numberValue(panel?.getAttribute("data-check-error-samples")),
warningSamples: numberValue(panel?.getAttribute("data-check-warning-samples")),
alertSamples: numberValue(panel?.getAttribute("data-check-alert-samples")),
visibleRowCount: document.querySelectorAll("[data-check-row='true']").length,
visibleAlertSamples: numberValue(panel?.getAttribute("data-visible-check-alert-samples")),
selectedRunErrorTag: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]),
selectedRunWarningTag: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]),
selectedRunRowSelected: Boolean(selectedRunRow?.classList.contains("selected")),
};
};
const waitForRun = async (runId) => {
for (let index = 0; index < 30; index += 1) {
const current = panelCounts();
if (current.runId === runId && current.present === true) return true;
await new Promise((resolve) => window.setTimeout(resolve, 100));
}
return false;
};
const select = document.querySelector("select[aria-label='选择运行记录']");
if (!(select instanceof HTMLSelectElement)) {
return { ok: false, requestedRunId: null, reason: "run-select-missing" };
}
const options = Array.from(select.options).filter((option) => option.value);
const requestedOption = options[Math.min(2, Math.max(0, options.length - 1))] || options[0] || null;
const requestedRunId = requestedOption?.value || null;
const fallbackOption = requestedOption || null;
const targetRunId = fallbackOption?.value || requestedRunId;
if (!targetRunId) return { ok: false, requestedRunId, reason: "run-option-missing" };
select.value = targetRunId;
select.dispatchEvent(new Event("change", { bubbles: true }));
const panelReady = await waitForRun(targetRunId);
const detailPayload = await fetch(apiUrl("/api/runs/" + encodeURIComponent(targetRunId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null);
const detailRows = Array.isArray(detailPayload?.findings) ? detailPayload.findings : [];
const expected = summarizeRows(detailRows);
const observed = panelCounts();
const matchesRunDetail = observed.runId === targetRunId
&& observed.typeCount === expected.typeCount
&& observed.errorTypeCount === expected.errorTypeCount
&& observed.warningTypeCount === expected.warningTypeCount
&& observed.alertTypeCount === expected.alertTypeCount
&& observed.errorSamples === expected.errorSamples
&& observed.warningSamples === expected.warningSamples
&& observed.alertSamples === expected.alertSamples
&& observed.visibleRowCount === expected.alertTypeCount
&& observed.visibleAlertSamples === expected.alertSamples
&& observed.selectedRunErrorTag === expected.errorTypeCount
&& observed.selectedRunWarningTag === expected.warningTypeCount
&& observed.selectedRunRowSelected === true;
return {
ok: panelReady === true && matchesRunDetail === true,
requestedRunId,
requestedOptionPresent: Boolean(requestedOption),
targetRunId,
panelReady,
observed,
expected,
matchesRunDetail,
};
}, { expectedRoutePrefix });
dom.runFilterProbe = runFilterProbe;
const consoleErrors = consoleMessages.filter((item) => item.type === "error");
const ok = !navigationError
&& httpStatus !== null
&& httpStatus >= 200
&& httpStatus < 300
&& dom.shell === true
&& dom.ready === true
&& dom.sentinelBoundary?.datasetMatches === true
&& dom.sentinelBoundary?.overviewMatches === true
&& dom.sentinelBoundary?.runsPayloadMatches === true
&& dom.sentinelBoundary?.runRowsMatch === true
&& dom.sentinelBoundary?.routePrefixMatches === true
&& dom.errorVisible !== true
&& dom.trendCurve === true
&& dom.chartCounts?.ok === true
&& dom.chartCounts?.matchesLatestRun === true
&& dom.checkScope?.matchesLatestRun === true
&& dom.checkScope?.matchesRunDetail === true
&& dom.selectedRunTags?.matchesRunDetail === true
&& dom.runFilterProbe?.ok === true
&& dom.runFilterProbe?.requestedOptionPresent === true
&& dom.checkScope?.belowWorkspace === true
&& dom.checkScope?.fullWidth === true
&& dom.scopeLabels?.latestPointLegend === true
&& dom.scopeLabels?.historicalSamples === true
&& (dom.trendDotCount === 0 || (dom.trendTooltip?.visible === true && dom.trendTooltip?.hasValues === true && dom.trendTooltip?.hasTime === true))
&& dom.trendPanelCompact?.ok === true
&& (dom.checkScope?.expectsAlertRows !== true || (dom.checkRows > 0 && dom.checkDialog?.opened === true && dom.checkDialog?.large === true))
&& dom.badCardTitleCount === 0
&& dom.badCardBodyCount === 0
&& dom.timelineVisible === true
&& dom.scrollModel?.independentScroll === true
&& dom.scrollModel?.stickyHeader?.present === true
&& dom.scrollModel?.stickyHeader?.backgroundOpaque === true
&& dom.scrollModel?.stickyChecksHeader?.present === true
&& dom.scrollModel?.stickyChecksHeader?.backgroundOpaque === true
&& dom.panelHeights?.workspaceOk === true
&& dom.panelHeights?.checksOk === true
&& dom.layout?.horizontalOverflow !== true
&& pageErrors.length === 0;
console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
ok,
url,
httpStatus,
navigationError,
navigationAttempts,
executablePath: executablePath || null,
viewport: { width, height },
screenshotPath: captureScreenshot ? screenshotPath : null,
dom,
consoleCount: consoleMessages.length,
consoleErrorCount: consoleErrors.length,
pageErrorCount: pageErrors.length,
requestFailureCount: requestFailures.length,
consoleMessages: consoleMessages.slice(0, 8),
pageErrors: pageErrors.slice(0, 8),
requestFailures: requestFailures.slice(0, 8),
valuesRedacted: true,
}));
await context.close().catch(() => {});
await browser.close().catch(() => {});
`;
}
function parseDashboardBrowserPayload(textValue: string): Record<string, unknown> | null {
const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__";
const index = textValue.lastIndexOf(marker);
if (index < 0) return null;
try {
return record(JSON.parse(textValue.slice(index + marker.length).trim()));
} catch {
return null;
}
}
function dashboardScreenshotName(options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>, state: SentinelCicdState): string {
const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`;
const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120);
return safe.endsWith(".png") ? safe : `${safe}.png`;
}
function compactDashboardArtifact(artifact: Record<string, unknown>): Record<string, unknown> {
const transfer = record(artifact.transfer);
return {
remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null,
localPath: typeof artifact.localPath === "string" ? artifact.localPath : null,
bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null,
sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null,
verified: artifact.verified === true,
transfer: Object.keys(transfer).length === 0 ? null : {
strategy: transfer.strategy ?? null,
transport: transfer.transport ?? null,
chunks: transfer.chunks ?? null,
elapsedMs: transfer.elapsedMs ?? null,
throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null,
},
};
}
function dashboardDegradedReason(result: CommandResult, transport: Record<string, unknown>, page: Record<string, unknown> | null, screenshotOk: boolean): string {
if (result.timedOut) return "sentinel-dashboard-command-timeout";
if (transport.ok !== true) return "sentinel-dashboard-transport-failed";
if (page === null) return "sentinel-dashboard-browser-output-missing";
if (page.ok !== true) return "sentinel-dashboard-render-failed";
if (!screenshotOk) return "sentinel-dashboard-screenshot-missing";
return "sentinel-dashboard-unknown";
}
function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult {
const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)];
if (releaseId !== null) args.push("--release-id", releaseId);
if (reason !== null) args.push("--reason", reason);
if (quickVerify) args.push("--quick-verify");
const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`);
const command = `web-probe sentinel ${subcommand.join(" ")}`;
return rendered(true, command, renderAsyncJobResult({
ok: true,
command,
node: state.spec.nodeId,
lane: state.spec.lane,
mode: "async-job",
mutation: true,
job,
next: {
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
},
valuesRedacted: true,
}));
}
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
const namespace = stringAt(state.runtime, "namespace");
const serviceName = stringAt(state.runtime, "serviceName");
const servicePort = numberAt(state.runtime, "servicePort");
const deploymentName = stringAt(state.runtime, "deploymentName");
const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`;
if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") {
return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url);
}
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
const postScript = [
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
"}).catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(23);",
"});",
].join(" ");
const script = method === "GET"
? `kubectl get --raw ${shellQuote(proxyPath)}`
: [
"set -eu",
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
].join("\n");
const maxAttempts = method === "GET" ? 3 : 1;
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
const attempts: Record<string, unknown>[] = [];
let result: CommandResult | null = null;
let parsed: Record<string, unknown> | null = null;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 });
parsed = parseJsonObject(result.stdout);
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true });
if (result.exitCode === 0) break;
}
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
return {
ok: result?.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
httpStatus: result?.exitCode === 0 ? 200 : null,
bodyJson: record(compactBodyJson),
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
proxyPath,
result: result === null ? null : compactCommand(result),
attempts,
valuesRedacted: true,
};
}
function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number, url: string): Record<string, unknown> {
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const fetchScript = [
"const method = process.env.SENTINEL_METHOD || 'GET';",
"const url = process.env.SENTINEL_URL || '';",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
"const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);",
"const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);",
"const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;",
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
"(async () => {",
" let lastError = null;",
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
" try {",
" const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
" process.exit(0);",
" } catch (error) {",
" lastError = error;",
" console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));",
" if (attempt < attempts && delayMs > 0) await sleep(delayMs);",
" }",
" }",
" console.error(lastError && lastError.stack ? lastError.stack : String(lastError));",
" process.exit(23);",
"})().catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(24);",
"});",
].join(" ");
const attempts = method === "GET" ? 6 : 3;
const retryDelayMs = 1000;
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70));
const result = runCommand(["node", "-e", fetchScript], repoRoot, {
timeoutMs: attemptTimeoutSeconds * 1000,
env: {
...process.env,
SENTINEL_METHOD: method,
SENTINEL_URL: url,
SENTINEL_BODY_B64: bodyB64,
SENTINEL_ATTEMPTS: String(attempts),
SENTINEL_RETRY_DELAY_MS: String(retryDelayMs),
},
});
const parsed = parseJsonObject(result.stdout);
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
return {
ok: result.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: url,
httpStatus: result.exitCode === 0 ? 200 : null,
bodyJson: record(compactBodyJson),
bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "",
bodyBytes: Buffer.byteLength(result.stdout),
error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000),
proxyPath: null,
result: compactCommand(result),
attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }],
transport: "direct-service",
valuesRedacted: true,
};
}
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
if (value === null || typeof value.renderedText !== "string") return value;
return {
...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]),
run: pickFields(record(value.run), ["id", "scenario_id", "status", "node", "lane", "observer_id", "state_dir", "report_json_sha256", "finding_count", "artifact_count", "maintenance", "created_at", "updated_at"]),
summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]),
findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [],
renderedText: value.renderedText,
valuesRedacted: true,
};
}
function pickFields(value: Record<string, unknown>, keys: readonly string[]): Record<string, unknown> {
const picked: Record<string, unknown> = {};
for (const key of keys) {
if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key];
}
return picked;
}
function clipTail(value: string, maxChars: number): string {
return value.length <= maxChars ? value : value.slice(-maxChars);
}
function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`;
const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000));
const js = [
"const url=process.env.REQ_URL||'';",
"const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);",
"let out;",
"try{",
" const controller=new AbortController();",
" const timer=setTimeout(()=>controller.abort(), timeoutMs);",
" const started=Date.now();",
" const res=await fetch(url,{signal:controller.signal});",
" const text=await res.text();",
" clearTimeout(timer);",
" let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}",
" out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};",
"}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}",
"console.log(JSON.stringify(out));",
].join("");
const result = runCommand(["bun", "-e", js], repoRoot, {
timeoutMs: timeoutMs + 2000,
env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) },
});
const parsed = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && parsed?.ok === true,
method: "GET",
path: pathWithQuery,
publicUrl: url,
httpStatus: parsed?.httpStatus ?? null,
bodyJson: record(parsed?.bodyJson),
bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "",
bodyBytes: parsed?.bodyBytes ?? null,
error: parsed?.error ?? null,
result: compactCommand(result),
valuesRedacted: true,
};
}
function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl");
const hostname = stringAt(state.publicExposure, "hostname");
const expectedA = stringAt(state.publicExposure, "expectedA");
const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`;
const script = [
"set +e",
`host=${shellQuote(hostname)}`,
`expected=${shellQuote(expectedA)}`,
`url=${shellQuote(probeUrl)}`,
"dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)",
"headers=$(mktemp)",
"body=$(mktemp)",
"writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)",
"curl_rc=$?",
"body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')",
"node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'",
"const fs=require('node:fs');",
"const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);",
"const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);",
"const status=Number(statusRaw||0);",
"const ssl=Number(sslRaw||-1);",
"const addrs=dns?dns.split(',').filter(Boolean):[];",
"const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();",
"const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');",
"let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}",
"const authCovered=status===401||status===403||status>=200&&status<300;",
"const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;",
"const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));",
"const dnsMatches=addrs.includes(expected);",
"console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const rootUrl = `${publicBaseUrl}/`;
const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`;
const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`;
const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`;
const script = [
"set +e",
`root_url=${shellQuote(rootUrl)}`,
`css_url=${shellQuote(cssUrl)}`,
`js_url=${shellQuote(jsUrl)}`,
`vue_url=${shellQuote(vueUrl)}`,
"root_body=$(mktemp)",
"css_body=$(mktemp)",
"js_body=$(mktemp)",
"vue_body=$(mktemp)",
"root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?",
"css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?",
"js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?",
"vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?",
"node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'",
"const fs=require('node:fs');",
"const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);",
"function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}",
"const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);",
"const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&root.includes('monitor-web-bootstrap');",
"const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.includes('monitor-shell')&&css.includes('workspace-grid')&&css.includes('trend-stage')&&css.length>1000;",
"const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.includes('createApp')&&js.includes('/api/overview')&&js.includes('data-monitor-trend-curve')&&js.length>1000;",
"const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.includes('createApp')&&vue.length>80000;",
"console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contract:root.includes('draft-2026-06-27-p11-monitor-web-observability-dashboard')},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css),workspaceGrid:css.includes('workspace-grid'),trendStage:css.includes('trend-stage')},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js),vueApp:js.includes('createApp'),apiClient:js.includes('/api/overview'),trend:js.includes('data-monitor-trend-curve')},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue),runtime:vue.includes('createApp')},valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function renderMaintenanceResult(result: Record<string, unknown>): string {
const serviceHealth = record(result.serviceHealth);
const maintenance = record(result.maintenance);
const quickVerify = record(result.quickVerify);
const quickVerifyBusiness = record(quickVerify.businessStatus);
const planned = record(result.planned);
const blocker = record(result.blocker);
const next = record(result.next);
const maintenanceBody = record(maintenance.bodyJson);
const state = record(maintenanceBody.maintenance);
return [
String(result.command),
"",
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]),
"",
table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]),
"",
Object.keys(state).length > 0
? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]])
: Object.keys(planned).length > 0
? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]])
: "MAINTENANCE\n-",
"",
Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]),
"",
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
"",
"NEXT",
` validate: ${next.validate ?? "-"}`,
` report: ${next.report ?? "-"}`,
` maintenance-stop: ${next.maintenanceStop ?? "-"}`,
"",
"DISCLOSURE",
" maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.",
].join("\n");
}
function renderValidateResult(result: Record<string, unknown>): string {
const health = record(result.serviceHealth);
const metrics = record(result.metrics);
const report = record(result.report);
const publicExposure = record(result.publicExposure);
const publicDashboard = record(result.publicDashboard);
const quickVerify = record(result.quickVerify);
const blocker = record(result.blocker);
const next = record(result.next);
const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []);
return [
String(result.command),
"",
table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]),
"",
table(["CHECK", "OK", "DETAIL"], [
["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`],
["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`],
["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`],
["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`],
["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`],
["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`],
]),
"",
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
"",
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]),
"",
"NEXT",
` quick-verify: ${next.quickVerify ?? "-"}`,
` report: ${next.report ?? "-"}`,
` maintenance-start: ${next.maintenanceStart ?? "-"}`,
"",
"DISCLOSURE",
" validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.",
].join("\n");
}
function renderDashboardResult(result: Record<string, unknown>): string {
const page = record(result.page);
const dom = record(page.dom);
const dataset = record(dom.dataset);
const layout = record(dom.layout);
const chartCounts = record(dom.chartCounts);
const latestRunCounts = record(dom.latestRunCounts);
const latestDetailSummary = record(dom.latestDetailSummary);
const checkScope = record(dom.checkScope);
const selectedRunTags = record(dom.selectedRunTags);
const trendPanelCompact = record(dom.trendPanelCompact);
const checkDialog = record(dom.checkDialog);
const runFilterProbe = record(dom.runFilterProbe);
const runFilterObserved = record(runFilterProbe.observed);
const runFilterExpected = record(runFilterProbe.expected);
const overviewSamples = record(dom.overviewSamples);
const panelHeights = record(dom.panelHeights);
const workspaceHeight = record(panelHeights.workspace);
const checksHeight = record(panelHeights.checks);
const screenshot = record(result.screenshot);
const remote = record(result.remote);
const transport = record(result.transport);
const degradedReason = result.degradedReason ?? null;
return [
String(result.command),
"",
table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]),
"",
table(["HTTP", "SHELL", "RUN_ROWS", "CHECK_ROWS", "TABS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[
page.httpStatus ?? "-",
dom.shell,
dom.runRows,
dom.checkRows,
dom.detailTabs,
page.pageErrorCount,
page.consoleErrorCount,
page.requestFailureCount,
]]),
"",
table(["TITLE", "STATUS_TEXT", "CONTRACT", "BASE_PATH"], [[dom.title, dom.statusText, dataset.contractVersion, dataset.basePath ?? "-"]]),
"",
table(["TREND_ERR_TYPES", "TREND_ALERT_TYPES", "TREND_TOTAL_TYPES", "TREND_EXACT", "MATCH_LATEST", "BAD_TITLE", "BAD_BODY"], [[
chartCounts.error ?? "-",
chartCounts.warning ?? "-",
chartCounts.total ?? "-",
chartCounts.ok ?? "-",
chartCounts.matchesLatestRun ?? "-",
dom.badCardTitleCount ?? "-",
dom.badCardBodyCount ?? "-",
]]),
"",
table(["LATEST_RUN", "TYPE_COUNT", "ERR_TYPES", "ALERT_TYPES", "TOTAL_TYPES", "SAMPLE_TOTAL", "HIST_ERR", "HIST_ALERT"], [[
latestRunCounts.runId ?? "-",
latestRunCounts.typeCount ?? "-",
latestRunCounts.error ?? "-",
latestRunCounts.warning ?? "-",
latestRunCounts.total ?? "-",
latestRunCounts.alertSamples ?? "-",
overviewSamples.error ?? "-",
overviewSamples.warning ?? "-",
]]),
"",
table(["CHECK_SCOPE", "CHECK_RUN", "CHECK_TYPES", "CHECK_ERR_TYPES", "CHECK_ALERT_TYPES", "SAMPLE_ERR", "SAMPLE_ALERT", "CHECK_MATCH_LATEST", "CHECK_MATCH_DETAIL"], [[
checkScope.scope ?? "-",
checkScope.runId ?? "-",
`${checkScope.typeCount ?? "-"}/${latestDetailSummary.typeCount ?? "-"}`,
`${checkScope.errorTypeCount ?? "-"}/${latestDetailSummary.errorTypeCount ?? "-"}`,
`${checkScope.alertTypeCount ?? "-"}/${latestDetailSummary.alertTypeCount ?? "-"}`,
checkScope.errorSamples ?? "-",
checkScope.alertSamples ?? "-",
checkScope.matchesLatestRun ?? "-",
checkScope.matchesRunDetail ?? "-",
]]),
"",
table(["CHECK_VISIBLE_ROWS", "EXPECT_ALERT_ROWS", "CHECK_VISIBLE_ALERT", "RUN_TAG_ERR", "RUN_TAG_ALERT", "RUN_TAG_MATCH", "BELOW_WORKSPACE", "FULL_WIDTH"], [[
checkScope.visibleRowCount ?? "-",
checkScope.expectsAlertRows ?? "-",
checkScope.visibleAlertSamples ?? "-",
selectedRunTags.error ?? "-",
selectedRunTags.warning ?? "-",
selectedRunTags.matchesRunDetail ?? "-",
checkScope.belowWorkspace ?? "-",
checkScope.fullWidth ?? "-",
]]),
"",
table(["TREND_PANEL_SLACK", "TREND_PANEL_COMPACT", "DETAIL_DIALOG", "DIALOG_LARGE"], [[
trendPanelCompact.bottomSlackPx ?? "-",
trendPanelCompact.ok ?? "-",
checkDialog.opened ?? "-",
checkDialog.large ?? "-",
]]),
"",
table(["WORKSPACE_H", "WORKSPACE_RATIO", "WORKSPACE_80", "CHECKS_H", "CHECKS_RATIO", "CHECKS_80", "PANES_80"], [[
`${workspaceHeight.heightPx ?? "-"}/${workspaceHeight.targetPx ?? "-"}`,
workspaceHeight.ratio ?? "-",
panelHeights.workspaceOk ?? "-",
`${checksHeight.heightPx ?? "-"}/${checksHeight.targetPx ?? "-"}`,
checksHeight.ratio ?? "-",
panelHeights.checksOk ?? "-",
panelHeights.workspacePaneBounded ?? "-",
]]),
"",
table(["FILTER_RUN", "FILTER_OPTION", "FILTER_TYPES", "FILTER_ERR_TYPES", "FILTER_ALERT_TYPES", "FILTER_SAMPLE_ERR", "FILTER_SAMPLE_ALERT", "FILTER_MATCH_DETAIL"], [[
runFilterProbe.targetRunId ?? "-",
runFilterProbe.requestedOptionPresent ?? "-",
`${runFilterObserved.typeCount ?? "-"}/${runFilterExpected.typeCount ?? "-"}`,
`${runFilterObserved.errorTypeCount ?? "-"}/${runFilterExpected.errorTypeCount ?? "-"}`,
`${runFilterObserved.alertTypeCount ?? "-"}/${runFilterExpected.alertTypeCount ?? "-"}`,
runFilterObserved.errorSamples ?? "-",
runFilterObserved.alertSamples ?? "-",
runFilterProbe.matchesRunDetail ?? "-",
]]),
"",
table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[
result.viewport,
`${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`,
layout.horizontalOverflow,
layout.overflowCount,
]]),
"",
Object.keys(screenshot).length === 0
? "SCREENSHOT\n-"
: table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]),
"",
degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT"], [[degradedReason, remote.exitCode, transport.ok]]),
"",
"NEXT",
` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`,
"",
"DISCLOSURE",
" dashboard verify uses the YAML publicExposure URL and remote browser execution; it does not start a sentinel run or inspect provider payloads.",
].join("\n");
}
function renderReportResult(result: Record<string, unknown>): string {
const report = record(result.report);
const body = record(report.bodyJson);
const run = record(body.run);
return [
String(result.command),
"",
table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]),
"",
table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]),
"",
"DISCLOSURE",
" report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.",
].join("\n");
}
Reference in New Issue View Git Blame Copy Permalink