// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands. import type { CommandResult } from "./command"; import { runCommand } from "./command"; import { repoRoot } from "./config"; import { startJob } from "./jobs"; import type { RenderedCliResult } from "./output"; import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact"; import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd"; import { clipTail, compactCommand, compactSentinelServiceBodyJson, mergeWarnings, numberAt, parseJsonObject, pickFields, record, rendered, renderAsyncJobResult, safeJobSegment, sentinelCliSuffix, shellQuote, short, stringAt, stringAtNullable, table, targetValidationElapsedWarnings, text, withWarnings, } from "./hwlab-node-web-sentinel-cicd"; import { metricNames, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe"; export function runSentinelMaintenance(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel maintenance ${options.action}`; const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); if (options.action === "status") { const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds); const result = { ok: serviceHealth.ok && maintenance.ok, command, node: state.spec.nodeId, lane: state.spec.lane, serviceHealth, maintenance, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } if (!options.confirm) { const result = { ok: serviceHealth.ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "dry-run", serviceHealth, mutation: false, planned: { action: options.action, releaseId: options.releaseId, reason: options.reason, quickVerify: options.action === "stop" && options.quickVerify, }, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); if (!serviceHealth.ok) { const result = { ok: false, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: false, serviceHealth, blocker: serviceUnavailableBlocker(state), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(false, command, renderMaintenanceResult(result)); } const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true }; const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds); const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok ? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds) : null; const result = { ok: mutation.ok && (quickVerify === null || quickVerify.ok === true), command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: true, serviceHealth, maintenance: mutation, quickVerify, blocker: mutation.ok ? null : serviceUnavailableBlocker(state), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } export function runSentinelValidate(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = "web-probe sentinel validate"; const startedAt = Date.now(); const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); let quickVerify: Record | null = null; if (options.quickVerify) { if (!options.confirm) { const result = { ok: initialHealth.ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "dry-run", serviceHealth: initialHealth, planned: { quickVerify: true, waitRequired: true }, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderValidateResult(result)); } if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true); if (!initialHealth.ok) { const result = { ok: false, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", serviceHealth: initialHealth, blocker: serviceUnavailableBlocker(state), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(false, command, renderValidateResult(result)); } quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds); } const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20); const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds); const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds); const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds); const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health"); const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds); const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds); const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds); const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health; const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics; const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report; const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds); const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds); if (quickVerify !== null) { quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds")))); } const publicFallbackWarnings = [ ...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []), ...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []), ...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []), ]; const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health"); const ok = effectiveHealth.ok && record(effectiveHealth.bodyJson).ok === true && effectiveMetricsOk && effectiveReport.ok && publicExposure.ok === true && publicDashboard.ok === true && (quickVerify === null || quickVerify.ok === true); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: options.quickVerify ? "confirm-wait" : "status", serviceHealth: effectiveHealth, metrics: effectiveMetrics, report: effectiveReport, internalServiceHealth: health, internalMetrics: metrics, internalReport: report, publicServiceHealth: publicHealth, publicMetrics, publicReport, publicExposure, publicDashboard, quickVerify, warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []), blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(ok, command, renderValidateResult(result)); } export function runSentinelReport(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`; const query = new URLSearchParams({ view: options.view }); if (options.runId !== null) query.set("run", options.runId); if (options.traceId !== null) query.set("traceId", options.traceId); if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq)); const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds); const body = record(report.bodyJson); const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true }); const rawPayload = Object.keys(body).length > 0 ? body : report; if (options.full) return rendered(report.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2)); if (options.raw) { const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, 55)); return rendered(report.ok && body.ok !== false, command, JSON.stringify(compactSentinelReportRawPayload(state, body, report, artifactSummary), null, 2)); } return rendered(report.ok && body.ok !== false, command, renderedText); } function compactSentinelReportRawPayload( state: SentinelCicdState, body: Record, report: Record, artifactSummary: Record | null, ): Record { const run = record(body.run); const artifact = record(artifactSummary); const findings = Array.isArray(body.findings) ? body.findings.map(record) : []; const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : []; const rootCauseSignalFindings = artifactFindings .filter((item) => Object.keys(record(item.rootCauseSignals)).length > 0) .slice(0, 8) .map((item) => ({ id: stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"), severity: stringAtNullable(item, "severity") ?? stringAtNullable(item, "level"), summary: reportText(item.summary ?? item.message, 220), rootCauseSignals: compactRootCauseSignals(item.rootCauseSignals), valuesRedacted: true, })); return { ok: body.ok !== false && report.ok !== false, view: body.view ?? null, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, run: { id: run.id ?? null, scenarioId: run.scenario_id ?? run.scenarioId ?? null, status: run.status ?? null, observerId: run.observer_id ?? run.observerId ?? null, stateDir: run.state_dir ?? run.stateDir ?? null, reportJsonSha256: run.report_json_sha256 ?? run.reportJsonSha256 ?? artifact.reportJsonSha256 ?? null, findingCount: run.finding_count ?? run.findingCount ?? findings.length, artifactCount: run.artifact_count ?? run.artifactCount ?? artifact.artifactCount ?? null, updatedAt: run.updated_at ?? run.updatedAt ?? null, valuesRedacted: true, }, summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]), findings: findings.slice(0, 12).map(compactSentinelReportFinding), artifactSummary: Object.keys(artifact).length === 0 ? null : { ok: artifact.ok === true, reportOk: artifact.reportOk === true ? true : artifact.reportOk === false ? false : null, reportJsonPath: artifact.reportJsonPath ?? null, reportJsonSha256: artifact.reportJsonSha256 ?? null, reportMdSha256: artifact.reportMdSha256 ?? null, screenshot: record(artifact.screenshot), counts: record(artifact.counts), analysisWindow: compactSentinelAnalysisWindow(artifact.analysisWindow), pagePerformanceSlowApi: Array.isArray(artifact.pagePerformanceSlowApi) ? artifact.pagePerformanceSlowApi.slice(0, 8).map(record) : [], rootCauseSignalFindings, valuesRedacted: true, }, next: { text: "Default report is bounded text; use --full for the full indexed service payload.", report: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")}`, full: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")} --full`, }, valuesRedacted: true, }; } function compactSentinelReportFinding(value: Record): Record { const result: Record = { id: stringAtNullable(value, "finding_id") ?? stringAtNullable(value, "findingId") ?? stringAtNullable(value, "id") ?? stringAtNullable(value, "kind") ?? stringAtNullable(value, "code"), severity: stringAtNullable(value, "severity") ?? stringAtNullable(value, "level"), count: value.count ?? null, summary: reportText(value.summary ?? value.message, 220), valuesRedacted: true, }; const rootCause = stringAtNullable(value, "rootCause"); if (rootCause !== null) result.rootCause = rootCause; const rootCauseStatus = stringAtNullable(value, "rootCauseStatus"); if (rootCauseStatus !== null) result.rootCauseStatus = rootCauseStatus; const rootCauseConfidence = stringAtNullable(value, "rootCauseConfidence"); if (rootCauseConfidence !== null) result.rootCauseConfidence = rootCauseConfidence; const nextAction = reportText(value.nextAction, 240); if (nextAction !== null) result.nextAction = nextAction; const evidenceSummary = reportText(value.evidenceSummary, 240); if (evidenceSummary !== null) result.evidenceSummary = evidenceSummary; const timingSourceOfTruth = stringAtNullable(value, "timingSourceOfTruth"); if (timingSourceOfTruth !== null) result.timingSourceOfTruth = timingSourceOfTruth; const timingStatus = stringAtNullable(value, "timingStatus"); if (timingStatus !== null) result.timingStatus = timingStatus; if (value.timingAlert === true) result.timingAlert = true; const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals); if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals; const check = record(value.check); const checkCode = stringAtNullable(check, "code"); const checkTitle = stringAtNullable(check, "titleZh"); if (checkCode !== null || checkTitle !== null) { result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]); } return result; } function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Record, timeoutSeconds: number): Record | null { const run = record(body.run); const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir"); if (stateDir === null || !isSafeSentinelReportStateDir(stateDir)) return null; const script = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, "node - \"$state_dir\" <<'NODE'", "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", "let artifactCount=0; let screenshot=null;", "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", "walk(stateDir);", "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", "console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function isSafeSentinelReportStateDir(value: string): boolean { return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..") && !value.startsWith("/"); } function compactRootCauseSignals(value: unknown): Record | null { const item = record(value); const keys = [ "sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "consoleAlertCount", "requestfailedTop", "httpStatusTop", ]; const out: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; out[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw; } return Object.keys(out).length === 0 ? null : { ...out, valuesRedacted: true }; } function compactSentinelAnalysisWindow(value: unknown): Record | null { const item = record(value); if (Object.keys(item).length === 0) return null; return pickFields(item, ["name", "windowMs", "samples", "control", "network", "console", "valuesRedacted"]); } function reportText(value: unknown, maxChars: number): string | null { if (value === undefined || value === null || value === "") return null; const raw = text(value); return raw.length <= maxChars ? raw : `${raw.slice(0, Math.max(0, maxChars - 1))}…`; } export function runSentinelDashboard(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel dashboard ${options.action}`; const result = probeSentinelDashboardBrowser(state, options); return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result)); } function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const [widthRaw, heightRaw] = options.viewport.split("x"); const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : ""; const script = [ "set -eu", `export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(`${publicBaseUrl}/`)}`, `export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`, `export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`, `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`, `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`, `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`, `export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`, "export PLAYWRIGHT_BROWSERS_PATH=0", "if command -v chromium >/dev/null 2>&1; then", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)", "elif command -v chromium-browser >/dev/null 2>&1; then", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)", "elif command -v google-chrome >/dev/null 2>&1; then", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)", "else", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=", "fi", "cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'", sentinelDashboardBrowserModule(), "WEB_PROBE_SENTINEL_DASHBOARD_JS", "node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"", ].join("\n"); const route = `${state.spec.nodeId}:${state.spec.workspace}`; const job = runWebProbeRemoteArtifactJob({ route, localDir: options.localDir, waitTimeoutMs: options.waitTimeoutMs, commandTimeoutMs: options.commandTimeoutSeconds * 1000, inactivityTimeoutMs: 30000, runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`, stdoutTailBytes: 32768, }, script); const result = job.result; const transport = record(job.transport); const remote = record(transport.remote); const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : ""); const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : []; const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null; const browserOk = page?.ok === true; const screenshotOk = options.action === "verify" || screenshot !== null && screenshot.verified === true; const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk; return { ok, status: ok ? "pass" : "blocked", command: `web-probe sentinel dashboard ${options.action}`, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, publicUrl: `${publicBaseUrl}/`, route, viewport: options.viewport, page, screenshot, artifacts, artifactCount: artifacts.length, remote: { exitCode: remote.exitCode ?? null, remoteDir: remote.remoteDir ?? null, stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "", stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "", }, transport: { ok: transport.ok ?? null, runId: transport.runId ?? null, artifactCount: transport.artifactCount ?? null, expectedArtifactCount: transport.expectedArtifactCount ?? null, downloadFailure: transport.downloadFailure ?? null, }, result: compactCommand(result), degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk), valuesRedacted: true, }; } function sentinelDashboardBrowserModule(): string { return String.raw`import { pathToFileURL } from "node:url"; const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || ""; const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright"; const { chromium } = await import(playwrightModuleSpecifier); const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL; const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || ""; const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1"; const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440); const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900); const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000); const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"; const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || ""; const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || ""; const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/"; if (!url) throw new Error("missing dashboard URL"); const consoleMessages = []; const pageErrors = []; const requestFailures = []; const browser = await chromium.launch({ headless: true, args: ["--disable-gpu", "--no-sandbox"], ...(executablePath ? { executablePath } : {}), }); const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); const page = await context.newPage(); page.on("console", (message) => { if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) }); }); page.on("pageerror", (error) => { if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) }); }); page.on("requestfailed", (request) => { if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null }); }); let httpStatus = null; let navigationError = null; let navigationAttempts = 0; const maxNavigationAttempts = 3; const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts)); for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) { navigationAttempts = attempt; navigationError = null; try { const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }); httpStatus = response?.status() ?? null; await page.waitForLoadState("networkidle", { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {}); await page.waitForFunction(() => { const root = document.querySelector("#monitor-web-root"); if (!root) return false; const ready = root.getAttribute("data-monitor-ready") === "true"; const error = document.querySelector("#monitor-web-error"); const runs = document.querySelectorAll(".run-list .run-row").length; const trend = document.querySelector("[data-monitor-trend-curve]"); return ready && (error || runs > 0 || trend); }, null, { timeout: Math.min(15000, perAttemptTimeout) }).catch(() => {}); await page.waitForTimeout(500); const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false); if (appReady || attempt === maxNavigationAttempts) break; } catch (error) { navigationError = String(error?.message || error).slice(0, 500); if (attempt === maxNavigationAttempts) break; } await page.waitForTimeout(750 * attempt); } await page.evaluate(() => { const detailPane = document.querySelector(".workspace-grid .pane-detail"); if (detailPane instanceof HTMLElement) detailPane.scrollTop = Math.min(96, Math.max(0, detailPane.scrollHeight - detailPane.clientHeight)); }).catch(() => {}); await page.waitForTimeout(150); const trendHoverPoint = await page.evaluate(() => { const target = document.querySelector(".trend-dot-hit .trend-dot-red") || document.querySelector(".trend-dot-hit .trend-dot-warning"); if (!(target instanceof SVGElement)) return null; const rect = target.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return null; return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; }).catch(() => null); if (trendHoverPoint) { await page.mouse.move(trendHoverPoint.x, trendHoverPoint.y); await page.waitForTimeout(250); } if (captureScreenshot && screenshotPath) { await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => { pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) }); }); } const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId }) => { const visible = (element) => Boolean(element && !element.hidden); const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim(); const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; const errorSampleCount = (counts) => numberValue(counts?.red) + numberValue(counts?.critical) + numberValue(counts?.error); const warningSampleCount = (counts) => numberValue(counts?.warning) + numberValue(counts?.warn) + numberValue(counts?.amber); const allSampleCount = (counts) => Object.values(counts || {}).reduce((sum, value) => sum + numberValue(value), 0); const root = document.querySelector("#monitor-web-root"); const shell = document.querySelector("[data-monitor-shell='true']"); const error = document.querySelector("#monitor-web-error"); const trend = document.querySelector("[data-monitor-trend-curve]"); const trendTooltip = document.querySelector("[data-monitor-trend-tooltip='true']"); const timeline = document.querySelector("[data-monitor-timeline='true']"); const workspace = document.querySelector("[data-monitor-independent-scroll='true']"); const checksPanel = document.querySelector("[data-monitor-checks='true']"); const panes = Array.from(document.querySelectorAll(".workspace-grid .pane, [data-monitor-checks='true']")); const detailPane = document.querySelector(".workspace-grid .pane-detail"); const detailHeader = detailPane?.querySelector(".pane-header"); const checksHeader = checksPanel?.querySelector(".pane-header"); const internalTextPattern = /水合|投影|Trace|trace|Shell|API|DOM|Console|console|Runner|runner|JSONL|steer|facts|分页|HTTP|http|requestfailed|pageerror|Final Response|Code Agent|web-probe|observe|analyzer|终态/u; const checkRows = Array.from(document.querySelectorAll("[data-check-row='true']")); const cards = checkRows.slice(0, 8).map((row) => ({ code: String(row.querySelector(".check-code")?.textContent || "").trim(), title: String(row.querySelector(".check-title-cell strong")?.textContent || row.querySelector("strong")?.textContent || "").trim(), body: String(row.textContent || "").replace(/\s+/g, " ").trim().slice(0, 180), })); const badCardTitles = cards.filter((card) => internalTextPattern.test(card.title)); const badCardBodies = cards.filter((card) => internalTextPattern.test(card.body)); const legendTexts = Array.from(document.querySelectorAll(".trend-legend .legend-item")).map((item) => String(item.textContent || "").replace(/\s+/g, " ").trim()); const legendNumber = (label) => { const row = legendTexts.find((item) => item.includes(label)) || ""; const match = /(\d+)/u.exec(row); return match ? Number(match[1]) : null; }; const chartCounts = { error: legendNumber("最新点错误"), warning: legendNumber("最新点告警"), total: legendNumber("错误+告警合计"), }; chartCounts.ok = typeof chartCounts.error === "number" && typeof chartCounts.warning === "number" && typeof chartCounts.total === "number" ? chartCounts.total === chartCounts.error + chartCounts.warning : false; const apiUrl = (path) => { const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; const prefix = basePath.replace(/\/+$/u, ""); return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; }; const runsPayload = await fetch(apiUrl("/api/runs?limit=30&sort=updated"), { cache: "no-store" }).then((item) => item.json()).catch(() => null); const overviewPayload = await fetch(apiUrl("/api/overview"), { cache: "no-store" }).then((item) => item.json()).catch(() => null); const runs = Array.isArray(runsPayload?.runs) ? runsPayload.runs : Array.isArray(runsPayload?.items) ? runsPayload.items : []; const latestRun = runs[0] || null; const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts) ? latestRun.severityCounts : {}; const latestRunCounts = { runId: latestRun?.id || latestRun?.runId || null, typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count), error: 0, warning: 0, total: 0, all: 0, errorSamples: errorSampleCount(latestCounts), warningSamples: warningSampleCount(latestCounts), alertSamples: errorSampleCount(latestCounts) + warningSampleCount(latestCounts), allSamples: allSampleCount(latestCounts), }; const latestDetailPayload = latestRunCounts.runId ? await fetch(apiUrl("/api/runs/" + encodeURIComponent(latestRunCounts.runId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null) : null; const latestDetailRows = Array.isArray(latestDetailPayload?.findings) ? latestDetailPayload.findings : []; const rowSeverity = (row) => { const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase(); if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red"; if (["warning", "warn", "amber"].includes(raw)) return "warning"; if (["info", "notice"].includes(raw)) return "info"; return "healthy"; }; const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1; const summarizeRows = (rows) => { const errorRows = rows.filter((row) => rowSeverity(row) === "red"); const warningRows = rows.filter((row) => rowSeverity(row) === "warning"); const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0); return { typeCount: rows.length, errorTypeCount: errorRows.length, warningTypeCount: warningRows.length, alertTypeCount: errorRows.length + warningRows.length, errorSamples: sum(errorRows), warningSamples: sum(warningRows), alertSamples: sum(errorRows) + sum(warningRows), }; }; const latestDetailSummary = summarizeRows(latestDetailRows); const expectsAlertRows = latestDetailSummary.alertTypeCount > 0; latestRunCounts.typeCount = latestDetailSummary.typeCount; latestRunCounts.error = latestDetailSummary.errorTypeCount; latestRunCounts.warning = latestDetailSummary.warningTypeCount; latestRunCounts.total = latestDetailSummary.alertTypeCount; latestRunCounts.all = latestDetailSummary.typeCount; const workspaceRect = workspace?.getBoundingClientRect(); const checksRect = checksPanel?.getBoundingClientRect(); const heightSummary = (rect) => { const viewportHeight = window.innerHeight || 1; const heightPx = rect ? Math.round(rect.height) : null; const ratio = rect ? Math.round((rect.height / viewportHeight) * 1000) / 1000 : null; return { present: Boolean(rect), heightPx, ratio, targetPx: Math.round(viewportHeight * 0.8), bounded80: Boolean(rect && rect.height <= viewportHeight * 0.82 + 1), near80: Boolean(rect && rect.height >= viewportHeight * 0.74 && rect.height <= viewportHeight * 0.82 + 1), }; }; const workspacePaneHeights = Array.from(document.querySelectorAll(".workspace-grid > .pane")).map((pane) => heightSummary(pane.getBoundingClientRect())); const stackedWorkspace = window.matchMedia("(max-width: 1120px)").matches; const panelHeights = { viewportHeight: window.innerHeight, stackedWorkspace, workspace: heightSummary(workspaceRect), checks: heightSummary(checksRect), workspacePanes: workspacePaneHeights, workspacePaneBounded: workspacePaneHeights.length >= 2 && workspacePaneHeights.every((item) => item.bounded80 === true), workspaceOk: false, checksOk: false, }; panelHeights.workspaceOk = stackedWorkspace ? panelHeights.workspacePaneBounded === true : panelHeights.workspace.near80 === true && panelHeights.workspacePaneBounded === true; panelHeights.checksOk = panelHeights.checks.near80 === true; const checkScope = { present: Boolean(checksPanel), scope: checksPanel?.getAttribute("data-check-scope") || null, runId: checksPanel?.getAttribute("data-check-run-id") || null, typeCount: numberValue(checksPanel?.getAttribute("data-check-type-count")), errorTypeCount: numberValue(checksPanel?.getAttribute("data-check-error-type-count")), warningTypeCount: numberValue(checksPanel?.getAttribute("data-check-warning-type-count")), alertTypeCount: numberValue(checksPanel?.getAttribute("data-check-alert-type-count")), errorSamples: numberValue(checksPanel?.getAttribute("data-check-error-samples")), warningSamples: numberValue(checksPanel?.getAttribute("data-check-warning-samples")), alertSamples: numberValue(checksPanel?.getAttribute("data-check-alert-samples")), visibleRowCount: document.querySelectorAll("[data-check-row='true']").length, visibleAlertSamples: numberValue(checksPanel?.getAttribute("data-visible-check-alert-samples")), expectsAlertRows, matchesLatestRun: false, matchesRunDetail: false, belowWorkspace: Boolean(workspaceRect && checksRect && checksRect.top >= workspaceRect.bottom - 2), fullWidth: Boolean(workspaceRect && checksRect && checksRect.width >= workspaceRect.width - 2), }; const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row")) .find((row) => row.getAttribute("data-run-id") === runId) || null; const selectedRunRow = runListRowFor(latestRunCounts.runId) || document.querySelector(".run-list .run-row.selected"); const selectedRunTags = { error: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), warning: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), errorVisible: Boolean(selectedRunRow?.querySelector("[data-run-error-tag='true']")), warningVisible: Boolean(selectedRunRow?.querySelector("[data-run-warning-tag='true']")), rowSelected: Boolean(selectedRunRow?.classList.contains("selected")), matchesRunDetail: false, }; selectedRunTags.matchesRunDetail = selectedRunTags.error === latestDetailSummary.errorTypeCount && selectedRunTags.warning === latestDetailSummary.warningTypeCount && selectedRunTags.errorVisible === (latestDetailSummary.errorTypeCount > 0) && selectedRunTags.warningVisible === (latestDetailSummary.warningTypeCount > 0) && selectedRunTags.rowSelected === true; checkScope.matchesLatestRun = checkScope.present === true && checkScope.scope === "run" && checkScope.runId === latestRunCounts.runId && checkScope.errorTypeCount === latestRunCounts.error && checkScope.warningTypeCount === latestRunCounts.warning && checkScope.alertTypeCount === latestRunCounts.total; checkScope.matchesRunDetail = checkScope.present === true && checkScope.typeCount === latestDetailSummary.typeCount && checkScope.errorTypeCount === latestDetailSummary.errorTypeCount && checkScope.warningTypeCount === latestDetailSummary.warningTypeCount && checkScope.alertTypeCount === latestDetailSummary.alertTypeCount && checkScope.errorSamples === latestDetailSummary.errorSamples && checkScope.warningSamples === latestDetailSummary.warningSamples && checkScope.alertSamples === latestDetailSummary.alertSamples && checkScope.visibleRowCount === latestDetailSummary.alertTypeCount && checkScope.visibleAlertSamples === latestDetailSummary.alertSamples; const overviewCounts = overviewPayload?.severityCounts && typeof overviewPayload.severityCounts === "object" && !Array.isArray(overviewPayload.severityCounts) ? overviewPayload.severityCounts : {}; const overviewSamples = { error: errorSampleCount(overviewCounts), warning: warningSampleCount(overviewCounts), total: errorSampleCount(overviewCounts) + warningSampleCount(overviewCounts), allSamples: allSampleCount(overviewCounts), }; chartCounts.matchesLatestRun = chartCounts.ok === true && chartCounts.error === latestRunCounts.error && chartCounts.warning === latestRunCounts.warning && chartCounts.total === latestRunCounts.total; const trendPanel = document.querySelector(".trend-panel"); const trendLegend = document.querySelector(".trend-panel .trend-legend"); const trendPanelRect = trendPanel?.getBoundingClientRect(); const trendLegendRect = trendLegend?.getBoundingClientRect(); const trendPanelCompact = { present: Boolean(trendPanelRect && trendLegendRect), bottomSlackPx: trendPanelRect && trendLegendRect ? Math.round(trendPanelRect.bottom - trendLegendRect.bottom) : null, ok: Boolean(trendPanelRect && trendLegendRect && trendPanelRect.bottom - trendLegendRect.bottom <= 28), }; const firstCheckRow = document.querySelector("[data-check-row='true']"); let checkDialog = { opened: false, title: "", width: null, height: null, large: false }; if (firstCheckRow instanceof HTMLElement) { firstCheckRow.click(); await new Promise((resolve) => window.setTimeout(resolve, 80)); const dialog = document.querySelector("[data-check-dialog='true'] .check-dialog"); const rect = dialog?.getBoundingClientRect(); checkDialog = { opened: Boolean(dialog), title: String(dialog?.querySelector("#check-dialog-title")?.textContent || "").trim(), width: rect ? Math.round(rect.width) : null, height: rect ? Math.round(rect.height) : null, large: Boolean(rect && rect.width >= Math.min(900, window.innerWidth * 0.7) && rect.height >= Math.min(460, window.innerHeight * 0.5)), }; const close = dialog?.querySelector("button[aria-label='关闭监测项详情']"); if (close instanceof HTMLElement) close.click(); } const datasetSentinelId = root?.getAttribute("data-sentinel-id") || ""; const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/"; const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/"; const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/"); const sentinelBoundary = { expectedSentinelId, expectedRoutePrefix, datasetSentinelId, overviewSentinelId: overviewPayload?.sentinelId || null, runsSentinelId: runsPayload?.sentinelId || null, finalPath, routePrefixMatches, datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true, overviewMatches: expectedSentinelId ? overviewPayload?.sentinelId === expectedSentinelId : true, runsPayloadMatches: expectedSentinelId ? runsPayload?.sentinelId === expectedSentinelId : true, runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true, }; const statusText = text(".status-strip"); const doc = document.documentElement; const body = document.body; const viewport = { width: window.innerWidth, height: window.innerHeight }; const documentSize = { width: Math.max(doc.scrollWidth, body?.scrollWidth || 0), height: Math.max(doc.scrollHeight, body?.scrollHeight || 0), }; const overflow = []; let overflowCount = 0; for (const element of Array.from(document.querySelectorAll("body *"))) { const rect = element.getBoundingClientRect(); const overflowRight = rect.right - viewport.width; const overflowLeft = -rect.left; if (overflowRight > 1 || overflowLeft > 1) { overflowCount += 1; if (overflow.length < 5) { overflow.push({ tag: element.tagName.toLowerCase(), className: String(element.className || "").slice(0, 40), x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height), overflowRight: Math.max(0, Math.round(overflowRight)), overflowLeft: Math.max(0, Math.round(overflowLeft)), }); } } } return { shell: Boolean(root && shell), ready: root?.getAttribute("data-monitor-ready") === "true", dataset: root ? { node: root.getAttribute("data-node"), lane: root.getAttribute("data-lane"), sentinelId: datasetSentinelId, basePath: root.getAttribute("data-base-path"), contractVersion: root.getAttribute("data-contract-version"), } : {}, sentinelBoundary, title: document.title, finalUrl: window.location.href, statusText: text(".topbar .pill"), subtitle: text(".subtitle"), summaryText: text(".status-strip"), runRows: document.querySelectorAll(".run-list .run-row").length, checkRows: document.querySelectorAll("[data-check-row='true']").length, badCardTitleCount: badCardTitles.length, badCardBodyCount: badCardBodies.length, trendCurve: Boolean(trend), trendDotCount: document.querySelectorAll(".trend-dot-hit").length, trendTooltip: tooltipSummary(trendTooltip), trendPanelText: text("#trend-heading"), chartCounts, latestRunCounts, latestDetailSummary, checkScope, selectedRunTags, trendPanelCompact, checkDialog, overviewSamples, panelHeights, scopeLabels: { latestPointLegend: legendTexts.some((item) => item.includes("最新点")), historicalSamples: legendTexts.some((item) => item.includes("历史样本累计")) || statusText.includes("历史错误样本"), }, timelineItems: document.querySelectorAll(".timeline-list .timeline-item").length, timelineVisible: Boolean(timeline), errorVisible: visible(error), errorText: visible(error) ? text("#monitor-web-error").slice(0, 500) : "", scrollModel: { workspace: Boolean(workspace), paneCount: panes.length, independentScroll: panes.length >= 3 && panes.every((pane) => { const style = window.getComputedStyle(pane); return style.overflowY === "auto" || style.overflowY === "scroll"; }), stickyHeader: stickyHeaderSummary(detailPane, detailHeader), stickyChecksHeader: stickyHeaderSummary(checksPanel, checksHeader), }, layout: { viewport, documentSize, horizontalOverflow: documentSize.width > viewport.width + 1, overflowCount, overflow: overflow.slice(0, 2), }, }; function tooltipSummary(element) { const body = String(element?.textContent || "").replace(/\s+/g, " ").trim(); return { visible: Boolean(element && body.length > 0), text: body.slice(0, 240), hasValues: /错误\s+\d+/u.test(body) && /告警\s+\d+/u.test(body) && /合计\s+\d+/u.test(body), hasTime: /UTC/u.test(body) || /\d{4}-\d{2}-\d{2}/u.test(body), }; } function stickyHeaderSummary(pane, header) { if (!(pane instanceof HTMLElement) || !(header instanceof HTMLElement)) { return { present: false, coversScroll: false, backgroundOpaque: false, detailScrollTop: null }; } const rect = header.getBoundingClientRect(); const style = window.getComputedStyle(header); const sampleX = Math.round(rect.left + Math.min(32, Math.max(2, rect.width / 2))); const sampleY = Math.round(rect.top + Math.min(12, Math.max(2, rect.height / 2))); const topElement = document.elementFromPoint(sampleX, sampleY); return { present: true, detailScrollTop: pane.scrollTop, headerTop: Math.round(rect.top), headerBottom: Math.round(rect.bottom), zIndex: style.zIndex, backgroundColor: style.backgroundColor, coversScroll: Boolean(topElement && header.contains(topElement)), backgroundOpaque: backgroundIsOpaque(style.backgroundColor), topElementClass: String(topElement?.className || "").slice(0, 80), }; } function backgroundIsOpaque(value) { const rgba = /rgba?\(([^)]+)\)/u.exec(value); if (rgba === null) return value.length > 0 && value !== "transparent"; const parts = rgba[1].split(",").map((part) => part.trim()); if (parts.length < 4) return true; return Number(parts[3]) >= 0.99; } }, { expectedRoutePrefix, expectedSentinelId }); const runFilterProbe = await page.evaluate(async ({ expectedRoutePrefix }) => { const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; const root = document.querySelector("#monitor-web-root"); const apiUrl = (path) => { const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; const prefix = basePath.replace(/\/+$/u, ""); return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; }; const rowSeverity = (row) => { const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase(); if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red"; if (["warning", "warn", "amber"].includes(raw)) return "warning"; if (["info", "notice"].includes(raw)) return "info"; return "healthy"; }; const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1; const summarizeRows = (rows) => { const errorRows = rows.filter((row) => rowSeverity(row) === "red"); const warningRows = rows.filter((row) => rowSeverity(row) === "warning"); const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0); return { typeCount: rows.length, errorTypeCount: errorRows.length, warningTypeCount: warningRows.length, alertTypeCount: errorRows.length + warningRows.length, errorSamples: sum(errorRows), warningSamples: sum(warningRows), alertSamples: sum(errorRows) + sum(warningRows), }; }; const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row")) .find((row) => row.getAttribute("data-run-id") === runId) || null; const panelCounts = () => { const panel = document.querySelector("[data-monitor-checks='true']"); const panelRunId = panel?.getAttribute("data-check-run-id") || null; const selectedRunRow = runListRowFor(panelRunId) || document.querySelector(".run-list .run-row.selected"); return { present: Boolean(panel), runId: panelRunId, typeCount: numberValue(panel?.getAttribute("data-check-type-count")), errorTypeCount: numberValue(panel?.getAttribute("data-check-error-type-count")), warningTypeCount: numberValue(panel?.getAttribute("data-check-warning-type-count")), alertTypeCount: numberValue(panel?.getAttribute("data-check-alert-type-count")), errorSamples: numberValue(panel?.getAttribute("data-check-error-samples")), warningSamples: numberValue(panel?.getAttribute("data-check-warning-samples")), alertSamples: numberValue(panel?.getAttribute("data-check-alert-samples")), visibleRowCount: document.querySelectorAll("[data-check-row='true']").length, visibleAlertSamples: numberValue(panel?.getAttribute("data-visible-check-alert-samples")), selectedRunErrorTag: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), selectedRunWarningTag: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), selectedRunRowSelected: Boolean(selectedRunRow?.classList.contains("selected")), }; }; const waitForRun = async (runId) => { for (let index = 0; index < 30; index += 1) { const current = panelCounts(); if (current.runId === runId && current.present === true) return true; await new Promise((resolve) => window.setTimeout(resolve, 100)); } return false; }; const select = document.querySelector("select[aria-label='选择运行记录']"); if (!(select instanceof HTMLSelectElement)) { return { ok: false, requestedRunId: null, reason: "run-select-missing" }; } const options = Array.from(select.options).filter((option) => option.value); const requestedOption = options[Math.min(2, Math.max(0, options.length - 1))] || options[0] || null; const requestedRunId = requestedOption?.value || null; const fallbackOption = requestedOption || null; const targetRunId = fallbackOption?.value || requestedRunId; if (!targetRunId) return { ok: false, requestedRunId, reason: "run-option-missing" }; select.value = targetRunId; select.dispatchEvent(new Event("change", { bubbles: true })); const panelReady = await waitForRun(targetRunId); const detailPayload = await fetch(apiUrl("/api/runs/" + encodeURIComponent(targetRunId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null); const detailRows = Array.isArray(detailPayload?.findings) ? detailPayload.findings : []; const expected = summarizeRows(detailRows); const observed = panelCounts(); const matchesRunDetail = observed.runId === targetRunId && observed.typeCount === expected.typeCount && observed.errorTypeCount === expected.errorTypeCount && observed.warningTypeCount === expected.warningTypeCount && observed.alertTypeCount === expected.alertTypeCount && observed.errorSamples === expected.errorSamples && observed.warningSamples === expected.warningSamples && observed.alertSamples === expected.alertSamples && observed.visibleRowCount === expected.alertTypeCount && observed.visibleAlertSamples === expected.alertSamples && observed.selectedRunErrorTag === expected.errorTypeCount && observed.selectedRunWarningTag === expected.warningTypeCount && observed.selectedRunRowSelected === true; return { ok: panelReady === true && matchesRunDetail === true, requestedRunId, requestedOptionPresent: Boolean(requestedOption), targetRunId, panelReady, observed, expected, matchesRunDetail, }; }, { expectedRoutePrefix }); dom.runFilterProbe = runFilterProbe; const consoleErrors = consoleMessages.filter((item) => item.type === "error"); const ok = !navigationError && httpStatus !== null && httpStatus >= 200 && httpStatus < 300 && dom.shell === true && dom.ready === true && dom.sentinelBoundary?.datasetMatches === true && dom.sentinelBoundary?.overviewMatches === true && dom.sentinelBoundary?.runsPayloadMatches === true && dom.sentinelBoundary?.runRowsMatch === true && dom.sentinelBoundary?.routePrefixMatches === true && dom.errorVisible !== true && dom.trendCurve === true && dom.chartCounts?.ok === true && dom.chartCounts?.matchesLatestRun === true && dom.checkScope?.matchesLatestRun === true && dom.checkScope?.matchesRunDetail === true && dom.selectedRunTags?.matchesRunDetail === true && dom.runFilterProbe?.ok === true && dom.runFilterProbe?.requestedOptionPresent === true && dom.checkScope?.belowWorkspace === true && dom.checkScope?.fullWidth === true && dom.scopeLabels?.latestPointLegend === true && dom.scopeLabels?.historicalSamples === true && (dom.trendDotCount === 0 || (dom.trendTooltip?.visible === true && dom.trendTooltip?.hasValues === true && dom.trendTooltip?.hasTime === true)) && dom.trendPanelCompact?.ok === true && (dom.checkScope?.expectsAlertRows !== true || (dom.checkRows > 0 && dom.checkDialog?.opened === true && dom.checkDialog?.large === true)) && dom.badCardTitleCount === 0 && dom.badCardBodyCount === 0 && dom.timelineVisible === true && dom.scrollModel?.independentScroll === true && dom.scrollModel?.stickyHeader?.present === true && dom.scrollModel?.stickyHeader?.backgroundOpaque === true && dom.scrollModel?.stickyChecksHeader?.present === true && dom.scrollModel?.stickyChecksHeader?.backgroundOpaque === true && dom.panelHeights?.workspaceOk === true && dom.panelHeights?.checksOk === true && dom.layout?.horizontalOverflow !== true && pageErrors.length === 0; console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ ok, url, httpStatus, navigationError, navigationAttempts, executablePath: executablePath || null, viewport: { width, height }, screenshotPath: captureScreenshot ? screenshotPath : null, dom, consoleCount: consoleMessages.length, consoleErrorCount: consoleErrors.length, pageErrorCount: pageErrors.length, requestFailureCount: requestFailures.length, consoleMessages: consoleMessages.slice(0, 8), pageErrors: pageErrors.slice(0, 8), requestFailures: requestFailures.slice(0, 8), valuesRedacted: true, })); await context.close().catch(() => {}); await browser.close().catch(() => {}); `; } function parseDashboardBrowserPayload(textValue: string): Record | null { const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__"; const index = textValue.lastIndexOf(marker); if (index < 0) return null; try { return record(JSON.parse(textValue.slice(index + marker.length).trim())); } catch { return null; } } function dashboardScreenshotName(options: Extract, state: SentinelCicdState): string { const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`; const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120); return safe.endsWith(".png") ? safe : `${safe}.png`; } function compactDashboardArtifact(artifact: Record): Record { const transfer = record(artifact.transfer); return { remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null, localPath: typeof artifact.localPath === "string" ? artifact.localPath : null, bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null, sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null, verified: artifact.verified === true, transfer: Object.keys(transfer).length === 0 ? null : { strategy: transfer.strategy ?? null, transport: transfer.transport ?? null, chunks: transfer.chunks ?? null, elapsedMs: transfer.elapsedMs ?? null, throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null, }, }; } function dashboardDegradedReason(result: CommandResult, transport: Record, page: Record | null, screenshotOk: boolean): string { if (result.timedOut) return "sentinel-dashboard-command-timeout"; if (transport.ok !== true) return "sentinel-dashboard-transport-failed"; if (page === null) return "sentinel-dashboard-browser-output-missing"; if (page.ok !== true) return "sentinel-dashboard-render-failed"; if (!screenshotOk) return "sentinel-dashboard-screenshot-missing"; return "sentinel-dashboard-unknown"; } function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult { const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; if (releaseId !== null) args.push("--release-id", releaseId); if (reason !== null) args.push("--reason", reason); if (quickVerify) args.push("--quick-verify"); const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`); const command = `web-probe sentinel ${subcommand.join(" ")}`; return rendered(true, command, renderAsyncJobResult({ ok: true, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "async-job", mutation: true, job, next: { status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, wait: ["bun", "scripts/cli.ts", ...args].join(" "), }, valuesRedacted: true, })); } function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { const namespace = stringAt(state.runtime, "namespace"); const serviceName = stringAt(state.runtime, "serviceName"); const servicePort = numberAt(state.runtime, "servicePort"); const deploymentName = stringAt(state.runtime, "deploymentName"); const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`; if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") { return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url); } const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); const postScript = [ "const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');", "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", `const url = 'http://127.0.0.1:${servicePort}' + path;`, "fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {", " const text = await response.text();", " process.stdout.write(text);", " if (!response.ok) process.exit(22);", "}).catch((error) => {", " console.error(error && error.stack ? error.stack : String(error));", " process.exit(23);", "});", ].join(" "); const script = method === "GET" ? `kubectl get --raw ${shellQuote(proxyPath)}` : [ "set -eu", `kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`, ].join("\n"); const maxAttempts = method === "GET" ? 3 : 1; const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); const attempts: Record[] = []; let result: CommandResult | null = null; let parsed: Record | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); parsed = parseJsonObject(result.stdout); attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true }); if (result.exitCode === 0) break; } const compactBodyJson = compactSentinelServiceBodyJson(parsed); return { ok: result?.exitCode === 0, method, path: pathWithQuery, internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, httpStatus: result?.exitCode === 0 ? 200 : null, bodyJson: record(compactBodyJson), bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "", bodyBytes: Buffer.byteLength(result?.stdout ?? ""), error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000), proxyPath, result: result === null ? null : compactCommand(result), attempts, valuesRedacted: true, }; } function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number, url: string): Record { const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const fetchScript = [ "const method = process.env.SENTINEL_METHOD || 'GET';", "const url = process.env.SENTINEL_URL || '';", "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", "const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);", "const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);", "const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;", "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", "(async () => {", " let lastError = null;", " for (let attempt = 1; attempt <= attempts; attempt += 1) {", " try {", " const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });", " const text = await response.text();", " process.stdout.write(text);", " if (!response.ok) process.exit(22);", " process.exit(0);", " } catch (error) {", " lastError = error;", " console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));", " if (attempt < attempts && delayMs > 0) await sleep(delayMs);", " }", " }", " console.error(lastError && lastError.stack ? lastError.stack : String(lastError));", " process.exit(23);", "})().catch((error) => {", " console.error(error && error.stack ? error.stack : String(error));", " process.exit(24);", "});", ].join(" "); const attempts = method === "GET" ? 6 : 3; const retryDelayMs = 1000; const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70)); const result = runCommand(["node", "-e", fetchScript], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000, env: { ...process.env, SENTINEL_METHOD: method, SENTINEL_URL: url, SENTINEL_BODY_B64: bodyB64, SENTINEL_ATTEMPTS: String(attempts), SENTINEL_RETRY_DELAY_MS: String(retryDelayMs), }, }); const parsed = parseJsonObject(result.stdout); const compactBodyJson = compactSentinelServiceBodyJson(parsed); return { ok: result.exitCode === 0, method, path: pathWithQuery, internalUrl: url, httpStatus: result.exitCode === 0 ? 200 : null, bodyJson: record(compactBodyJson), bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "", bodyBytes: Buffer.byteLength(result.stdout), error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000), proxyPath: null, result: compactCommand(result), attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }], transport: "direct-service", valuesRedacted: true, }; } function compactSentinelServiceBodyJson(value: Record | null): unknown { if (value === null || typeof value.renderedText !== "string") return value; return { ...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]), run: pickFields(record(value.run), ["id", "scenario_id", "status", "node", "lane", "observer_id", "state_dir", "report_json_sha256", "finding_count", "artifact_count", "maintenance", "created_at", "updated_at"]), summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]), findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [], renderedText: value.renderedText, valuesRedacted: true, }; } function pickFields(value: Record, keys: readonly string[]): Record { const picked: Record = {}; for (const key of keys) { if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key]; } return picked; } function clipTail(value: string, maxChars: number): string { return value.length <= maxChars ? value : value.slice(-maxChars); } function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`; const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000)); const js = [ "const url=process.env.REQ_URL||'';", "const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);", "let out;", "try{", " const controller=new AbortController();", " const timer=setTimeout(()=>controller.abort(), timeoutMs);", " const started=Date.now();", " const res=await fetch(url,{signal:controller.signal});", " const text=await res.text();", " clearTimeout(timer);", " let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}", " out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};", "}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}", "console.log(JSON.stringify(out));", ].join(""); const result = runCommand(["bun", "-e", js], repoRoot, { timeoutMs: timeoutMs + 2000, env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) }, }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, method: "GET", path: pathWithQuery, publicUrl: url, httpStatus: parsed?.httpStatus ?? null, bodyJson: record(parsed?.bodyJson), bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "", bodyBytes: parsed?.bodyBytes ?? null, error: parsed?.error ?? null, result: compactCommand(result), valuesRedacted: true, }; } function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl"); const hostname = stringAt(state.publicExposure, "hostname"); const expectedA = stringAt(state.publicExposure, "expectedA"); const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`; const script = [ "set +e", `host=${shellQuote(hostname)}`, `expected=${shellQuote(expectedA)}`, `url=${shellQuote(probeUrl)}`, "dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)", "headers=$(mktemp)", "body=$(mktemp)", "writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)", "curl_rc=$?", "body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')", "node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'", "const fs=require('node:fs');", "const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);", "const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);", "const status=Number(statusRaw||0);", "const ssl=Number(sslRaw||-1);", "const addrs=dns?dns.split(',').filter(Boolean):[];", "const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();", "const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');", "let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}", "const authCovered=status===401||status===403||status>=200&&status<300;", "const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;", "const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));", "const dnsMatches=addrs.includes(expected);", "console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const rootUrl = `${publicBaseUrl}/`; const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`; const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`; const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`; const script = [ "set +e", `root_url=${shellQuote(rootUrl)}`, `css_url=${shellQuote(cssUrl)}`, `js_url=${shellQuote(jsUrl)}`, `vue_url=${shellQuote(vueUrl)}`, "root_body=$(mktemp)", "css_body=$(mktemp)", "js_body=$(mktemp)", "vue_body=$(mktemp)", "root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?", "css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?", "js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?", "vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?", "node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'", "const fs=require('node:fs');", "const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);", "function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}", "const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);", "const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&root.includes('monitor-web-bootstrap');", "const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.includes('monitor-shell')&&css.includes('workspace-grid')&&css.includes('trend-stage')&&css.length>1000;", "const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.includes('createApp')&&js.includes('/api/overview')&&js.includes('data-monitor-trend-curve')&&js.length>1000;", "const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.includes('createApp')&&vue.length>80000;", "console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contract:root.includes('draft-2026-06-27-p11-monitor-web-observability-dashboard')},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css),workspaceGrid:css.includes('workspace-grid'),trendStage:css.includes('trend-stage')},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js),vueApp:js.includes('createApp'),apiClient:js.includes('/api/overview'),trend:js.includes('data-monitor-trend-curve')},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue),runtime:vue.includes('createApp')},valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function renderMaintenanceResult(result: Record): string { const serviceHealth = record(result.serviceHealth); const maintenance = record(result.maintenance); const quickVerify = record(result.quickVerify); const quickVerifyBusiness = record(quickVerify.businessStatus); const planned = record(result.planned); const blocker = record(result.blocker); const next = record(result.next); const maintenanceBody = record(maintenance.bodyJson); const state = record(maintenanceBody.maintenance); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]), "", table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]), "", Object.keys(state).length > 0 ? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]]) : Object.keys(planned).length > 0 ? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]]) : "MAINTENANCE\n-", "", Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), "", "NEXT", ` validate: ${next.validate ?? "-"}`, ` report: ${next.report ?? "-"}`, ` maintenance-stop: ${next.maintenanceStop ?? "-"}`, "", "DISCLOSURE", " maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.", ].join("\n"); } function renderValidateResult(result: Record): string { const health = record(result.serviceHealth); const metrics = record(result.metrics); const report = record(result.report); const publicExposure = record(result.publicExposure); const publicDashboard = record(result.publicDashboard); const quickVerify = record(result.quickVerify); const blocker = record(result.blocker); const next = record(result.next); const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]), "", table(["CHECK", "OK", "DETAIL"], [ ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`], ["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`], ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], ["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`], ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`], ]), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]), "", "NEXT", ` quick-verify: ${next.quickVerify ?? "-"}`, ` report: ${next.report ?? "-"}`, ` maintenance-start: ${next.maintenanceStart ?? "-"}`, "", "DISCLOSURE", " validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.", ].join("\n"); } function renderDashboardResult(result: Record): string { const page = record(result.page); const dom = record(page.dom); const dataset = record(dom.dataset); const layout = record(dom.layout); const chartCounts = record(dom.chartCounts); const latestRunCounts = record(dom.latestRunCounts); const latestDetailSummary = record(dom.latestDetailSummary); const checkScope = record(dom.checkScope); const selectedRunTags = record(dom.selectedRunTags); const trendPanelCompact = record(dom.trendPanelCompact); const checkDialog = record(dom.checkDialog); const runFilterProbe = record(dom.runFilterProbe); const runFilterObserved = record(runFilterProbe.observed); const runFilterExpected = record(runFilterProbe.expected); const overviewSamples = record(dom.overviewSamples); const panelHeights = record(dom.panelHeights); const workspaceHeight = record(panelHeights.workspace); const checksHeight = record(panelHeights.checks); const screenshot = record(result.screenshot); const remote = record(result.remote); const transport = record(result.transport); const degradedReason = result.degradedReason ?? null; return [ String(result.command), "", table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]), "", table(["HTTP", "SHELL", "RUN_ROWS", "CHECK_ROWS", "TABS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[ page.httpStatus ?? "-", dom.shell, dom.runRows, dom.checkRows, dom.detailTabs, page.pageErrorCount, page.consoleErrorCount, page.requestFailureCount, ]]), "", table(["TITLE", "STATUS_TEXT", "CONTRACT", "BASE_PATH"], [[dom.title, dom.statusText, dataset.contractVersion, dataset.basePath ?? "-"]]), "", table(["TREND_ERR_TYPES", "TREND_ALERT_TYPES", "TREND_TOTAL_TYPES", "TREND_EXACT", "MATCH_LATEST", "BAD_TITLE", "BAD_BODY"], [[ chartCounts.error ?? "-", chartCounts.warning ?? "-", chartCounts.total ?? "-", chartCounts.ok ?? "-", chartCounts.matchesLatestRun ?? "-", dom.badCardTitleCount ?? "-", dom.badCardBodyCount ?? "-", ]]), "", table(["LATEST_RUN", "TYPE_COUNT", "ERR_TYPES", "ALERT_TYPES", "TOTAL_TYPES", "SAMPLE_TOTAL", "HIST_ERR", "HIST_ALERT"], [[ latestRunCounts.runId ?? "-", latestRunCounts.typeCount ?? "-", latestRunCounts.error ?? "-", latestRunCounts.warning ?? "-", latestRunCounts.total ?? "-", latestRunCounts.alertSamples ?? "-", overviewSamples.error ?? "-", overviewSamples.warning ?? "-", ]]), "", table(["CHECK_SCOPE", "CHECK_RUN", "CHECK_TYPES", "CHECK_ERR_TYPES", "CHECK_ALERT_TYPES", "SAMPLE_ERR", "SAMPLE_ALERT", "CHECK_MATCH_LATEST", "CHECK_MATCH_DETAIL"], [[ checkScope.scope ?? "-", checkScope.runId ?? "-", `${checkScope.typeCount ?? "-"}/${latestDetailSummary.typeCount ?? "-"}`, `${checkScope.errorTypeCount ?? "-"}/${latestDetailSummary.errorTypeCount ?? "-"}`, `${checkScope.alertTypeCount ?? "-"}/${latestDetailSummary.alertTypeCount ?? "-"}`, checkScope.errorSamples ?? "-", checkScope.alertSamples ?? "-", checkScope.matchesLatestRun ?? "-", checkScope.matchesRunDetail ?? "-", ]]), "", table(["CHECK_VISIBLE_ROWS", "EXPECT_ALERT_ROWS", "CHECK_VISIBLE_ALERT", "RUN_TAG_ERR", "RUN_TAG_ALERT", "RUN_TAG_MATCH", "BELOW_WORKSPACE", "FULL_WIDTH"], [[ checkScope.visibleRowCount ?? "-", checkScope.expectsAlertRows ?? "-", checkScope.visibleAlertSamples ?? "-", selectedRunTags.error ?? "-", selectedRunTags.warning ?? "-", selectedRunTags.matchesRunDetail ?? "-", checkScope.belowWorkspace ?? "-", checkScope.fullWidth ?? "-", ]]), "", table(["TREND_PANEL_SLACK", "TREND_PANEL_COMPACT", "DETAIL_DIALOG", "DIALOG_LARGE"], [[ trendPanelCompact.bottomSlackPx ?? "-", trendPanelCompact.ok ?? "-", checkDialog.opened ?? "-", checkDialog.large ?? "-", ]]), "", table(["WORKSPACE_H", "WORKSPACE_RATIO", "WORKSPACE_80", "CHECKS_H", "CHECKS_RATIO", "CHECKS_80", "PANES_80"], [[ `${workspaceHeight.heightPx ?? "-"}/${workspaceHeight.targetPx ?? "-"}`, workspaceHeight.ratio ?? "-", panelHeights.workspaceOk ?? "-", `${checksHeight.heightPx ?? "-"}/${checksHeight.targetPx ?? "-"}`, checksHeight.ratio ?? "-", panelHeights.checksOk ?? "-", panelHeights.workspacePaneBounded ?? "-", ]]), "", table(["FILTER_RUN", "FILTER_OPTION", "FILTER_TYPES", "FILTER_ERR_TYPES", "FILTER_ALERT_TYPES", "FILTER_SAMPLE_ERR", "FILTER_SAMPLE_ALERT", "FILTER_MATCH_DETAIL"], [[ runFilterProbe.targetRunId ?? "-", runFilterProbe.requestedOptionPresent ?? "-", `${runFilterObserved.typeCount ?? "-"}/${runFilterExpected.typeCount ?? "-"}`, `${runFilterObserved.errorTypeCount ?? "-"}/${runFilterExpected.errorTypeCount ?? "-"}`, `${runFilterObserved.alertTypeCount ?? "-"}/${runFilterExpected.alertTypeCount ?? "-"}`, runFilterObserved.errorSamples ?? "-", runFilterObserved.alertSamples ?? "-", runFilterProbe.matchesRunDetail ?? "-", ]]), "", table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[ result.viewport, `${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`, layout.horizontalOverflow, layout.overflowCount, ]]), "", Object.keys(screenshot).length === 0 ? "SCREENSHOT\n-" : table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]), "", degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT"], [[degradedReason, remote.exitCode, transport.ok]]), "", "NEXT", ` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, ` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, "", "DISCLOSURE", " dashboard verify uses the YAML publicExposure URL and remote browser execution; it does not start a sentinel run or inspect provider payloads.", ].join("\n"); } function renderReportResult(result: Record): string { const report = record(result.report); const body = record(report.bodyJson); const run = record(body.run); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]), "", table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]), "", "DISCLOSURE", " report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.", ].join("\n"); }