// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. // Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands. import { existsSync, readFileSync } from "node:fs"; import type { CommandResult } from "./command"; import { runCommand } from "./command"; import { repoRoot } from "./config"; import { startJob } from "./jobs"; import type { RenderedCliResult } from "./output"; import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact"; import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd"; import { clipTail, compactCommand, compactSentinelServiceBodyJson, mergeWarnings, numberAt, numberAtNullable, parseJsonObject, pickFields, record, rendered, renderAsyncJobResult, safeJobSegment, sentinelCliSuffix, shellQuote, short, stringAt, stringAtNullable, table, targetValidationElapsedWarnings, text, withWarnings, } from "./hwlab-node-web-sentinel-cicd"; import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe"; const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55; export function runSentinelMaintenance(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel maintenance ${options.action}`; const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); if (options.action === "status") { const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds); const result = { ok: serviceHealth.ok && maintenance.ok, command, node: state.spec.nodeId, lane: state.spec.lane, serviceHealth, maintenance, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } if (!options.confirm) { const result = { ok: serviceHealth.ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "dry-run", serviceHealth, mutation: false, planned: { action: options.action, releaseId: options.releaseId, reason: options.reason, quickVerify: options.action === "stop" && options.quickVerify, }, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); if (!serviceHealth.ok) { const result = { ok: false, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: false, serviceHealth, blocker: serviceUnavailableBlocker(state), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(false, command, renderMaintenanceResult(result)); } const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true }; const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds); const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok ? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds) : null; const result = { ok: mutation.ok && (quickVerify === null || quickVerify.ok === true), command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", mutation: true, serviceHealth, maintenance: mutation, quickVerify, blocker: mutation.ok ? null : serviceUnavailableBlocker(state), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } export function runSentinelValidate(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = "web-probe sentinel validate"; const startedAt = Date.now(); const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); let quickVerify: Record | null = null; if (options.quickVerify) { if (!options.confirm) { const result = { ok: initialHealth.ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "dry-run", serviceHealth: initialHealth, planned: { quickVerify: true, waitRequired: true }, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderValidateResult(result)); } if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true); if (!initialHealth.ok) { const result = { ok: false, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "confirm-wait", serviceHealth: initialHealth, blocker: serviceUnavailableBlocker(state), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(false, command, renderValidateResult(result)); } quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds); } const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20); const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds); const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds); const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds); const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health"); const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds); const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds); const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds); const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health; const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics; const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report; const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds); const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds); if (quickVerify !== null) { quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds")))); } const publicFallbackWarnings = [ ...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []), ...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []), ...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []), ]; const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health"); const ok = effectiveHealth.ok && record(effectiveHealth.bodyJson).ok === true && effectiveMetricsOk && effectiveReport.ok && publicExposure.ok === true && publicDashboard.ok === true && (quickVerify === null || quickVerify.ok === true); const result = { ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: options.quickVerify ? "confirm-wait" : "status", serviceHealth: effectiveHealth, metrics: effectiveMetrics, report: effectiveReport, internalServiceHealth: health, internalMetrics: metrics, internalReport: report, publicServiceHealth: publicHealth, publicMetrics, publicReport, publicExposure, publicDashboard, quickVerify, warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []), blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(ok, command, renderValidateResult(result)); } export function probeSentinelRuntimeHealthEndpoint(state: SentinelCicdState, timeoutSeconds: number): Record { const endpoint = stringAt(state.runtime, "healthPath"); const serviceProbeTimeoutSeconds = Math.min(timeoutSeconds, 20); const health = callSentinelService(state, "GET", endpoint, null, serviceProbeTimeoutSeconds); const publicHealth = health.ok ? null : probePublicSentinelService(state, endpoint, serviceProbeTimeoutSeconds); const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health; const bodyJson = record(effectiveHealth.bodyJson); const ok = effectiveHealth.ok === true && bodyJson.ok === true; return { ok, endpoint, health: effectiveHealth, internalHealth: health, publicHealth, httpStatus: effectiveHealth.httpStatus ?? null, publicUrl: effectiveHealth.publicUrl ?? null, internalUrl: effectiveHealth.internalUrl ?? null, degradedReason: ok ? null : "sentinel-runtime-health-endpoint-failed", valuesRedacted: true, }; } export function runSentinelReport(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`; const query = new URLSearchParams({ view: options.view }); if (options.runId !== null) query.set("run", options.runId); if (options.traceId !== null) query.set("traceId", options.traceId); if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq)); const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds); const body = record(report.bodyJson); const rawPayload = Object.keys(body).length > 0 ? body : report; if (options.full) return rendered(report.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2)); if (options.raw) { const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS)); return rendered(report.ok && body.ok !== false, command, JSON.stringify(compactSentinelReportRawPayload(state, body, report, artifactSummary), null, 2)); } if (options.view === "summary") { const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS)); const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary); return rendered(report.ok && body.ok !== false, command, renderSentinelReportSummary(payload, state)); } if (options.view === "findings") { const artifactSummary = readSentinelReportArtifactSummary(state, body, Math.min(options.timeoutSeconds, SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS)); const payload = compactSentinelReportRawPayload(state, body, report, artifactSummary); return rendered(report.ok && body.ok !== false, command, renderSentinelReportFindings(payload)); } const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true }); return rendered(report.ok && body.ok !== false, command, renderedText); } function compactSentinelReportRawPayload( state: SentinelCicdState, body: Record, report: Record, artifactSummary: Record | null, ): Record { const run = record(body.run); const artifact = record(artifactSummary); const findings = Array.isArray(body.findings) ? body.findings.map(record) : []; const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : []; const visibleFindings = mergeSentinelReportFindings(findings, artifactFindings); const offlineReclassify = offlineQuickVerifyReclassify(state, run, visibleFindings); const storedFindingCount = numberAtNullable(run, "finding_count") ?? numberAtNullable(run, "findingCount") ?? findings.length; const artifactFindingCount = numberAtNullable(artifact, "findingCount") ?? artifactFindings.length; const visibleFindingCount = Math.max(storedFindingCount, artifactFindingCount, visibleFindings.length); const rootCauseSignalFindings = artifactFindings .filter((item) => Object.keys(record(item.rootCauseSignals)).length > 0) .slice(0, 8) .map((item) => ({ id: stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"), severity: stringAtNullable(item, "severity") ?? stringAtNullable(item, "level"), summary: reportText(item.summary ?? item.message, 220), rootCauseSignals: compactRootCauseSignals(item.rootCauseSignals), valuesRedacted: true, })); return { ok: body.ok !== false && report.ok !== false, view: body.view ?? null, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, run: { id: run.id ?? null, scenarioId: run.scenario_id ?? run.scenarioId ?? null, status: run.status ?? null, observerId: run.observer_id ?? run.observerId ?? null, stateDir: run.state_dir ?? run.stateDir ?? null, reportJsonSha256: run.report_json_sha256 ?? run.reportJsonSha256 ?? artifact.reportJsonSha256 ?? null, findingCount: visibleFindingCount, storedFindingCount, artifactFindingCount, artifactCount: run.artifact_count ?? run.artifactCount ?? artifact.artifactCount ?? null, durationMinutes: run.durationMinutes ?? run.runDurationMinutes ?? record(run.timing).durationMinutes ?? null, runDurationMinutes: run.runDurationMinutes ?? run.durationMinutes ?? record(run.timing).durationMinutes ?? null, durationSource: run.durationSource ?? record(run.timing).durationSource ?? null, scenarioCadenceMinutes: run.scenarioCadenceMinutes ?? record(run.timing).scenarioCadenceMinutes ?? null, scenarioMaxRunMinutes: run.scenarioMaxRunMinutes ?? record(run.timing).scenarioMaxRunMinutes ?? null, schedulerIntervalMinutes: run.schedulerIntervalMinutes ?? record(run.timing).schedulerIntervalMinutes ?? null, timing: pickFields(record(run.timing), ["startedAt", "finishedAt", "durationMs", "durationMinutes", "durationSource", "scenarioCadence", "scenarioCadenceMinutes", "scenarioMaxRunMinutes", "schedulerIntervalMinutes", "sourceOfTruth", "valuesRedacted"]), updatedAt: run.updated_at ?? run.updatedAt ?? null, valuesRedacted: true, }, summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]), findings: visibleFindings.slice(0, 12).map(compactSentinelReportFinding), offlineReclassify, artifactSummary: Object.keys(artifact).length === 0 ? null : { ok: artifact.ok === true, reason: artifact.reason ?? null, reportReadWaitMs: artifact.reportReadWaitMs ?? null, reportParseError: artifact.reportParseError ?? null, reportOk: artifact.reportOk === true ? true : artifact.reportOk === false ? false : null, reportJsonPath: artifact.reportJsonPath ?? null, reportJsonSha256: artifact.reportJsonSha256 ?? null, reportMdSha256: artifact.reportMdSha256 ?? null, findingCount: artifactFindingCount, findings: artifactFindings.slice(0, 12).map(compactSentinelReportFinding), screenshot: record(artifact.screenshot), counts: record(artifact.counts), analysisWindow: compactSentinelAnalysisWindow(artifact.analysisWindow), pagePerformanceSlowApi: Array.isArray(artifact.pagePerformanceSlowApi) ? artifact.pagePerformanceSlowApi.slice(0, 8).map(record) : [], rootCauseSignalFindings, valuesRedacted: true, }, next: { text: "Default report is bounded text; use --full for the full indexed service payload.", report: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")}`, full: `bun scripts/cli.ts web-probe sentinel report --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --latest --view ${String(body.view ?? "summary")} --full`, }, valuesRedacted: true, }; } function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record, findings: readonly Record[]): Record | null { const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("quick-verify-no-business-turn")); if (!hasLegacyQuickVerifyControl) return null; const catalog = sentinelReportCheckCatalogById(state); const result = reclassifyQuickVerifyControlFindings(state, { runId: stringAtNullable(run, "id"), scenarioId: stringAtNullable(run, "scenario_id") ?? stringAtNullable(run, "scenarioId"), observerId: stringAtNullable(run, "observer_id") ?? stringAtNullable(run, "observerId"), failure: stringAtNullable(record(run.summary), "failure"), timeoutSeconds: SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS, }); const resultFindings = Array.isArray(result.findings) ? result.findings.map(record) : []; return { ...pickFields(result, ["ok", "reason", "runId", "scenarioId", "observerId", "promptIndex", "findingCount", "turnSummary", "traceFrame", "valuesRedacted"]), source: "offline-existing-observe-artifact", note: "This is a local CLI reclassification of existing turn-summary/trace-frame artifacts; it does not mutate the runner index.", findings: resultFindings.slice(0, 8).map((item) => compactSentinelReportFinding(enrichSentinelReportFindingWithCatalog(item, catalog))), valuesRedacted: true, }; } function sentinelReportCheckCatalogById(state: SentinelCicdState): Map> { try { const reportViews = record(readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.reportViews)); const catalogRef = stringAtNullable(reportViews, "checkCatalogRef"); if (catalogRef === null) return new Map(); const catalog = record(readWebProbeSentinelConfigRefTarget(state.spec, catalogRef)); const items = Array.isArray(catalog.items) ? catalog.items.map(record) : Array.isArray(catalog.checks) ? catalog.checks.map(record) : []; return new Map(items.map((item) => [stringAtNullable(item, "id") ?? "", item]).filter((item): item is [string, Record] => item[0].length > 0)); } catch { return new Map(); } } function enrichSentinelReportFindingWithCatalog(item: Record, catalog: ReadonlyMap>): Record { const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"); const check = id === null ? null : catalog.get(id) ?? null; if (check === null) return item; return { ...item, check, checkCode: stringAtNullable(check, "code"), checkTitleZh: stringAtNullable(check, "titleZh"), valuesRedacted: true, }; } function mergeSentinelReportFindings(primary: readonly Record[], artifact: readonly Record[]): Record[] { const merged: Record[] = []; const seen = new Set(); const aliases = sentinelReportFindingIdentityAliases(primary); for (const item of [...primary, ...artifact]) { const id = sentinelReportCanonicalFindingIdentity(item, aliases); const key = id ?? `${JSON.stringify(item).slice(0, 160)}:${stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? ""}`; if (seen.has(key)) continue; seen.add(key); merged.push(item); } return merged; } function sentinelReportFindingIdentityAliases(items: readonly Record[]): Map { const aliases = new Map(); for (const item of items) { const canonical = sentinelReportFindingCanonicalCandidate(item) ?? sentinelReportFindingIdentity(item); if (canonical === null) continue; for (const candidate of sentinelReportFindingIdentityCandidates(item)) aliases.set(candidate, canonical); } return aliases; } function sentinelReportCanonicalFindingIdentity(item: Record, aliases: ReadonlyMap): string | null { for (const candidate of sentinelReportFindingIdentityCandidates(item)) { const canonical = aliases.get(candidate); if (canonical !== undefined) return canonical; } return sentinelReportFindingCanonicalCandidate(item) ?? sentinelReportFindingIdentity(item); } function sentinelReportFindingCanonicalCandidate(item: Record): string | null { const check = record(item.check); return stringAtNullable(item, "checkCode") ?? stringAtNullable(check, "code") ?? stringAtNullable(item, "checkId") ?? stringAtNullable(check, "id"); } function sentinelReportFindingIdentityCandidates(item: Record): readonly string[] { const check = record(item.check); const candidates = [ stringAtNullable(item, "finding_id"), stringAtNullable(item, "findingId"), stringAtNullable(item, "id"), stringAtNullable(item, "kind"), stringAtNullable(item, "code"), stringAtNullable(item, "checkId"), stringAtNullable(item, "checkCode"), stringAtNullable(check, "id"), stringAtNullable(check, "code"), ]; return [...new Set(candidates.filter((item): item is string => item !== null))]; } function sentinelReportFindingIdentity(item: Record): string | null { return stringAtNullable(item, "finding_id") ?? stringAtNullable(item, "findingId") ?? stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code"); } function compactSentinelReportFinding(value: Record): Record { const result: Record = { id: stringAtNullable(value, "finding_id") ?? stringAtNullable(value, "findingId") ?? stringAtNullable(value, "id") ?? stringAtNullable(value, "kind") ?? stringAtNullable(value, "code"), severity: stringAtNullable(value, "severity") ?? stringAtNullable(value, "level"), count: value.count ?? null, summary: reportText(value.summary ?? value.message, 220), valuesRedacted: true, }; const rootCause = stringAtNullable(value, "rootCause"); if (rootCause !== null) result.rootCause = rootCause; const rootCauseStatus = stringAtNullable(value, "rootCauseStatus"); if (rootCauseStatus !== null) result.rootCauseStatus = rootCauseStatus; const rootCauseConfidence = stringAtNullable(value, "rootCauseConfidence"); if (rootCauseConfidence !== null) result.rootCauseConfidence = rootCauseConfidence; const nextAction = reportText(value.nextAction, 240); if (nextAction !== null) result.nextAction = nextAction; const evidenceSummary = reportText(value.evidenceSummary, 240); if (evidenceSummary !== null) result.evidenceSummary = evidenceSummary; const timingSourceOfTruth = stringAtNullable(value, "timingSourceOfTruth"); if (timingSourceOfTruth !== null) result.timingSourceOfTruth = timingSourceOfTruth; const timingStatus = stringAtNullable(value, "timingStatus"); if (timingStatus !== null) result.timingStatus = timingStatus; if (value.timingAlert === true) result.timingAlert = true; const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals); if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals; const check = record(value.check); const checkCodeFromValue = stringAtNullable(value, "checkCode"); const checkTitleFromValue = stringAtNullable(value, "checkTitleZh"); const checkCode = stringAtNullable(check, "code"); const checkTitle = stringAtNullable(check, "titleZh"); if (checkCodeFromValue !== null) result.checkCode = checkCodeFromValue; if (checkTitleFromValue !== null) result.checkTitleZh = checkTitleFromValue; if (checkCode !== null || checkTitle !== null) { result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]); } return result; } function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Record, timeoutSeconds: number): Record | null { const run = record(body.run); const stateDir = stringAtNullable(run, "state_dir") ?? stringAtNullable(run, "stateDir"); if (stateDir === null || !isSafeSentinelReportStateDir(stateDir)) return null; const waitMs = Math.max(0, Math.min(50_000, (Math.max(5, timeoutSeconds) * 1000) - 5000)); const script = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, `wait_ms=${waitMs}`, "node - \"$state_dir\" \"$wait_ms\" <<'NODE'", "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", "const stateDir=process.argv[2]; const waitMs=Math.max(0, Math.min(60000, Number(process.argv[3]||0)||0)); const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}};", "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};", "const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};", "let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;", "while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }", "let artifactCount=0; let screenshot=null;", "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", "walk(stateDir);", "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", "console.log(JSON.stringify({ok:!!report,reason:report?null:(jsonBuf?'report-json-parse-failed':'report-json-missing'),reportReadWaitMs:waitMs,reportParseError:clip(reportParseError,220),reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 55)) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function isSafeSentinelReportStateDir(value: string): boolean { return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..") && !value.startsWith("/"); } function renderSentinelReportSummary(payload: Record, state: SentinelCicdState): string { const run = record(payload.run); const summary = record(payload.summary); const artifactSummary = record(payload.artifactSummary); const offlineReclassify = record(payload.offlineReclassify); const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : []; const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : []; const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256"); const findingCount = run.findingCount ?? findings.length; const analyzerArtifactCount = numberAtNullable(artifactSummary, "artifactCount") ?? numberAtNullable(record(artifactSummary.counts), "artifacts"); const runArtifactCount = numberAtNullable(run, "artifactCount"); const artifactCount = runArtifactCount === null || runArtifactCount === 0 && analyzerArtifactCount !== null ? analyzerArtifactCount ?? "-" : runArtifactCount; const publicOrigin = stringAtNullable(state.publicExposure, "publicBaseUrl") ?? "-"; const findingRows = findings.length === 0 ? "-" : table(["ID", "SEVERITY", "COUNT", "SUMMARY"], findings.slice(0, 12).map((item) => [ stringAtNullable(item, "id") ?? "-", stringAtNullable(item, "severity") ?? "-", item.count ?? "-", reportText(item.summary, 120) ?? "-", ])); return [ "Web Probe Sentinel Quick Verify", "=======================================================", `run=${run.id ?? "-"} scenario=${run.scenarioId ?? "-"} observer=${run.observerId ?? "-"}`, `status=${run.status ?? summary.status ?? "-"} reason=${summary.reason ?? "-"} failure=${summary.failure ?? "-"}`, `report=${reportSha ?? "-"} artifacts=${artifactCount} findings=${findingCount}`, `publicOrigin=${publicOrigin}`, "", "Findings", findingRows, offlineFindings.length === 0 ? "" : "", offlineFindings.length === 0 ? "" : "Offline Reclassify", offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"), ].join("\n"); } function renderSentinelReportFindings(payload: Record): string { const run = record(payload.run); const artifactSummary = record(payload.artifactSummary); const offlineReclassify = record(payload.offlineReclassify); const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : []; const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : []; const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256"); return [ "Web Probe Sentinel Findings", "=======================================================", `run=${run.id ?? "-"} report=${reportSha ?? "-"} findings=${run.findingCount ?? findings.length}`, findings.length === 0 ? "-" : findings.map(formatSentinelReportFindingLine).join("\n"), offlineFindings.length === 0 ? "" : "", offlineFindings.length === 0 ? "" : "Offline Reclassify", offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"), ].join("\n"); } function formatSentinelReportFindingLine(item: Record): string { const check = record(item.check); const code = stringAtNullable(item, "checkCode") ?? stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-"; const title = stringAtNullable(item, "checkTitleZh") ?? stringAtNullable(check, "titleZh") ?? ""; const summary = reportText(item.summary, 180) ?? ""; const rootCause = reportText(item.rootCause, 180); const evidence = reportText(item.evidenceSummary, 180); const nextAction = reportText(item.nextAction, 180); return [ `${item.severity ?? "-"} ${code} ${title} count=${item.count ?? "-"} ${summary}`.trim(), rootCause === null ? null : `rootCause=${rootCause}`, evidence === null ? null : `evidence=${evidence}`, nextAction === null ? null : `next=${nextAction}`, ].filter((part): part is string => part !== null && part.length > 0).join(" | "); } function compactRootCauseSignals(value: unknown): Record | null { const item = record(value); const keys = [ "sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "consoleAlertCount", "requestfailedTop", "httpStatusTop", ]; const out: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; out[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw; } return Object.keys(out).length === 0 ? null : { ...out, valuesRedacted: true }; } function compactSentinelAnalysisWindow(value: unknown): Record | null { const item = record(value); if (Object.keys(item).length === 0) return null; return pickFields(item, ["name", "windowMs", "samples", "control", "network", "console", "valuesRedacted"]); } function reportText(value: unknown, maxChars: number): string | null { if (value === undefined || value === null || value === "") return null; const raw = text(value); return raw.length <= maxChars ? raw : `${raw.slice(0, Math.max(0, maxChars - 1))}…`; } export function runSentinelDashboard(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel dashboard ${options.action}${options.runId === null ? "" : ` --run ${options.runId}`}`; const result = probeSentinelDashboardBrowser(state, options); return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result)); } export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const dashboardUrl = sentinelDashboardUrl(publicBaseUrl, options.runId); const [widthRaw, heightRaw] = options.viewport.split("x"); const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : ""; const script = [ "set -eu", `export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`, `export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`, `export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_TRIGGER=${shellQuote(options.action === "trigger" ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`, `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`, `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`, `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`, `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`, `export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`, `export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`, "export PLAYWRIGHT_BROWSERS_PATH=0", "if command -v chromium >/dev/null 2>&1; then", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)", "elif command -v chromium-browser >/dev/null 2>&1; then", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)", "elif command -v google-chrome >/dev/null 2>&1; then", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)", "else", " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=", "fi", "cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'", sentinelDashboardBrowserModule(), "WEB_PROBE_SENTINEL_DASHBOARD_JS", "node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"", ].join("\n"); const route = `${state.spec.nodeId}:${state.spec.workspace}`; const job = runWebProbeRemoteArtifactJob({ route, localDir: options.localDir, waitTimeoutMs: options.waitTimeoutMs, commandTimeoutMs: options.commandTimeoutSeconds * 1000, inactivityTimeoutMs: 30000, runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`, stdoutTailBytes: 32768, }, script); const result = job.result; const transport = record(job.transport); const remote = record(transport.remote); const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : ""); const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : []; const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null; const browserOk = page?.ok === true; const screenshotOk = options.action !== "screenshot" || screenshot !== null && screenshot.verified === true; const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk; return { ok, status: ok ? "pass" : "blocked", command: `web-probe sentinel dashboard ${options.action}`, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, runId: options.runId, publicUrl: `${publicBaseUrl}/`, route, viewport: options.viewport, page, screenshot, artifacts, artifactCount: artifacts.length, remote: { exitCode: remote.exitCode ?? null, remoteDir: remote.remoteDir ?? null, stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "", stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "", }, transport: { ok: transport.ok ?? null, runId: transport.runId ?? null, artifactCount: transport.artifactCount ?? null, expectedArtifactCount: transport.expectedArtifactCount ?? null, downloadFailure: transport.downloadFailure ?? null, pollFailure: transport.pollFailure ?? null, }, result: compactCommand(result), degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk), valuesRedacted: true, }; } function sentinelDashboardUrl(publicBaseUrl: string, runId: string | null): string { const url = new URL(`${publicBaseUrl.replace(/\/$/u, "")}/`); if (runId !== null && runId.length > 0) { url.searchParams.set("run", runId); url.searchParams.set("focus", "memory"); } return url.toString(); } function sentinelDashboardBrowserModule(): string { return String.raw`import { pathToFileURL } from "node:url"; const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || ""; const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright"; const { chromium } = await import(playwrightModuleSpecifier); const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL; const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || ""; const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1"; const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1"; const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440); const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900); const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000); const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"; const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || ""; const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || ""; const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/"; const requestedRunId = process.env.UNIDESK_SENTINEL_DASHBOARD_RUN_ID || ""; if (!url) throw new Error("missing dashboard URL"); const consoleMessages = []; const pageErrors = []; const requestFailures = []; const browser = await chromium.launch({ headless: true, args: ["--disable-gpu", "--no-sandbox"], ...(executablePath ? { executablePath } : {}), }); const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); const page = await context.newPage(); page.on("console", (message) => { if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) }); }); page.on("pageerror", (error) => { if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) }); }); page.on("requestfailed", (request) => { if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null }); }); let httpStatus = null; let navigationError = null; let navigationAttempts = 0; const maxNavigationAttempts = 2; const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts)); for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) { navigationAttempts = attempt; navigationError = null; try { const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }); httpStatus = response?.status() ?? null; await page.waitForFunction(() => { const root = document.querySelector("#monitor-web-root"); if (!root) return false; const ready = root.getAttribute("data-monitor-ready") === "true"; const error = document.querySelector("#monitor-web-error"); return ready || Boolean(error); }, null, { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {}); await page.waitForTimeout(300); const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false); if (appReady || attempt === maxNavigationAttempts) break; } catch (error) { navigationError = String(error?.message || error).slice(0, 500); if (attempt === maxNavigationAttempts) break; } await page.waitForTimeout(750 * attempt); } let requestedRunSelection = { requestedRunId, ok: requestedRunId.length === 0, reason: requestedRunId.length === 0 ? "not-requested" : "not-attempted" }; if (requestedRunId) { requestedRunSelection = await page.evaluate(async (runId) => { const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms)); const rowForRun = () => document.querySelector('[data-run-id="' + CSS.escape(runId) + '"]'); const row = rowForRun(); if (row instanceof HTMLElement) row.click(); for (let index = 0; index < 80; index += 1) { const selected = rowForRun(); if (selected instanceof HTMLElement) return { requestedRunId: runId, ok: true, reason: "run-row-present" }; await wait(150); } return { requestedRunId: runId, ok: false, reason: "run-row-missing" }; }, requestedRunId).catch((error) => ({ requestedRunId, ok: false, reason: "selection-error", error: String(error?.message || error).slice(0, 300) })); await page.waitForTimeout(150); } let manualTrigger = { requested: triggerManual, ok: !triggerManual, status: triggerManual ? "not-attempted" : "not-requested", jobName: null, statusText: "" }; if (triggerManual) { for (let attempt = 1; attempt <= 2; attempt += 1) { const buttonReady = await page.waitForSelector("[data-monitor-manual-trigger='true']", { timeout: Math.min(20000, timeout) }).then(() => true).catch(() => false); if (buttonReady) break; if (attempt === 1) { await page.reload({ timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }).catch(() => null); await page.waitForFunction(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true", null, { timeout: Math.min(12000, perAttemptTimeout) }).catch(() => {}); } } manualTrigger = await page.evaluate(async () => { const wait = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms)); const root = document.querySelector("[data-monitor-shell='true']"); const button = document.querySelector("[data-monitor-manual-trigger='true']"); if (!(button instanceof HTMLButtonElement)) return { requested: true, ok: false, status: "button-missing", jobName: null, statusText: "" }; if (button.disabled) return { requested: true, ok: false, status: "button-disabled", jobName: button.getAttribute("data-trigger-job") || null, statusText: "" }; button.click(); for (let index = 0; index < 160; index += 1) { const state = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || ""; const jobName = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null; const statusText = String(document.querySelector("[data-monitor-manual-trigger-status='true']")?.textContent || "").replace(/\s+/g, " ").trim(); if (state === "triggered" || state === "already-active" || state === "failed") { return { requested: true, ok: state === "triggered" || state === "already-active", status: state, jobName, statusText }; } await wait(250); } const finalState = root?.getAttribute("data-manual-trigger-state") || button.getAttribute("data-trigger-state") || "timeout"; const finalJob = root?.getAttribute("data-manual-trigger-job") || button.getAttribute("data-trigger-job") || null; return { requested: true, ok: false, status: finalState || "timeout", jobName: finalJob, statusText: "timeout" }; }).catch((error) => ({ requested: true, ok: false, status: "trigger-error", jobName: null, statusText: String(error?.message || error).slice(0, 300) })); await page.waitForTimeout(300); } const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }) => { const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; const objectOrNull = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : null; const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim(); const root = document.querySelector("#monitor-web-root"); const shell = document.querySelector("[data-monitor-shell='true']"); const error = document.querySelector("#monitor-web-error"); const trend = document.querySelector("[data-monitor-trend-curve]"); const trendErrorCurve = document.querySelector("[data-monitor-trend-error-curve='true']"); const trendWarningCurve = document.querySelector("[data-monitor-trend-warning-curve='true']"); const memoryChart = document.querySelector("[data-run-memory-chart='true']"); const manualTriggerButton = document.querySelector("[data-monitor-manual-trigger='true']"); const manualTriggerStatus = document.querySelector("[data-monitor-manual-trigger-status='true']"); const chartTiming = { hasDurationCurve: Boolean(trend), hasErrorCurve: Boolean(trendErrorCurve), hasWarningCurve: Boolean(trendWarningCurve), ok: false, }; chartTiming.ok = chartTiming.hasDurationCurve === true && chartTiming.hasErrorCurve === true && chartTiming.hasWarningCurve === true; const dataset = root ? { node: root.getAttribute("data-node"), lane: root.getAttribute("data-lane"), sentinelId: root.getAttribute("data-sentinel-id"), basePath: root.getAttribute("data-base-path"), contractVersion: root.getAttribute("data-contract-version"), ready: root.getAttribute("data-monitor-ready"), } : {}; const apiUrl = (path) => { const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; const prefix = basePath.replace(/\/+$/u, ""); return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; }; const fetchJson = async (path) => { try { const response = await fetch(apiUrl(path), { cache: "no-store" }); return { ok: response.ok, httpStatus: response.status, body: await response.json().catch(() => null), }; } catch (error) { return { ok: false, httpStatus: null, body: null, error: String(error?.message || error).slice(0, 300), }; } }; const [overviewResult, runsResult] = await Promise.all([ fetchJson("/api/overview"), fetchJson("/api/runs?limit=30&sort=updated"), ]); const overviewPayload = objectOrNull(overviewResult.body) || {}; const runsPayload = objectOrNull(runsResult.body) || {}; const runs = Array.isArray(runsPayload.runs) ? runsPayload.runs : Array.isArray(runsPayload.items) ? runsPayload.items : []; const latestRun = runs[0] || null; const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts) ? latestRun.severityCounts : {}; const latestRunCounts = { runId: latestRun?.id || latestRun?.runId || null, typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count), durationMinutes: numberValue(latestRun?.runDurationMinutes ?? latestRun?.durationMinutes ?? latestRun?.timing?.durationMinutes), severityKeys: Object.keys(latestCounts).sort(), }; const targetRunId = requestedRunId || latestRunCounts.runId; const targetRun = targetRunId ? runs.find((run) => (run?.id || run?.runId) === targetRunId) || latestRun : latestRun; const targetDetailResult = targetRunId === null ? { ok: false, httpStatus: null, body: null } : await fetchJson("/api/runs/" + encodeURIComponent(targetRunId)); const targetDetailPayload = objectOrNull(targetDetailResult.body) || {}; const targetMemory = objectOrNull(targetDetailPayload.memory) || {}; const targetPageSeries = Array.isArray(targetMemory.pageSeries) ? targetMemory.pageSeries : []; const memorySummary = { present: Boolean(memoryChart), runId: memoryChart?.getAttribute("data-memory-run-id") || null, targetRunId, targetScenarioId: targetRun?.scenarioId || targetRun?.scenario_id || null, matchesTargetRun: memoryChart?.getAttribute("data-memory-run-id") === targetRunId, pageCount: numberValue(memoryChart?.getAttribute("data-memory-page-count")), sampleCount: numberValue(memoryChart?.getAttribute("data-memory-sample-count")), source: memoryChart?.getAttribute("data-memory-source") || null, apiOk: targetDetailResult.ok === true && targetDetailPayload.ok !== false, apiPageCount: targetPageSeries.length, apiSampleCount: numberValue(targetMemory.sampleCount), apiSource: targetMemory.source || null, expectedFromApi: targetPageSeries.length > 0 || numberValue(targetMemory.sampleCount) > 0, contractOk: false, status: "unknown", }; memorySummary.contractOk = memorySummary.apiOk === true && (memorySummary.expectedFromApi !== true || ( memorySummary.present === true && memorySummary.matchesTargetRun === true && memorySummary.pageCount === memorySummary.apiPageCount )); memorySummary.status = memorySummary.apiOk !== true ? "api-unavailable" : memorySummary.expectedFromApi === true ? memorySummary.contractOk === true ? "rendered" : "mismatch" : "no-samples"; const datasetSentinelId = String(dataset.sentinelId || ""); const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/"; const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/"; const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/"); const sentinelBoundary = { expectedSentinelId, expectedRoutePrefix, datasetSentinelId, overviewSentinelId: overviewPayload.sentinelId || null, runsSentinelId: runsPayload.sentinelId || null, finalPath, routePrefixMatches, datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true, overviewMatches: expectedSentinelId ? overviewPayload.sentinelId === expectedSentinelId : true, runsPayloadMatches: expectedSentinelId ? runsPayload.sentinelId === expectedSentinelId : true, runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true, }; const doc = document.documentElement; const body = document.body; const viewport = { width: window.innerWidth, height: window.innerHeight }; const documentSize = { width: Math.max(doc.scrollWidth, body?.scrollWidth || 0), height: Math.max(doc.scrollHeight, body?.scrollHeight || 0), }; const overflow = []; let overflowCount = 0; for (const element of Array.from(document.querySelectorAll("body *"))) { const rect = element.getBoundingClientRect(); const overflowRight = rect.right - viewport.width; const overflowLeft = -rect.left; if (overflowRight > 1 || overflowLeft > 1) { overflowCount += 1; if (overflow.length < 5) { overflow.push({ tag: element.tagName.toLowerCase(), x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height), overflowRight: Math.max(0, Math.round(overflowRight)), overflowLeft: Math.max(0, Math.round(overflowLeft)), }); } } } return { shell: Boolean(root && shell), ready: root?.getAttribute("data-monitor-ready") === "true", dataset, contract: { htmlShell: Boolean(root && shell), appReady: root?.getAttribute("data-monitor-ready") === "true", apiOverview: overviewResult.ok === true && overviewPayload.ok !== false, apiRuns: runsResult.ok === true && Array.isArray(runs), runCount: runs.length, latestRunId: latestRunCounts.runId, latestFindingTypeCount: latestRunCounts.typeCount, trendCurves: chartTiming.ok, memoryContract: memorySummary.contractOk, }, sentinelBoundary, title: document.title, finalUrl: window.location.href, runRows: runs.length, checkRows: document.querySelectorAll("[data-check-row='true']").length, latestRunCounts, targetRunCounts: { runId: targetRunId, scenarioId: targetRun?.scenarioId || targetRun?.scenario_id || null, durationMinutes: numberValue(targetRun?.runDurationMinutes ?? targetRun?.durationMinutes ?? targetRun?.timing?.durationMinutes), requested: Boolean(requestedRunId), requestMatched: !requestedRunId || targetRunId === requestedRunId, }, requestedRunSelection, manualTriggerUi: { requested: manualTrigger.requested, ok: manualTrigger.ok, status: manualTrigger.status, jobName: manualTrigger.jobName, buttonPresent: Boolean(manualTriggerButton), buttonDisabled: manualTriggerButton instanceof HTMLButtonElement ? manualTriggerButton.disabled : null, buttonState: manualTriggerButton?.getAttribute("data-trigger-state") || null, statusText: String(manualTriggerStatus?.textContent || "").replace(/\s+/g, " ").trim(), }, chartTiming, memorySummary, api: { overview: { ok: overviewResult.ok, httpStatus: overviewResult.httpStatus }, runs: { ok: runsResult.ok, httpStatus: runsResult.httpStatus }, targetDetail: { ok: targetDetailResult.ok, httpStatus: targetDetailResult.httpStatus }, }, errorVisible: Boolean(error && !error.hidden), errorText: error && !error.hidden ? String(error.textContent || "").replace(/\s+/g, " ").trim().slice(0, 500) : "", layout: { viewport, documentSize, horizontalOverflow: documentSize.width > viewport.width + 1, overflowCount, overflow: overflow.slice(0, 2), }, }; }, { expectedRoutePrefix, expectedSentinelId, requestedRunId, requestedRunSelection, manualTrigger }); if (captureScreenshot && screenshotPath) { if (requestedRunId && dom.memorySummary?.present === true && dom.memorySummary?.matchesTargetRun === true) { await page.evaluate(() => { const chart = document.querySelector("[data-run-memory-chart='true']"); if (chart instanceof HTMLElement) chart.scrollIntoView({ block: "center", inline: "nearest", behavior: "auto" }); }).catch(() => null); await page.waitForTimeout(150); } await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => { pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) }); }); } const consoleErrors = consoleMessages.filter((item) => item.type === "error"); const navigationOk = !navigationError || (dom.shell === true && dom.ready === true); const ok = navigationOk && httpStatus !== null && httpStatus >= 200 && httpStatus < 300 && dom.shell === true && dom.contract?.apiOverview === true && dom.contract?.apiRuns === true && dom.sentinelBoundary?.datasetMatches === true && dom.sentinelBoundary?.overviewMatches === true && dom.sentinelBoundary?.runsPayloadMatches === true && dom.sentinelBoundary?.runRowsMatch === true && dom.sentinelBoundary?.routePrefixMatches === true && dom.errorVisible !== true && dom.requestedRunSelection?.ok === true && manualTrigger.ok === true && dom.chartTiming?.ok === true && dom.memorySummary?.contractOk === true && dom.layout?.horizontalOverflow !== true && pageErrors.length === 0; console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ ok, url, httpStatus, navigationError, navigationAttempts, executablePath: executablePath || null, viewport: { width, height }, screenshotPath: captureScreenshot ? screenshotPath : null, manualTrigger, dom, consoleCount: consoleMessages.length, consoleErrorCount: consoleErrors.length, pageErrorCount: pageErrors.length, requestFailureCount: requestFailures.length, consoleMessages: consoleMessages.slice(0, 8), pageErrors: pageErrors.slice(0, 8), requestFailures: requestFailures.slice(0, 8), valuesRedacted: true, })); await context.close().catch(() => {}); await browser.close().catch(() => {}); `; } function parseDashboardBrowserPayload(textValue: string): Record | null { const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__"; const index = textValue.lastIndexOf(marker); if (index < 0) return null; try { return record(JSON.parse(textValue.slice(index + marker.length).trim())); } catch { return null; } } function dashboardScreenshotName(options: Extract, state: SentinelCicdState): string { const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`; const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120); return safe.endsWith(".png") ? safe : `${safe}.png`; } function compactDashboardArtifact(artifact: Record): Record { const transfer = record(artifact.transfer); return { remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null, localPath: typeof artifact.localPath === "string" ? artifact.localPath : null, bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null, sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null, verified: artifact.verified === true, transfer: Object.keys(transfer).length === 0 ? null : { strategy: transfer.strategy ?? null, transport: transfer.transport ?? null, chunks: transfer.chunks ?? null, elapsedMs: transfer.elapsedMs ?? null, throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null, }, }; } function dashboardDegradedReason(result: CommandResult, transport: Record, page: Record | null, screenshotOk: boolean): string { if (result.timedOut) return "sentinel-dashboard-command-timeout"; if (transport.ok !== true) { const pollFailure = record(transport.pollFailure); if (pollFailure.phase === "timeout") return "sentinel-dashboard-transport-timeout"; if (Object.keys(pollFailure).length > 0) return "sentinel-dashboard-transport-failed"; return "sentinel-dashboard-transport-failed"; } if (page === null) return "sentinel-dashboard-browser-output-missing"; const manualTrigger = record(page.manualTrigger); if (manualTrigger.requested === true && manualTrigger.ok !== true) return "sentinel-dashboard-manual-trigger-failed"; if (page.ok !== true) return "sentinel-dashboard-render-failed"; if (!screenshotOk) return "sentinel-dashboard-screenshot-missing"; return "sentinel-dashboard-unknown"; } function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult { const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; if (releaseId !== null) args.push("--release-id", releaseId); if (reason !== null) args.push("--reason", reason); if (quickVerify) args.push("--quick-verify"); const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`); const command = `web-probe sentinel ${subcommand.join(" ")}`; return rendered(true, command, renderAsyncJobResult({ ok: true, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "async-job", mutation: true, job, next: { status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, wait: ["bun", "scripts/cli.ts", ...args].join(" "), }, valuesRedacted: true, })); } function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { const namespace = stringAt(state.runtime, "namespace"); const serviceName = stringAt(state.runtime, "serviceName"); const servicePort = numberAt(state.runtime, "servicePort"); const deploymentName = stringAt(state.runtime, "deploymentName"); const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`; if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") { return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url); } const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); const postScript = [ "const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');", "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", `const url = 'http://127.0.0.1:${servicePort}' + path;`, "fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {", " const text = await response.text();", " process.stdout.write(text);", " if (!response.ok) process.exit(22);", "}).catch((error) => {", " console.error(error && error.stack ? error.stack : String(error));", " process.exit(23);", "});", ].join(" "); const script = method === "GET" ? `kubectl get --raw ${shellQuote(proxyPath)}` : [ "set -eu", `kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`, ].join("\n"); const maxAttempts = method === "GET" ? 3 : 1; const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); const attempts: Record[] = []; let result: CommandResult | null = null; let parsed: Record | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); parsed = parseJsonObject(result.stdout); const recovered = parsed === null ? recoverTruncatedSshStdoutJson(result) : null; if (recovered !== null) parsed = recovered.parsed; attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, parsedFromDump: recovered !== null, dumpPath: recovered?.dumpPath ?? null, valuesRedacted: true }); if (result.exitCode === 0) break; } const compactBodyJson = compactSentinelServiceBodyJson(parsed); return { ok: result?.exitCode === 0, method, path: pathWithQuery, internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, httpStatus: result?.exitCode === 0 ? 200 : null, bodyJson: record(compactBodyJson), bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "", bodyBytes: Buffer.byteLength(result?.stdout ?? ""), error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000), proxyPath, result: result === null ? null : compactCommand(result), attempts, valuesRedacted: true, }; } function recoverTruncatedSshStdoutJson(result: CommandResult): { parsed: Record; dumpPath: string } | null { const dumpPath = sshStdoutDumpPathFromStderr(result.stderr); if (dumpPath === null || !existsSync(dumpPath)) return null; const parsed = parseJsonObject(readFileSync(dumpPath, "utf8")); return parsed === null ? null : { parsed, dumpPath }; } function sshStdoutDumpPathFromStderr(value: string): string | null { for (const rawLine of value.split(/\r?\n/u)) { const line = rawLine.trim(); const prefix = "UNIDESK_SSH_STDOUT_TRUNCATED "; if (!line.startsWith(prefix)) continue; try { const payload = JSON.parse(line.slice(prefix.length)) as unknown; const dumpPath = stringAtNullable(record(payload), "dumpPath"); if (dumpPath !== null) return dumpPath; } catch { return null; } } return null; } function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number, url: string): Record { const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const fetchScript = [ "const method = process.env.SENTINEL_METHOD || 'GET';", "const url = process.env.SENTINEL_URL || '';", "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", "const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);", "const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);", "const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;", "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", "(async () => {", " let lastError = null;", " for (let attempt = 1; attempt <= attempts; attempt += 1) {", " try {", " const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });", " const text = await response.text();", " process.stdout.write(text);", " if (!response.ok) process.exit(22);", " process.exit(0);", " } catch (error) {", " lastError = error;", " console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));", " if (attempt < attempts && delayMs > 0) await sleep(delayMs);", " }", " }", " console.error(lastError && lastError.stack ? lastError.stack : String(lastError));", " process.exit(23);", "})().catch((error) => {", " console.error(error && error.stack ? error.stack : String(error));", " process.exit(24);", "});", ].join(" "); const attempts = method === "GET" ? 6 : 3; const retryDelayMs = 1000; const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70)); const result = runCommand(["node", "-e", fetchScript], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000, env: { ...process.env, SENTINEL_METHOD: method, SENTINEL_URL: url, SENTINEL_BODY_B64: bodyB64, SENTINEL_ATTEMPTS: String(attempts), SENTINEL_RETRY_DELAY_MS: String(retryDelayMs), }, }); const parsed = parseJsonObject(result.stdout); const compactBodyJson = compactSentinelServiceBodyJson(parsed); return { ok: result.exitCode === 0, method, path: pathWithQuery, internalUrl: url, httpStatus: result.exitCode === 0 ? 200 : null, bodyJson: record(compactBodyJson), bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "", bodyBytes: Buffer.byteLength(result.stdout), error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000), proxyPath: null, result: compactCommand(result), attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }], transport: "direct-service", valuesRedacted: true, }; } function compactSentinelServiceBodyJson(value: Record | null): unknown { if (value === null || typeof value.renderedText !== "string") return value; return { ...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]), run: pickFields(record(value.run), ["id", "runId", "scenario_id", "scenarioId", "status", "node", "lane", "observer_id", "observerId", "state_dir", "stateDir", "report_json_sha256", "reportJsonSha256", "finding_count", "findingCount", "artifact_count", "artifactCount", "maintenance", "created_at", "createdAt", "updated_at", "updatedAt", "startedAt", "finishedAt", "durationMs", "runDurationMs", "durationMinutes", "runDurationMinutes", "durationSource", "scenarioCadence", "scenarioCadenceMinutes", "scenarioMaxRunMinutes", "schedulerIntervalMinutes", "timing"]), summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]), findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [], renderedText: value.renderedText, valuesRedacted: true, }; } function pickFields(value: Record, keys: readonly string[]): Record { const picked: Record = {}; for (const key of keys) { if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key]; } return picked; } function clipTail(value: string, maxChars: number): string { return value.length <= maxChars ? value : value.slice(-maxChars); } function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`; const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000)); const js = [ "const url=process.env.REQ_URL||'';", "const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);", "let out;", "try{", " const controller=new AbortController();", " const timer=setTimeout(()=>controller.abort(), timeoutMs);", " const started=Date.now();", " const res=await fetch(url,{signal:controller.signal});", " const text=await res.text();", " clearTimeout(timer);", " let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}", " out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};", "}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}", "console.log(JSON.stringify(out));", ].join(""); const result = runCommand(["bun", "-e", js], repoRoot, { timeoutMs: timeoutMs + 2000, env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) }, }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, method: "GET", path: pathWithQuery, publicUrl: url, httpStatus: parsed?.httpStatus ?? null, bodyJson: record(parsed?.bodyJson), bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "", bodyBytes: parsed?.bodyBytes ?? null, error: parsed?.error ?? null, result: compactCommand(result), valuesRedacted: true, }; } function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl"); const hostname = stringAt(state.publicExposure, "hostname"); const expectedA = stringAt(state.publicExposure, "expectedA"); const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`; const script = [ "set +e", `host=${shellQuote(hostname)}`, `expected=${shellQuote(expectedA)}`, `url=${shellQuote(probeUrl)}`, "dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)", "headers=$(mktemp)", "body=$(mktemp)", "writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)", "curl_rc=$?", "body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')", "node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'", "const fs=require('node:fs');", "const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);", "const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);", "const status=Number(statusRaw||0);", "const ssl=Number(sslRaw||-1);", "const addrs=dns?dns.split(',').filter(Boolean):[];", "const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();", "const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');", "let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}", "const authCovered=status===401||status===403||status>=200&&status<300;", "const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;", "const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));", "const dnsMatches=addrs.includes(expected);", "console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); const rootUrl = `${publicBaseUrl}/`; const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`; const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`; const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`; const script = [ "set +e", `root_url=${shellQuote(rootUrl)}`, `css_url=${shellQuote(cssUrl)}`, `js_url=${shellQuote(jsUrl)}`, `vue_url=${shellQuote(vueUrl)}`, "root_body=$(mktemp)", "css_body=$(mktemp)", "js_body=$(mktemp)", "vue_body=$(mktemp)", "root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?", "css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?", "js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?", "vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?", "node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'", "const fs=require('node:fs');", "const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);", "function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}", "const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);", "const contractMatch=/data-contract-version=\"([^\"]+)\"/u.exec(root);", "const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&Boolean(contractMatch);", "const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.length>1000;", "const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.length>1000;", "const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.length>80000;", "console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contractVersion:contractMatch?contractMatch[1]:null},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css)},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js)},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue)},valuesRedacted:true}));", "NODE", ].join("\n"); const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); const parsed = parseJsonObject(result.stdout); return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } function renderMaintenanceResult(result: Record): string { const serviceHealth = record(result.serviceHealth); const maintenance = record(result.maintenance); const quickVerify = record(result.quickVerify); const quickVerifyBusiness = record(quickVerify.businessStatus); const planned = record(result.planned); const blocker = record(result.blocker); const next = record(result.next); const maintenanceBody = record(maintenance.bodyJson); const state = record(maintenanceBody.maintenance); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]), "", table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]), "", Object.keys(state).length > 0 ? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]]) : Object.keys(planned).length > 0 ? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]]) : "MAINTENANCE\n-", "", Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), "", "NEXT", ` validate: ${next.validate ?? "-"}`, ` report: ${next.report ?? "-"}`, ` maintenance-stop: ${next.maintenanceStop ?? "-"}`, "", "DISCLOSURE", " maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.", ].join("\n"); } function renderValidateResult(result: Record): string { const health = record(result.serviceHealth); const metrics = record(result.metrics); const report = record(result.report); const publicExposure = record(result.publicExposure); const publicDashboard = record(result.publicDashboard); const quickVerify = record(result.quickVerify); const blocker = record(result.blocker); const next = record(result.next); const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]), "", table(["CHECK", "OK", "DETAIL"], [ ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`], ["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`], ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], ["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`], ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`], ]), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]), "", "NEXT", ` quick-verify: ${next.quickVerify ?? "-"}`, ` report: ${next.report ?? "-"}`, ` maintenance-start: ${next.maintenanceStart ?? "-"}`, "", "DISCLOSURE", " validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.", ].join("\n"); } function renderDashboardResult(result: Record): string { const page = record(result.page); const dom = record(page.dom); const dataset = record(dom.dataset); const contract = record(dom.contract); const sentinelBoundary = record(dom.sentinelBoundary); const api = record(dom.api); const apiOverview = record(api.overview); const apiRuns = record(api.runs); const layout = record(dom.layout); const latestRunCounts = record(dom.latestRunCounts); const targetRunCounts = record(dom.targetRunCounts); const chartTiming = record(dom.chartTiming); const memorySummary = record(dom.memorySummary); const requestedRunSelection = record(dom.requestedRunSelection); const manualTrigger = record(page.manualTrigger); const manualTriggerUi = record(dom.manualTriggerUi); const screenshot = record(result.screenshot); const remote = record(result.remote); const transport = record(result.transport); const pollFailure = record(transport.pollFailure); const degradedReason = result.degradedReason ?? null; return [ String(result.command), "", table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]), "", table(["HTTP", "SHELL", "READY", "API_OVERVIEW", "API_RUNS", "RUN_ROWS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[ page.httpStatus ?? "-", dom.shell, dom.ready, `${apiOverview.ok ?? "-"}/${apiOverview.httpStatus ?? "-"}`, `${apiRuns.ok ?? "-"}/${apiRuns.httpStatus ?? "-"}`, dom.runRows, page.pageErrorCount, page.consoleErrorCount, page.requestFailureCount, ]]), "", table(["CONTRACT", "BASE_PATH", "DATASET_SENTINEL", "OVERVIEW_SENTINEL", "RUNS_SENTINEL", "ROUTE_MATCH"], [[ dataset.contractVersion ?? "-", dataset.basePath ?? "-", sentinelBoundary.datasetSentinelId ?? "-", sentinelBoundary.overviewSentinelId ?? "-", sentinelBoundary.runsSentinelId ?? "-", sentinelBoundary.routePrefixMatches ?? "-", ]]), "", table(["HTML_SHELL", "APP_READY", "API_OVERVIEW_OK", "API_RUNS_OK", "LATEST_RUN", "TYPE_COUNT"], [[ contract.htmlShell ?? "-", contract.appReady ?? "-", contract.apiOverview ?? "-", contract.apiRuns ?? "-", latestRunCounts.runId ?? "-", latestRunCounts.typeCount ?? "-", ]]), "", table(["TARGET_RUN", "REQUESTED", "SELECTED", "TREND_OK", "ERR_CURVE", "WARN_CURVE"], [[ targetRunCounts.runId ?? "-", targetRunCounts.requested ?? "-", requestedRunSelection.ok ?? "-", chartTiming.ok ?? "-", chartTiming.hasErrorCurve ?? "-", chartTiming.hasWarningCurve ?? "-", ]]), "", manualTrigger.requested === true ? table(["TRIGGER", "STATUS", "JOB", "UI_STATE", "UI_TEXT"], [[ manualTrigger.ok ?? "-", manualTrigger.status ?? "-", manualTrigger.jobName ?? "-", manualTriggerUi.buttonState ?? "-", manualTriggerUi.statusText ?? "-", ]]) : "TRIGGER\n-", "", table(["MEMORY_CHART", "MEMORY_RUN", "MEMORY_MATCH", "MEMORY_PAGES", "MEMORY_SAMPLES", "API_PAGES", "API_SAMPLES", "CONTRACT", "STATUS", "MEMORY_SOURCE"], [[ memorySummary.present ?? "-", memorySummary.runId ?? "-", memorySummary.matchesTargetRun ?? "-", memorySummary.pageCount ?? "-", memorySummary.sampleCount ?? "-", memorySummary.apiPageCount ?? "-", memorySummary.apiSampleCount ?? "-", memorySummary.contractOk ?? "-", memorySummary.status ?? "-", memorySummary.source ?? memorySummary.apiSource ?? "-", ]]), "", table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[ result.viewport, `${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`, layout.horizontalOverflow, layout.overflowCount, ]]), "", Object.keys(screenshot).length === 0 ? "SCREENSHOT\n-" : table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]), "", degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT", "POLL_PHASE"], [[degradedReason, remote.exitCode, transport.ok, pollFailure.phase ?? "-"]]), "", "NEXT", ` trigger: bun scripts/cli.ts web-probe sentinel dashboard trigger --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, ` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, ` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, "", "DISCLOSURE", " dashboard verify uses YAML publicExposure, remote browser execution and API/data-contract checks; it does not assert UI copy or CSS class names.", ].join("\n"); } function renderReportResult(result: Record): string { const report = record(result.report); const body = record(report.bodyJson); const run = record(body.run); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]), "", table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]), "", "DISCLOSURE", " report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.", ].join("\n"); }