// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-observe-render module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:9001-10514 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { NodeWebProbeObserveAction, NodeWebProbeObserveOptions, WebObserveIndexEntry } from "./entry"; import { runTransWorkspaceStdinScript } from "./public-exposure"; import { parseJsonObject, record, shellQuote } from "./utils"; import { isSafeWebObserveJobId, isSafeWebObserveStateDir, safeWebObserveSegment } from "./web-observe-scripts"; import { webObserveShort, webObserveText } from "./web-probe-observe"; export function recoverWebObserveAnalyzeTurnDetails(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, analysis: Record | null): Record | null { if (analysis === null || typeof analysis !== "object") return analysis; const sampleMetrics = record(analysis.sampleMetrics); const hasRounds = Array.isArray(sampleMetrics?.rounds) && sampleMetrics.rounds.length > 0; const hasColumns = Array.isArray(sampleMetrics?.turnColumns) && sampleMetrics.turnColumns.length > 0; const columnsHavePageIdentity = hasColumns && (sampleMetrics?.turnColumns as unknown[]).some((item) => { const column = record(item); return typeof column?.pageRole === "string" || typeof column?.pageId === "string"; }); if (hasRounds && hasColumns && columnsHavePageIdentity) return analysis; const reportJsonPath = typeof analysis.reportJsonPath === "string" ? analysis.reportJsonPath : ""; if (!reportJsonPath) return analysis; const archiveMetrics = record(record(analysis.archiveSummary)?.sampleMetrics); const archiveHasTurnEvidence = Number(archiveMetrics?.rounds ?? 0) > 0 || Number(archiveMetrics?.turnColumns ?? 0) > 0 || Number(archiveMetrics?.turnTimingRows ?? 0) > 0; if (!archiveHasTurnEvidence) return analysis; const recoverScript = [ "set -eu", `report_json=${shellQuote(reportJsonPath)}`, "node - \"$report_json\" <<'UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS'", "const fs = require('fs');", "const reportJsonPath = process.argv[2];", "const readJson = (path) => { try { return JSON.parse(fs.readFileSync(path, 'utf8')); } catch { return null; } };", "const report = readJson(reportJsonPath) || {};", "const stdoutJson = readJson(String(reportJsonPath).replace(/\\/analysis\\/report\\.json$/u, '/analysis/analyzer-stdout.json')) || {};", "const objectOrNull = (value) => value && typeof value === 'object' && !Array.isArray(value) ? value : null;", "const firstNonEmptyArray = (...values) => { for (const value of values) if (Array.isArray(value) && value.length > 0) return value; return []; };", "const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);", "const slimRound = (item) => { const v = objectOrNull(item) || {}; return { promptIndex: v.promptIndex ?? null, sampleCount: v.sampleCount ?? null, loadingSamples: v.loadingSamples ?? null, maxLoadingCount: v.maxLoadingCount ?? null, loadingOwnerCount: v.loadingOwnerCount ?? null, lastTotalElapsedSeconds: v.lastTotalElapsedSeconds ?? null, lastRecentUpdateSeconds: v.lastRecentUpdateSeconds ?? null, diagnosticSamples: v.diagnosticSamples ?? null, terminalSamples: v.terminalSamples ?? null, turnTimingTotalElapsedForwardJumpCount: v.turnTimingTotalElapsedForwardJumpCount ?? null, turnTimingTotalElapsedForwardJumpMaxSeconds: v.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null, promptTextHash: clip(v.promptTextHash, 80) }; };", "const slimTurnColumn = (item) => { const v = objectOrNull(item) || {}; return { label: clip(v.label, 24), pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, lastPromptIndex: v.lastPromptIndex ?? null, traceId: clip(v.traceId, 48), firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, source: clip(v.source, 48) }; };", "const rounds = firstNonEmptyArray(stdoutJson.sampleMetrics?.rounds, stdoutJson.sampleMetrics?.roundItems, report.windows?.recent?.sampleMetrics?.rounds, report.sampleMetrics?.rounds).slice(-8).map(slimRound);", "const turnColumns = firstNonEmptyArray(stdoutJson.sampleMetrics?.turnColumns, report.windows?.recent?.sampleMetrics?.turnColumns, report.sampleMetrics?.turnColumns).slice(-12).map(slimTurnColumn);", "console.log(JSON.stringify({ ok: true, rounds, turnColumns }));", "UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS", ].join("\n"); const recovered = parseJsonObject(runTransWorkspaceStdinScript(options.node, spec.workspace, recoverScript, Math.min(options.commandTimeoutSeconds, 30)).stdout); const recoveredRounds = Array.isArray(recovered.rounds) ? recovered.rounds : []; const recoveredColumns = Array.isArray(recovered.turnColumns) ? recovered.turnColumns : []; if (recoveredRounds.length === 0 && recoveredColumns.length === 0) return analysis; const nextMetrics = { ...(sampleMetrics ?? {}) }; if (!hasRounds && recoveredRounds.length > 0) { nextMetrics.roundItems = recoveredRounds; nextMetrics.rounds = recoveredRounds; } if (!hasColumns && recoveredColumns.length > 0) nextMetrics.turnColumns = recoveredColumns; return { ...analysis, sampleMetrics: nextMetrics, recoveredTurnDetails: { rounds: recoveredRounds.length, turnColumns: recoveredColumns.length, source: "analysis-report", valuesRedacted: true } }; } export function withWebObserveAnalyzeRendered(value: Record): RenderedCliResult { return { ok: value.ok !== false, command: typeof value.command === "string" ? value.command : "web-probe observe analyze", contentType: "text/plain", renderedText: renderWebObserveAnalyzeTable(value), }; } export function renderWebObserveAnalyzeTable(value: Record): string { const analysis = record(value.analysis) ?? value; const analyzer = record(analysis?.analyzer); const result = record(value.result); const counts = record(analysis?.counts); const jsonlScope = record(analysis?.jsonlScope); const analysisWindow = record(analysis?.analysisWindow); const archiveSummary = record(analysis?.archiveSummary); const nestedSampleMetrics = record(analysis?.sampleMetrics); const directSampleMetrics = record(value.sampleMetrics); const hasRenderableTurnMetrics = (item: Record | null): boolean => { if (!item) return false; return (Array.isArray(item.roundItems) && item.roundItems.length > 0) || (Array.isArray(item.rounds) && item.rounds.length > 0) || (Array.isArray(item.turnColumns) && item.turnColumns.length > 0); }; const sampleMetrics = hasRenderableTurnMetrics(nestedSampleMetrics) ? nestedSampleMetrics : hasRenderableTurnMetrics(directSampleMetrics) ? directSampleMetrics : nestedSampleMetrics ?? directSampleMetrics; const archiveMetrics = record(archiveSummary?.sampleMetrics); const archivePagePerformance = record(archiveSummary?.pagePerformance); const runtimeAlerts = record(analysis?.runtimeAlerts); const pagePerformance = record(analysis?.pagePerformance); const pagePerformanceSummary = record(pagePerformance?.summary); const nonEmptyRecord = (item: unknown): Record | null => { const valueRecord = record(item); return Object.keys(valueRecord).length > 0 ? valueRecord : null; }; const requestRateSummarySource = nonEmptyRecord(analysis?.requestRate); const requestRateCurve = nonEmptyRecord(analysis?.requestRateCurve) ?? (requestRateSummarySource?.summary ? requestRateSummarySource : null); const requestRateSummary = nonEmptyRecord(requestRateCurve?.summary) ?? requestRateSummarySource; const requestRatePeaks = webObserveArray(analysis?.requestRatePeaks ?? requestRateCurve?.peaks ?? requestRateSummarySource?.peaks).map(record).filter((item): item is Record => item !== null).slice(0, 8); const requestRatePageCurves = webObserveArray(requestRateCurve?.pageCurves ?? requestRateSummarySource?.pageCurves).map(record).filter((item): item is Record => item !== null).slice(0, 8); const requestRateApiPathCurves = webObserveArray(requestRateCurve?.apiPathCurves ?? requestRateSummarySource?.apiPathCurves).map(record).filter((item): item is Record => item !== null).slice(0, 8); const projectManagement = record(analysis?.projectManagement) ?? record(value.projectManagement); const projectManagementSummary = record(projectManagement?.summary) ?? projectManagement; const projectManagementCommandsSource = Array.isArray(projectManagement?.launchCommands) && projectManagement.launchCommands.length > 0 ? projectManagement.launchCommands : projectManagement?.commands; const projectManagementCommands = webObserveArray(projectManagementCommandsSource).map(record).filter((item): item is Record => item !== null).slice(-8); const projectManagementSamples = webObserveArray(projectManagement?.samples).map(record).filter((item): item is Record => item !== null).slice(-8); const projectManagementApiByPath = webObserveArray(projectManagement?.projectApiByPath).map(record).filter((item): item is Record => item !== null).slice(0, 8); const projectManagementSlowApis = webObserveArray(projectManagement?.slowProjectApiPerformance).map(record).filter((item): item is Record => item !== null).slice(0, 8); const alertThresholds = record(analysis?.alertThresholds ?? pagePerformanceSummary?.alertThresholds ?? value.alertThresholds); const budgetLabel = (rawValue: unknown): string => { const parsed = Number(rawValue); if (!Number.isFinite(parsed) || parsed <= 0) return "unconfigured"; const ms = parsed; return ms % 1000 === 0 ? `${Math.round(ms / 1000)}s` : `${ms}ms`; }; const sameOriginApiBudgetLabel = budgetLabel(alertThresholds?.sameOriginApiSlowMs ?? pagePerformanceSummary?.budgetMs); const projectManagementApiBudgetLabel = budgetLabel(projectManagementSummary?.slowApiBudgetMs ?? alertThresholds?.sameOriginApiSlowMs ?? pagePerformanceSummary?.budgetMs); const partialApiBudgetLabel = budgetLabel(alertThresholds?.partialApiSlowMs); const streamOpenBudgetLabel = budgetLabel(alertThresholds?.longLivedStreamOpenSlowMs); const loadingBudgetLabel = budgetLabel(alertThresholds?.visibleLoadingSlowMs); const pageLabel = (item: Record): string => { const role = webObserveText(item.pageRole); const pageId = webObserveShort(webObserveText(item.pageId), 18); const epoch = webObserveText(item.pageEpoch); if (role === "-" && pageId === "-") return "-"; return epoch === "-" ? `${role}/${pageId}` : `${role}/${pageId}#${epoch}`; }; const promptNetwork = record(analysis?.promptNetwork); const loading = record(sampleMetrics?.loading); const loadingSummary = record(loading?.summary); const traceOrder = record(sampleMetrics?.traceOrder); const traceOrderSummary = record(traceOrder?.summary); const traceOrderAnomalies = Array.isArray(traceOrder?.orderAnomalies) ? traceOrder.orderAnomalies.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const loadingSegments = Array.isArray(loading?.longestSegments) ? loading.longestSegments.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const loadingOwners = Array.isArray(loading?.owners) ? loading.owners.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const sessionRailTitles = record(sampleMetrics?.sessionRailTitles); const sessionRailTitleSummary = record(sessionRailTitles?.summary); const sessionRailTitleSamples = Array.isArray(sessionRailTitles?.samples) ? sessionRailTitles.samples.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const sessionRailTitleExamples = Array.isArray(sessionRailTitles?.examples) ? sessionRailTitles.examples.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const roundSource = Array.isArray(sampleMetrics?.roundItems) && sampleMetrics.roundItems.length > 0 ? sampleMetrics.roundItems : Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds : sampleMetrics?.roundItems ?? sampleMetrics?.rounds; const rounds = webObserveArray(roundSource).map(record).filter((item): item is Record => item !== null).slice(-8); const turnColumns = webObserveArray(sampleMetrics?.turnColumns).map(record).filter((item): item is Record => item !== null).slice(-12); const roundCount = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.length : sampleMetrics?.rounds; const turnColumnCount = Array.isArray(sampleMetrics?.turnColumns) ? sampleMetrics.turnColumns.length : sampleMetrics?.turnColumns; const slowApis = Array.isArray(analysis?.pagePerformanceSlowApi) ? analysis.pagePerformanceSlowApi.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const archiveSlowApis = Array.isArray(analysis?.archivePagePerformanceSlowApi) ? analysis.archivePagePerformanceSlowApi.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const partialApis = Array.isArray(analysis?.pagePerformancePartialApi) ? analysis.pagePerformancePartialApi.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const sseStreams = Array.isArray(analysis?.pagePerformanceSseStreams) ? analysis.pagePerformanceSseStreams.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const allFindingRecords = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record => item !== null) : []; const explicitToolFindings = Array.isArray(analysis?.toolFindings) ? analysis.toolFindings.map(record).filter((item): item is Record => item !== null) : []; const inferredToolFindings = allFindingRecords.filter((item) => String(item.id ?? item.kind ?? item.code ?? "").startsWith("tool-")); const toolFindings = dedupeWebObserveRecords(explicitToolFindings.length > 0 ? explicitToolFindings : inferredToolFindings, (item) => String(item.id ?? item.kind ?? item.code ?? "")).slice(0, 8); const commandState = record(analysis?.commandState); const findings = allFindingRecords.slice(0, 8); const archiveRedFindings = Array.isArray(analysis?.archiveRedFindings) ? analysis.archiveRedFindings.map(record).filter((item): item is Record => item !== null).slice(0, 8) : Array.isArray(archiveSummary?.redFindings) ? archiveSummary.redFindings.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const httpErrorGroups = Array.isArray(analysis?.httpErrorGroups) ? analysis.httpErrorGroups.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const requestFailedGroups = Array.isArray(analysis?.requestFailedGroups) ? analysis.requestFailedGroups.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const domDiagnostics = Array.isArray(analysis?.domDiagnosticGroups) ? analysis.domDiagnosticGroups.map(record).filter((item): item is Record => item !== null).slice(0, 5) : []; const domDiagnosticSamples = Array.isArray(analysis?.domDiagnosticSamples) ? analysis.domDiagnosticSamples.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const consoleAlertGroups = Array.isArray(analysis?.consoleAlertGroups) ? analysis.consoleAlertGroups.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const consoleAlertSamples = Array.isArray(analysis?.consoleAlertSamples) ? analysis.consoleAlertSamples.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const runnerErrors = Array.isArray(analysis?.runnerErrors) ? analysis.runnerErrors.map(record).filter((item): item is Record => item !== null).slice(-8) : []; const commandFailures = Array.isArray(analysis?.commandFailures) ? analysis.commandFailures.map(record).filter((item): item is Record => item !== null).slice(-8) : []; const recentUpdateJumps = Array.isArray(analysis?.turnTimingRecentUpdateJumps) ? analysis.turnTimingRecentUpdateJumps.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const elapsedZeroResets = Array.isArray(analysis?.turnTimingElapsedZeroResets) ? analysis.turnTimingElapsedZeroResets.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const elapsedForwardJumps = Array.isArray(analysis?.turnTimingTotalElapsedForwardJumps) ? analysis.turnTimingTotalElapsedForwardJumps.map(record).filter((item): item is Record => item !== null).slice(0, 8) : []; const domDiagnosticRows = domDiagnostics.length > 0 ? domDiagnostics.map((item) => [ webObserveText(item.count), webObserveShort(webObserveText(item.firstAt), 24), webObserveShort(webObserveText(item.lastAt), 24), webObserveShort(webObserveText(item.text), 96), ]) : domDiagnosticSamples.map((item) => [ webObserveText(item.seq), webObserveShort(webObserveText(item.ts), 24), webObserveShort([ webObserveText(item.source), webObserveText(item.diagnosticCode), item.traceId ? `trace=${webObserveText(item.traceId)}` : "", item.httpStatus !== undefined && item.httpStatus !== null ? `http=${webObserveText(item.httpStatus)}` : "", item.idleSeconds !== undefined && item.idleSeconds !== null ? `idle=${webObserveText(item.idleSeconds)}` : "", item.waitingFor ? `waitingFor=${webObserveText(item.waitingFor)}` : "", item.lastEventLabel ? `lastEvent=${webObserveText(item.lastEventLabel)}` : "", ].filter((part) => part !== "" && part !== "-").join(" "), 64), webObserveShort(webObserveText(item.text), 96), ]); const consoleAlertRows = consoleAlertGroups.length > 0 ? consoleAlertGroups.map((item) => [ webObserveText(item.count), webObserveShort(webObserveText(item.type), 14), webObserveText(item.status), webObserveShort(webObserveText(item.path), 52), webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24), webObserveShort(webObserveArray(item.traceIds).join(",") || "-", 28), ]) : consoleAlertSamples.map((item) => [ webObserveShort(webObserveText(item.ts), 24), webObserveShort(webObserveText(item.type), 14), webObserveText(item.status), webObserveShort(webObserveText(item.path), 52), webObserveShort(webObserveText(item.traceId), 28), webObserveShort(webObserveText(item.text), 72), ]); const recentUpdateJumpRows = recentUpdateJumps.map((item) => [ webObserveShort(webObserveText(item.columnLabel), 14), pageLabel(item), webObserveText(item.promptIndex), webObserveText(item.fromSeq), webObserveText(item.toSeq), `${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`, webObserveText(item.delta), webObserveText(item.sampleDeltaSeconds), webObserveText(item.allowedIncreaseSeconds), webObserveShort(webObserveText(item.traceId), 24), ]); const elapsedZeroResetRows = elapsedZeroResets.map((item) => [ webObserveShort(webObserveText(item.columnLabel), 14), pageLabel(item), webObserveText(item.promptIndex), webObserveText(item.fromSeq), webObserveText(item.toSeq), `${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`, webObserveText(item.delta), webObserveShort(webObserveText(item.traceId), 24), ]); const elapsedForwardJumpRows = elapsedForwardJumps.map((item) => [ webObserveShort(webObserveText(item.columnLabel), 14), pageLabel(item), webObserveText(item.promptIndex), webObserveText(item.fromSeq), webObserveText(item.toSeq), `${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`, webObserveText(item.delta), webObserveText(item.sampleDeltaSeconds), webObserveText(item.allowedIncreaseSeconds), webObserveShort(webObserveText(item.traceId), 24), ]); const httpErrorRows = httpErrorGroups.map((item) => [ webObserveText(item.count), webObserveText(item.method), webObserveText(item.status), webObserveShort(webObserveText(item.path), 52), webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24), webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40), ]); const requestFailedRows = requestFailedGroups.map((item) => [ webObserveText(item.count), webObserveText(item.method), webObserveText(item.status), webObserveShort(webObserveText(item.path), 52), webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24), webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40), ]); const projectManagementCommandRows = projectManagementCommands.map((item) => [ webObserveShort(webObserveText(item.ts), 24), webObserveShort(webObserveText(item.phase), 12), webObserveShort(webObserveText(item.type), 26), webObserveText(item.launchStatus ?? item.status), webObserveShort(webObserveText(item.sessionId), 24), webObserveShort(webObserveText(item.otelTraceId), 18), webObserveShort(webObserveText(item.selectedTaskRefHash ?? item.taskHash), 18), webObserveShort(webObserveText(item.workbenchUrl ?? item.afterPath), 44), webObserveShort(webObserveText(item.message ?? item.errorMessageHash), 72), ]); const projectManagementSampleRows = projectManagementSamples.map((item) => [ webObserveText(item.seq), webObserveShort(webObserveText(item.ts), 24), webObserveShort(webObserveText(item.pageRole), 12), webObserveShort(webObserveText(item.pageKind), 28), webObserveShort(webObserveText(item.path), 36), webObserveText(item.sourceCount), webObserveText(item.fileCount), webObserveText(item.taskCount), webObserveShort(webObserveText(item.selectedTaskRefHash), 18), webObserveText(item.launchButtonEnabled), webObserveText(item.workbenchLinkCount), ]); const projectManagementApiRows = projectManagementApiByPath.map((item) => [ webObserveText(item.count ?? item.sampleCount), webObserveShort(webObserveText(item.type), 12), webObserveText(item.method), webObserveText(item.status), webObserveShort(webObserveText(item.path ?? item.urlPath), 52), webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24), webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40), ]); const projectManagementSlowRows = projectManagementSlowApis.map((item) => [ webObserveShort(webObserveText(item.path ?? item.route), 52), webObserveText(item.sampleCount), webObserveText(item.p95Ms ?? item.p95), webObserveText(item.maxMs ?? item.max), webObserveText(item.overBudgetCount ?? item.overFiveSecondCount), webObserveShort(webObserveArray(item.slowSamples).map((sample) => webObserveText(record(sample)?.otelTraceId)).filter((text) => text !== "-").join(",") || "-", 36), ]); const lines = [ `web-probe observe analyze (${webObserveText(value.status)})`, "", ...(value.ok === false ? [ "Blocked detail:", webObserveTable(["EXIT", "TIMEOUT", "ANALYZER", "STDOUT", "STDERR"], [[ webObserveText(result.exitCode), webObserveText(result.timedOut), webObserveShort([ analyzer?.recoveredFrom ? `source=${webObserveText(analyzer.recoveredFrom)}` : "", analyzer?.exitCode !== undefined && analyzer?.exitCode !== null ? `exit=${webObserveText(analyzer.exitCode)}` : "", analyzer?.stdoutBytes !== undefined && analyzer?.stdoutBytes !== null ? `stdout=${webObserveText(analyzer.stdoutBytes)}B` : "", analyzer?.stderrBytes !== undefined && analyzer?.stderrBytes !== null ? `stderr=${webObserveText(analyzer.stderrBytes)}B` : "", ].filter((part) => part !== "").join(" "), 120), webObserveShort(webObserveText(result.stdoutTail), 120), webObserveShort(webObserveText(analyzer?.stderrTail ?? result.stderr), 500), ]]), "Analyzer artifacts:", webObserveTable(["KIND", "PATH"], [ ["stdout", webObserveShort(webObserveText(analyzer?.stdoutPath), 96)], ["stderr", webObserveShort(webObserveText(analyzer?.stderrPath), 96)], ]), "", ] : []), webObserveTable(["ID", "NODE", "LANE", "SCOPE", "SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS"], [[ webObserveText(value.id), webObserveText(value.node), webObserveText(value.lane), webObserveShort([webObserveText(jsonlScope?.mode ?? "current"), webObserveText(record(jsonlScope?.focus)?.mode), jsonlScope?.archivePrefix ? webObserveText(jsonlScope.archivePrefix) : ""].filter((part) => part && part !== "-").join(":"), 36), webObserveText(counts?.samples), webObserveText(counts?.control), webObserveText(counts?.network), webObserveText(counts?.console), webObserveText(counts?.artifacts), ]]), "", "Analysis window:", webObserveTable(["WINDOW", "FROM", "TO", "SAMPLES", "NETWORK", "CONSOLE", "ARCHIVE_SAMPLES", "ARCHIVE_RED"], [[ webObserveText(analysisWindow?.name ?? "recent-5m"), webObserveShort(webObserveText(analysisWindow?.fromAt), 24), webObserveShort(webObserveText(analysisWindow?.toAt), 24), webObserveText(analysisWindow?.samples), webObserveText(analysisWindow?.network), webObserveText(analysisWindow?.console), webObserveText(record(archiveSummary?.sampleMetrics)?.sampleCount ?? counts?.samples), webObserveText(archiveSummary?.redFindingCount), ]]), "", "Reports:", webObserveTable(["KIND", "PATH", "SHA256"], [ ["json", webObserveShort(webObserveText(analysis?.reportJsonPath), 96), webObserveShort(webObserveText(analysis?.reportJsonSha256), 24)], ["md", webObserveShort(webObserveText(analysis?.reportMdPath), 96), webObserveShort(webObserveText(analysis?.reportMdSha256), 24)], ]), "", "Project management:", webObserveTable(["ENABLED", "SAMPLES", "MDTODO", "LATEST", "SRC", "FILES", "TASKS", "SELECTED_TASK", "LAUNCH", "OTEL", "API", `SLOW>${projectManagementApiBudgetLabel}`], [[ webObserveText(projectManagementSummary?.enabled), webObserveText(projectManagementSummary?.projectSampleCount), webObserveText(projectManagementSummary?.mdtodoSampleCount), webObserveShort([webObserveText(projectManagementSummary?.latestPageKind), webObserveText(projectManagementSummary?.latestPath)].filter((part) => part !== "-" && part !== "").join(" "), 52), webObserveText(projectManagementSummary?.latestSourceCount), webObserveText(projectManagementSummary?.latestFileCount), webObserveText(projectManagementSummary?.latestTaskCount), webObserveShort(webObserveText(projectManagementSummary?.latestSelectedTaskRefHash), 18), `ok=${webObserveText(projectManagementSummary?.launchSuccessCount)} fail=${webObserveText(projectManagementSummary?.launchFailureCount)} total=${webObserveText(projectManagementSummary?.launchCommandCount)}`, `${webObserveText(projectManagementSummary?.launchWithOtelTraceHeaderCount)}/${webObserveText(projectManagementSummary?.launchSuccessCount)}`, `resp=${webObserveText(projectManagementSummary?.projectApiResponseCount)} fail=${webObserveText(projectManagementSummary?.projectApiFailureCount ?? 0)}/${webObserveText(projectManagementSummary?.projectApiRequestFailedCount ?? 0)}`, webObserveText(projectManagementSummary?.projectApiSlowPathCount), ]]), "", "Project management samples:", webObserveTable(["SEQ", "TS", "ROLE", "KIND", "PATH", "SRC", "FILES", "TASKS", "SELECTED", "LAUNCH", "LINKS"], projectManagementSampleRows.length > 0 ? projectManagementSampleRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Project management commands:", webObserveTable(["TS", "PHASE", "TYPE", "STATUS", "SESSION", "OTEL", "TASK", "URL", "MESSAGE"], projectManagementCommandRows.length > 0 ? projectManagementCommandRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Project management API:", webObserveTable(["COUNT", "TYPE", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], projectManagementApiRows.length > 0 ? projectManagementApiRows : [["-", "-", "-", "-", "-", "-", "-"]]), "", `Project management slow API (>${projectManagementApiBudgetLabel}):`, webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET", "OTEL"], projectManagementSlowRows.length > 0 ? projectManagementSlowRows : [["-", "-", "-", "-", "-", "-"]]), "", "Turn timing:", webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "ZERO_RESET", "ELAPSED_JUMP", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[ webObserveText(roundCount), webObserveText(turnColumnCount), webObserveText(sampleMetrics?.turnTimingRows), webObserveText(sampleMetrics?.turnTimingNonMonotonicCount), webObserveText(sampleMetrics?.turnTimingTotalElapsedDecreaseCount), webObserveText(sampleMetrics?.turnTimingTotalElapsedZeroResetCount ?? elapsedZeroResets.length), webObserveText(sampleMetrics?.turnTimingTotalElapsedForwardJumpCount ?? elapsedForwardJumps.length), webObserveText(sampleMetrics?.turnTimingTerminalElapsedGrowthCount), webObserveText(sampleMetrics?.turnTimingRecentUpdateJumpCount ?? sampleMetrics?.turnTimingRecentUpdateSawtoothJumpCount), webObserveText(sampleMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds), ]]), "", "Trace row order:", webObserveTable(["SAMPLES", "TRACE_ROWS", "ORDER_ANOMALY", "COMPLETION_NOT_LAST"], [[ webObserveText(traceOrderSummary?.sampleCount), webObserveText(traceOrderSummary?.traceRowCount), webObserveText(traceOrderSummary?.orderAnomalyCount), webObserveText(traceOrderSummary?.completionNotLastCount), ]]), "", "Trace row order anomalies:", traceOrderAnomalies.length === 0 ? "-" : webObserveTable(["SEQ", "PAGE", "TRACE", "ROWS", "REASONS", "TOTAL", "CLOCK", "PREVIEW"], traceOrderAnomalies.map((item) => [ webObserveText(item.sampleIndex ?? item.seq), pageLabel(item), webObserveShort(webObserveText(item.traceId), 24), `${webObserveText(item.previousRowIndex)}->${webObserveText(item.currentRowIndex)}`, webObserveShort(webObserveArray(item.reasons).map(webObserveText).join(",") || "-", 48), `${webObserveText(item.previousTotalSeconds)}->${webObserveText(item.currentTotalSeconds)}`, `${webObserveText(item.previousClockSeconds)}->${webObserveText(item.currentClockSeconds)}`, webObserveShort(`${webObserveText(item.previousPreview)} / ${webObserveText(item.currentPreview)}`, 96), ])), "", "Archive turn timing:", webObserveTable(["SAMPLES", "ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "ZERO_RESET", "ELAPSED_JUMP", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[ webObserveText(archiveMetrics?.sampleCount), webObserveText(archiveMetrics?.rounds), webObserveText(archiveMetrics?.turnColumns), webObserveText(archiveMetrics?.turnTimingRows), webObserveText(archiveMetrics?.turnTimingNonMonotonicCount), webObserveText(archiveMetrics?.turnTimingTotalElapsedDecreaseCount), webObserveText(archiveMetrics?.turnTimingTotalElapsedZeroResetCount), webObserveText(archiveMetrics?.turnTimingTotalElapsedForwardJumpCount), webObserveText(archiveMetrics?.turnTimingTerminalElapsedGrowthCount), webObserveText(archiveMetrics?.turnTimingRecentUpdateJumpCount ?? archiveMetrics?.turnTimingRecentUpdateSawtoothJumpCount), webObserveText(archiveMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds), ]]), "", "Rounds:", webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "CARD_DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [ webObserveText(item.promptIndex), webObserveText(item.sampleCount), webObserveText(item.loadingSamples), webObserveText(item.maxLoadingCount), webObserveText(item.lastTotalElapsedSeconds), webObserveText(item.lastRecentUpdateSeconds), webObserveText(item.diagnosticSamples), webObserveText(item.terminalSamples), webObserveShort(webObserveText(item.promptTextHash), 24), ]) : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Loading visibility:", webObserveTable(["SAMPLES", "LOADING_SAMPLES", "MAX_COUNT", "MAX_OWNERS", "OWNERS", "LONGEST_S", "CURRENT_S", `OVER_${loadingBudgetLabel}`], [[ webObserveText(loadingSummary?.sampleCount ?? sampleMetrics?.sampleCount), webObserveText(loadingSummary?.loadingSampleCount ?? sampleMetrics?.loadingSampleCount), webObserveText(loadingSummary?.maxSimultaneousCount ?? sampleMetrics?.loadingMaxCount), webObserveText(loadingSummary?.maxSimultaneousOwnerCount ?? sampleMetrics?.loadingMaxOwnerCount), webObserveText(loadingSummary?.ownerCount ?? sampleMetrics?.loadingOwnerCount), webObserveText(loadingSummary?.longestContinuousSeconds ?? sampleMetrics?.loadingLongestContinuousSeconds), webObserveText(loadingSummary?.currentContinuousSeconds ?? sampleMetrics?.loadingCurrentContinuousSeconds), webObserveText(loadingSummary?.overBudgetSegmentCount ?? loadingSummary?.overFiveSecondSegmentCount ?? sampleMetrics?.loadingOverFiveSecondSegmentCount), ]]), "", "Session rail titles:", webObserveTable(["SAMPLES", "VISIBLE", "MAJORITY", "MAX_RATIO", "MAX_VISIBLE", "MAX_FALLBACK"], [[ webObserveText(sessionRailTitleSummary?.sampleCount ?? sampleMetrics?.sessionRailSampleCount ?? archiveMetrics?.sessionRailSampleCount), webObserveText(sessionRailTitleSummary?.visibleSampleCount ?? sampleMetrics?.sessionRailVisibleSampleCount ?? archiveMetrics?.sessionRailVisibleSampleCount), webObserveText(sessionRailTitleSummary?.majorityFallbackSampleCount ?? sampleMetrics?.sessionRailFallbackMajoritySampleCount ?? archiveMetrics?.sessionRailFallbackMajoritySampleCount), webObserveText(sessionRailTitleSummary?.maxFallbackRatio ?? sampleMetrics?.sessionRailFallbackMaxRatio ?? archiveMetrics?.sessionRailFallbackMaxRatio), webObserveText(sessionRailTitleSummary?.maxVisibleCount ?? sampleMetrics?.sessionRailFallbackMaxVisibleCount ?? archiveMetrics?.sessionRailFallbackMaxVisibleCount), webObserveText(sessionRailTitleSummary?.maxFallbackTitleCount ?? sampleMetrics?.sessionRailFallbackMaxCount ?? archiveMetrics?.sessionRailFallbackMaxCount), ]]), sessionRailTitleSamples.length === 0 ? "Session rail fallback samples:\n-" : webObserveTable(["SEQ", "ROLE", "VISIBLE", "FALLBACK", "RATIO", "EXAMPLE"], sessionRailTitleSamples.map((item) => [ webObserveText(item.seq), webObserveText(item.pageRole), webObserveText(item.visibleCount), webObserveText(item.fallbackTitleCount), webObserveText(item.fallbackTitleRatio), webObserveShort(webObserveText((Array.isArray(item.examples) ? record(item.examples[0])?.titlePreview : null) ?? ""), 48), ])), sessionRailTitleExamples.length === 0 ? "Session rail fallback examples:\n-" : webObserveTable(["SEQ", "ROLE", "ACTIVE", "SESSION", "TITLE"], sessionRailTitleExamples.map((item) => [ webObserveText(item.firstSeq), webObserveText(item.pageRole), webObserveText(item.active), webObserveText(item.sessionIdPrefix), webObserveShort(webObserveText(item.titlePreview ?? item.titleHash), 64), ])), "", "Loading owners:", webObserveTable(["SAMPLES", "OCCUR", "MAX", "LONGEST_S", "KIND", "TRACE", "MESSAGE", "SESSION", "OWNER"], loadingOwners.length > 0 ? loadingOwners.map((item) => [ webObserveText(item.sampleCount), webObserveText(item.occurrenceCount), webObserveText(item.maxSimultaneousCount), webObserveText(item.longestContinuousSeconds), webObserveShort(webObserveText(item.ownerKind), 16), webObserveShort(webObserveText(item.ownerTraceId), 24), webObserveShort(webObserveText(item.ownerMessageId), 24), webObserveShort(webObserveText(item.ownerSessionId), 24), webObserveShort(webObserveText(item.ownerLabel ?? item.ownerKey), 64), ]) : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Loading segments:", webObserveTable(["OBSERVED_S", "UPPER_S", "SAMPLES", "MAX", "OWNERS", "SEQ", "LAST", "STATUS"], loadingSegments.length > 0 ? loadingSegments.map((item) => [ webObserveText(item.durationSeconds), webObserveText(item.upperBoundSeconds ?? item.durationSeconds), webObserveText(item.sampleCount), webObserveText(item.maxCount), webObserveText(item.ownerCount), `${webObserveText(item.firstSeq)}..${webObserveText(item.lastSeq)}`, webObserveShort(webObserveText(item.lastAt), 24), item.ongoing === true ? "ongoing" : webObserveShort(webObserveText(item.endedAt), 24), ]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Turn columns:", webObserveTable(["TURN", "PAGE", "PROMPT", "TRACE", "FIRST_SEQ", "LAST_SEQ", "SOURCE"], turnColumns.length > 0 ? turnColumns.map((item) => [ webObserveText(item.label), pageLabel(item), webObserveText(item.promptIndex ?? item.lastPromptIndex), webObserveShort(webObserveText(item.traceId), 28), webObserveText(item.firstSeq), webObserveText(item.lastSeq), webObserveShort(webObserveText(item.source), 24), ]) : [["-", "-", "-", "-", "-", "-", "-"]]), "", "Recent update jumps:", webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "SAMPLE_S", "ALLOWED", "TRACE"], recentUpdateJumpRows.length > 0 ? recentUpdateJumpRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Total elapsed zero resets:", webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "TRACE"], elapsedZeroResetRows.length > 0 ? elapsedZeroResetRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Total elapsed forward jumps:", webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "SAMPLE_S", "ALLOWED", "TRACE"], elapsedForwardJumpRows.length > 0 ? elapsedForwardJumpRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Runtime alerts:", webObserveTable(["HTTP_ERR", "REQUEST_FAILED", "DOM_DIAG", "CONSOLE_ALERT", "PROMPT_SEGMENTS", "PROMPT_NETWORK"], [[ webObserveText(runtimeAlerts?.httpErrorCount), webObserveText(runtimeAlerts?.requestFailedCount), webObserveText(runtimeAlerts?.domDiagnosticSampleCount), webObserveText(runtimeAlerts?.consoleAlertCount), webObserveText(sampleMetrics?.promptSegments), webObserveText(promptNetwork?.promptSegments), ]]), "", "Request rate:", webObserveTable(["BUCKET", "REQUESTS", "TOTAL_PEAK", "PAGE_PEAK", "API_PEAK", "OVER_PEAKS", "THRESHOLDS"], [[ requestRateSummary?.bucketSeconds ? `${webObserveText(requestRateSummary.bucketSeconds)}s` : webObserveText(requestRateSummary?.bucketMs), webObserveText(requestRateSummary?.requestCount), webObserveText(requestRateSummary?.totalPeakPerMinute), webObserveText(requestRateSummary?.pagePeakPerMinute), webObserveText(requestRateSummary?.apiPathPeakPerMinute), webObserveText(requestRateSummary?.overThresholdPeakCount), `total=${webObserveText(requestRateSummary?.totalRedPerMinute)} page=${webObserveText(requestRateSummary?.pageRedPerMinute)} api=${webObserveText(requestRateSummary?.apiPathRedPerMinute)}`, ]]), requestRatePeaks.length === 0 ? "Request rate peaks:\n-" : webObserveTable(["SCOPE", "RPM", "COUNT", "START", "PAGE", "API"], requestRatePeaks.map((item) => [ webObserveText(item.scope), webObserveText(item.requestPerMinute), webObserveText(item.count), webObserveShort(webObserveText(item.startAt), 24), webObserveShort(webObserveText(item.path ?? item.pageKey), 48), webObserveShort(webObserveText(item.apiKey), 64), ])), "Request page curves:", webObserveTable(["PAGE", "COUNT", "PEAK_RPM", "PEAK_AT"], requestRatePageCurves.length > 0 ? requestRatePageCurves.map((item) => [ webObserveShort(webObserveText(item.path ?? item.pageKey), 56), webObserveText(item.count), webObserveText(item.peakRequestPerMinute), webObserveShort(webObserveText(record(item.peakBucket)?.startAt), 24), ]) : [["-", "-", "-", "-"]]), "Request API curves:", webObserveTable(["API", "COUNT", "PEAK_RPM", "PEAK_AT"], requestRateApiPathCurves.length > 0 ? requestRateApiPathCurves.map((item) => [ webObserveShort(webObserveText(item.apiKey ?? item.path), 72), webObserveText(item.count), webObserveText(item.peakRequestPerMinute), webObserveShort(webObserveText(record(item.peakBucket)?.startAt), 24), ]) : [["-", "-", "-", "-"]]), "", "Runner errors:", webObserveTable(["TS", "TYPE", "RETRY", "EXH", "ATTEMPTS", "LAST_KIND", "READY", "MESSAGE"], runnerErrors.length > 0 ? runnerErrors.map((item) => [ webObserveShort(webObserveText(item.ts), 24), webObserveShort(webObserveText(item.type), 18), webObserveText(item.retry), webObserveText(item.retryExhausted), webObserveText(item.attemptCount), webObserveShort(webObserveText(item.lastFailureKind), 24), webObserveShort(webObserveText(item.lastReadinessReason), 24), webObserveShort(webObserveText(item.message || item.lastError), 96), ]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Command failures:", webObserveTable(["TS", "TYPE", "COMMAND", "DURATION", "SAMPLE", "PATH", "MESSAGE"], commandFailures.length > 0 ? commandFailures.map((item) => [ webObserveShort(webObserveText(item.ts), 24), webObserveShort(webObserveText(item.type), 18), webObserveShort(webObserveText(item.commandId), 28), webObserveText(item.durationMs), webObserveText(item.failureSampleOk === true ? item.sampleSeq : "-"), webObserveShort([webObserveText(item.beforePath), webObserveText(item.afterPath)].filter((part) => part !== "-" && part !== "").join("->") || "-", 52), webObserveShort(webObserveText(item.message || item.failureKind || item.name), 96), ]) : [["-", "-", "-", "-", "-", "-", "-"]]), "", "Tool findings:", webObserveTable(["KIND", "SEVERITY", "COUNT", "SUMMARY"], toolFindings.length > 0 ? toolFindings.map((item) => [ webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 36), webObserveText(item.severity ?? item.level), webObserveText(item.count ?? item.sampleCount), webObserveShort(webObserveText(item.summary ?? item.message), 110), ]) : [["-", "-", "-", "-"]]), webObserveTable(["TOOL_COMMANDS", "COUNT"], [[ `pending=${webObserveText(commandState?.pendingCount)}`, `processing=${webObserveText(commandState?.processingCount)} abandoned=${webObserveText(commandState?.abandonedCount)} failed=${webObserveText(commandState?.failedCount)}`, ]]), "", "HTTP error groups:", webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], httpErrorRows.length > 0 ? httpErrorRows : [["-", "-", "-", "-", "-", "-"]]), "", "Request failed groups:", webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], requestFailedRows.length > 0 ? requestFailedRows : [["-", "-", "-", "-", "-", "-"]]), "", `Slow API (>${sameOriginApiBudgetLabel}):`, webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET"], slowApis.length > 0 ? slowApis.map((item) => [ webObserveShort(webObserveText(item.path ?? item.route), 52), webObserveText(item.sampleCount), webObserveText(item.p95Ms ?? item.p95), webObserveText(item.maxMs ?? item.max), webObserveText(item.overBudgetCount ?? item.overFiveSecondCount), ]) : [["-", "-", "-", "-", "-"]]), "", `Slow API samples (>${sameOriginApiBudgetLabel}):`, webObserveTable(["TS", "SEQ", "PATH", "DURATION", "REQ_WAIT", "RESP_XFER", "TIMING", "INIT", "PROTO", "SERVER", "OTEL"], (() => { const rows = slowApis.flatMap((item) => (Array.isArray(item.slowSamples) ? item.slowSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [ webObserveShort(webObserveText(sample.ts), 24), webObserveText(sample.seq), webObserveShort(webObserveText(sample.path), 44), webObserveText(sample.durationMs), webObserveText(sample.requestToResponseStartMs ?? sample.streamOpenMs), webObserveText(sample.responseTransferMs), webObserveShort(webObserveText(sample.timingStatus), 12), webObserveShort(webObserveText(sample.initiatorType), 12), webObserveShort(webObserveText(sample.nextHopProtocol), 12), webObserveShort(Array.isArray(sample.serverTimingNames) ? sample.serverTimingNames.join(",") : "-", 24), webObserveShort(webObserveText(sample.otelTraceId), 16), ]); return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]; })()), "", `Partial API timing (>${partialApiBudgetLabel}, not counted as completed slow API):`, webObserveTable(["PATH", "SAMPLES", "COMPLETE", "PARTIAL", "PARTIAL_OVER_BUDGET"], partialApis.length > 0 ? partialApis.map((item) => [ webObserveShort(webObserveText(item.path ?? item.route), 52), webObserveText(item.sampleCount), webObserveText(item.completeTimingSampleCount), webObserveText(item.partialTimingSampleCount), webObserveText(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount), ]) : [["-", "-", "-", "-", "-"]]), "", "Partial API samples:", webObserveTable(["TS", "SEQ", "PATH", "DURATION", "TIMING", "INIT", "PROTO"], (() => { const rows = partialApis.flatMap((item) => (Array.isArray(item.partialSamples) ? item.partialSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [ webObserveShort(webObserveText(sample.ts), 24), webObserveText(sample.seq), webObserveShort(webObserveText(sample.path), 44), webObserveText(sample.durationMs), webObserveShort(webObserveText(sample.timingStatus), 12), webObserveShort(webObserveText(sample.initiatorType), 12), webObserveShort(webObserveText(sample.nextHopProtocol), 12), ]); return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-"]]; })()), "", "Long-lived streams (SSE):", webObserveTable(["PATH", "SAMPLES", "OPEN_P95", "OPEN_MAX", `OPEN>${streamOpenBudgetLabel}`, "LIFE>5S"], sseStreams.length > 0 ? sseStreams.map((item) => [ webObserveShort(webObserveText(item.path ?? item.route), 52), webObserveText(item.sampleCount), webObserveText(item.streamOpenP95Ms), webObserveText(item.streamOpenMaxMs), webObserveText(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount), webObserveText(item.streamLifetimeOverFiveSecondCount), ]) : [["-", "-", "-", "-", "-", "-"]]), "", "Archive page performance:", webObserveTable(["SAME_ORIGIN_API", "SLOW_PATHS", "SLOW_SAMPLES", "WORST_P95"], [[ webObserveText(archivePagePerformance?.sameOriginApiPathCount ?? archivePagePerformance?.sameOriginApiPaths), webObserveText(archivePagePerformance?.slowPathCount), webObserveText(archivePagePerformance?.slowSampleCount), webObserveText(archivePagePerformance?.worstP95Ms), ]]), "", `Archive slow API (>${sameOriginApiBudgetLabel}):`, webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET"], archiveSlowApis.length > 0 ? archiveSlowApis.map((item) => [ webObserveShort(webObserveText(item.path ?? item.route), 52), webObserveText(item.sampleCount), webObserveText(item.p95Ms ?? item.p95), webObserveText(item.maxMs ?? item.max), webObserveText(item.overBudgetCount ?? item.overFiveSecondCount), ]) : [["-", "-", "-", "-", "-"]]), "", `Archive slow API samples (>${sameOriginApiBudgetLabel}):`, webObserveTable(["TS", "SEQ", "PATH", "DURATION", "REQ_WAIT", "TIMING", "OTEL"], (() => { const rows = archiveSlowApis.flatMap((item) => (Array.isArray(item.slowSamples) ? item.slowSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [ webObserveShort(webObserveText(sample.ts), 24), webObserveText(sample.seq), webObserveShort(webObserveText(sample.path), 44), webObserveText(sample.durationMs), webObserveText(sample.requestToResponseStartMs ?? sample.streamOpenMs), webObserveShort(webObserveText(sample.timingStatus), 12), webObserveShort(webObserveText(sample.otelTraceId), 16), ]); return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-"]]; })()), "", "DOM diagnostics:", webObserveTable(domDiagnostics.length > 0 ? ["COUNT", "FIRST", "LAST", "TEXT"] : ["SEQ", "TS", "META", "TEXT"], domDiagnosticRows.length > 0 ? domDiagnosticRows : [["-", "-", "-", "-"]]), "", "Console alerts:", webObserveTable(consoleAlertGroups.length > 0 ? ["COUNT", "TYPE", "STATUS", "PATH", "LAST", "TRACES"] : ["TS", "TYPE", "STATUS", "PATH", "TRACE", "TEXT"], consoleAlertRows.length > 0 ? consoleAlertRows : [["-", "-", "-", "-", "-", "-"]]), "", "Findings:", webObserveTable(["KIND", "SEVERITY", "COUNT", "TIMING", "ROOT_CAUSE", "SUMMARY"], findings.length > 0 ? findings.map((item) => [ webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 32), webObserveText(item.severity ?? item.level), webObserveText(item.count ?? item.sampleCount), webObserveShort(webObserveFindingTiming(item), 42), webObserveShort(webObserveText(item.rootCause ?? item.rootCauseStatus), 48), webObserveShort(webObserveText(item.summary ?? item.message), 96), ]) : [["-", "-", "-", "-", "-", "-"]]), "", "Archive red findings:", webObserveTable(["KIND", "COUNT", "SUMMARY"], archiveRedFindings.length > 0 ? archiveRedFindings.map((item) => [ webObserveShort(webObserveText(item.kind ?? item.code), 36), webObserveText(item.count ?? item.sampleCount), webObserveShort(webObserveText(item.summary ?? item.message), 110), ]) : [["-", "-", "-"]]), ]; if (pagePerformance !== null && Object.keys(pagePerformance).length > 0) { lines.push("", "Page performance:", ` ${webObserveShort(JSON.stringify(pagePerformance), 180)}`); } lines.push( "", "Next:", ` cat ${webObserveText(analysis?.reportMdPath)}`, ` cat ${webObserveText(analysis?.reportJsonPath)}`, "", "Disclosure:", " default view is a bounded report summary; use report.md/report.json paths for full offline analysis.", ); return lines.join("\n"); } export function dedupeWebObserveRecords(items: Record[], keyOf: (item: Record) => string): Record[] { const out: Record[] = []; const seen = new Set(); for (const item of items) { const key = keyOf(item); if (!key || key === "-" || seen.has(key)) continue; seen.add(key); out.push(item); } return out; } export function nodeWebObserveResolveStateDirShell(options: Pick): string { if (options.stateDir !== null) { return [ `state_dir=${shellQuote(options.stateDir)}`, "test -d \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"state-dir-missing\",\"stateDir\":\"%s\"}\\n' \"$state_dir\"; exit 2; }", ].join("\n"); } if (options.jobId === null) throw new Error("web-probe observe requires --state-dir or --job-id"); const root = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`; return [ `observe_root=${shellQuote(root)}`, `job_id=${shellQuote(options.jobId)}`, "state_dir=$(find \"$observe_root\" -type d -name \"*_${job_id}\" 2>/dev/null | sort | tail -n 1 || true)", "test -n \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"job-id-not-found\",\"jobId\":\"%s\",\"root\":\"%s\"}\\n' \"$job_id\" \"$observe_root\"; exit 2; }", ].join("\n"); } export function webObserveIndexPath(): string { return rootPath(".state/web-observe/index.json"); } export function readWebObserveIndex(): Record { const path = webObserveIndexPath(); if (!existsSync(path)) return {}; const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown; if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`web-probe observe index is invalid: ${path}`); const entries: Record = {}; for (const [id, value] of Object.entries(parsed)) { if (!isSafeWebObserveJobId(id)) continue; if (typeof value !== "object" || value === null || Array.isArray(value)) continue; const recordValue = value as Record; const stateDir = typeof recordValue.stateDir === "string" ? recordValue.stateDir : ""; const node = typeof recordValue.node === "string" ? recordValue.node : ""; const lane = typeof recordValue.lane === "string" ? recordValue.lane : ""; const workspace = typeof recordValue.workspace === "string" ? recordValue.workspace : ""; if (!isSafeWebObserveStateDir(stateDir) || node.length === 0 || lane.length === 0 || workspace.length === 0) continue; entries[id] = { id, node, lane, workspace, stateDir, url: typeof recordValue.url === "string" ? recordValue.url : "", targetPath: typeof recordValue.targetPath === "string" ? recordValue.targetPath : "", status: typeof recordValue.status === "string" ? recordValue.status : null, pid: typeof recordValue.pid === "number" ? recordValue.pid : null, startedAt: typeof recordValue.startedAt === "string" ? recordValue.startedAt : null, updatedAt: typeof recordValue.updatedAt === "string" ? recordValue.updatedAt : "", }; } return entries; } export function readWebObserveIndexEntry(id: string): WebObserveIndexEntry | null { return readWebObserveIndex()[id] ?? null; } export function discoverWebObserveIndexEntry(id: string): WebObserveIndexEntry | null { const matches: WebObserveIndexEntry[] = []; const errors: string[] = []; for (const lane of hwlabRuntimeLaneIds()) { for (const node of hwlabRuntimeNodeIds()) { let spec: HwlabRuntimeLaneSpec; try { spec = hwlabRuntimeLaneSpecForNode(lane, node); } catch { continue; } try { const entry = discoverWebObserveIndexEntryOnTarget(id, node, lane, spec); if (entry !== null) matches.push(entry); } catch (error) { const message = error instanceof Error ? error.message : String(error); errors.push(`${node}/${lane}: ${message.slice(0, 240)}`); } } } if (matches.length > 1) { const candidates = matches.map((entry) => `${entry.node}/${entry.lane}:${entry.stateDir}`).join(", "); throw new Error(`web-probe observe observer-id-ambiguous: ${id}; candidates=${candidates}`); } if (matches.length === 0) { if (errors.length > 0) { throw new Error(`web-probe observe observer-id-unknown: ${id}; discoveryErrors=${errors.join(" | ")}`); } return null; } upsertWebObserveIndexEntry(matches[0]!); return matches[0]!; } export function discoverWebObserveIndexEntryOnTarget(id: string, node: string, lane: HwlabRuntimeLane, spec: HwlabRuntimeLaneSpec): WebObserveIndexEntry | null { const observeRoot = `.state/web-observe/${safeWebObserveSegment(node)}/${safeWebObserveSegment(lane)}`; const script = [ "set -eu", `observe_root=${shellQuote(observeRoot)}`, `job_id=${shellQuote(id)}`, "node - \"$observe_root\" \"$job_id\" <<'NODE'", "const fs = require('fs');", "const path = require('path');", "const [root, jobId] = process.argv.slice(2);", "const matches = [];", "function readJson(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }", "function walk(dir, depth) {", " if (depth > 10 || matches.length > 1) return;", " let entries = [];", " try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }", " for (const entry of entries) {", " if (!entry.isDirectory()) continue;", " const full = path.join(dir, entry.name);", " if (entry.name.endsWith('_' + jobId)) matches.push(full);", " else walk(full, depth + 1);", " if (matches.length > 1) return;", " }", "}", "walk(root, 0);", "if (matches.length === 0) { console.log(JSON.stringify({ ok: true, found: false, root, jobId })); process.exit(0); }", "if (matches.length > 1) { console.log(JSON.stringify({ ok: false, found: true, ambiguous: true, root, jobId, matches })); process.exit(3); }", "const stateDir = matches[0];", "const manifest = readJson(path.join(stateDir, 'manifest.json')) || {};", "const heartbeat = readJson(path.join(stateDir, 'heartbeat.json')) || {};", "let pid = null;", "try { const raw = fs.readFileSync(path.join(stateDir, 'pid'), 'utf8').trim(); const value = Number(raw); if (Number.isFinite(value)) pid = value; } catch {}", "console.log(JSON.stringify({", " ok: true,", " found: true,", " ambiguous: false,", " entry: {", " id: jobId,", " stateDir,", " url: typeof manifest.baseUrl === 'string' ? manifest.baseUrl : '',", " targetPath: typeof manifest.targetPath === 'string' ? manifest.targetPath : '',", " status: typeof manifest.status === 'string' ? manifest.status : null,", " pid,", " startedAt: typeof manifest.startedAt === 'string' ? manifest.startedAt : null,", " updatedAt: typeof heartbeat.updatedAt === 'string' ? heartbeat.updatedAt : new Date().toISOString()", " }", "}));", "NODE", ].join("\n"); const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55); const payload = parseJsonObject(result.stdout); if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) { const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`; throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`); } if (payload.found !== true) return null; const entry = record(payload.entry); const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : ""; if (!isSafeWebObserveStateDir(stateDir)) throw new Error(`observer discovery returned unsafe stateDir: ${stateDir}`); return { id, node, lane, workspace: spec.workspace, stateDir, url: typeof entry.url === "string" ? entry.url : "", targetPath: typeof entry.targetPath === "string" ? entry.targetPath : "", status: typeof entry.status === "string" ? entry.status : null, pid: typeof entry.pid === "number" ? entry.pid : null, startedAt: typeof entry.startedAt === "string" ? entry.startedAt : null, updatedAt: typeof entry.updatedAt === "string" ? entry.updatedAt : new Date().toISOString(), }; } export function upsertWebObserveIndexEntry(entry: WebObserveIndexEntry): Record { const path = webObserveIndexPath(); try { const current = readWebObserveIndex(); mkdirSync(dirname(path), { recursive: true }); const next = { ...current, [entry.id]: entry }; const tmp = `${path}.tmp-${process.pid}`; writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 }); renameSync(tmp, path); return { ok: true, id: entry.id, path, statusCommand: `bun scripts/cli.ts web-probe observe status ${entry.id}`, valuesRedacted: true, }; } catch (error) { return { ok: false, id: entry.id, path, error: error instanceof Error ? error.message : String(error), fallback: `bun scripts/cli.ts web-probe observe status --node ${entry.node} --lane ${entry.lane} --state-dir ${entry.stateDir}`, valuesRedacted: true, }; } } export function webObserveIdFromOptions(options: Pick): string | null { return options.id ?? options.jobId; } export function webObserveIdFromStatus(status: Record | null, options: Pick): string | null { const manifest = record(status?.manifest); const heartbeat = record(status?.heartbeat); const manifestJobId = typeof manifest.jobId === "string" ? manifest.jobId : null; const heartbeatJobId = typeof heartbeat.jobId === "string" ? heartbeat.jobId : null; return manifestJobId ?? heartbeatJobId ?? webObserveIdFromOptions(options); } export function webObserveIndexEntryFromOptions(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, id: string, status: Record): WebObserveIndexEntry { const manifest = record(status.manifest); const heartbeat = record(status.heartbeat); return { id, node: options.node, lane: options.lane, workspace: spec.workspace, stateDir: options.stateDir ?? "", url: typeof manifest.baseUrl === "string" ? manifest.baseUrl : options.url, targetPath: typeof manifest.targetPath === "string" ? manifest.targetPath : options.targetPath, status: typeof manifest.status === "string" ? manifest.status : null, pid: typeof status.pid === "number" ? status.pid : null, startedAt: typeof manifest.startedAt === "string" ? manifest.startedAt : null, updatedAt: typeof heartbeat.updatedAt === "string" ? heartbeat.updatedAt : new Date().toISOString(), }; } export function webObserveCommandLabel(action: NodeWebProbeObserveAction, options: Pick): string { const id = webObserveIdFromOptions(options); return id === null ? `web-probe observe ${action} --node ${options.node} --lane ${options.lane}` : `web-probe observe ${action} ${id}`; } export function webObserveNextCommands(id: string): Record { return { status: `bun scripts/cli.ts web-probe observe status ${id}`, command: `bun scripts/cli.ts web-probe observe command ${id} --type mark --label checkpoint`, stop: `bun scripts/cli.ts web-probe observe stop ${id}`, analyze: `bun scripts/cli.ts web-probe observe analyze ${id}`, }; } export function webObserveCell(value: unknown, maxLength = 96): string { if (value === null || value === undefined) return "-"; const text = typeof value === "string" ? value : String(value); const compact = text.replace(/\s+/gu, " ").trim(); if (compact.length === 0) return "-"; if (compact.length <= maxLength) return compact; return `${compact.slice(0, Math.max(1, maxLength - 1))}...`; } function webObserveFindingTiming(item: Record): string { const status = item.timingStatus ?? null; const source = item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind ?? null; const parts = [status, source].filter((value) => value !== null && value !== undefined && value !== ""); if (parts.length === 0) return "-"; return parts.map((value) => webObserveText(value)).join("/"); } export function webObserveTable(headers: string[], rows: unknown[][]): string { const stringRows = rows.map((row) => row.map((value) => webObserveCell(value))); const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0))); const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); return [renderRow(headers), ...stringRows.map(renderRow)].join("\n"); } export function webObserveArray(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } export function withWebObserveRendered(result: Record, renderedText: string): Record { return { ...result, renderedText, contentType: "text/plain", }; } export function renderWebProbeRunResult(result: Record): Record { const summary = record(result.summary); const probe = record(result.probe); const trace = record(probe.trace); const session = record(probe.session); const artifacts = record(probe.artifacts); const reportLoad = record(result.reportLoad); const commandTimeout = record(result.commandTimeout); const job = record(result.job); const start = record(result.start); const commandResult = record(result.result ?? result.statusResult); const reportPath = reportLoad.path ?? summary.reportPath ?? artifacts.reportPath ?? start.reportPath ?? "-"; const reportSource = reportLoad.source ?? (result.mode === "async" ? "-" : "stdout"); const blockedRows = result.ok === true ? [] : [ "", webObserveTable( ["BLOCKED_FIELD", "VALUE"], [ ["failureKind", result.failureKind ?? summary.failureKind ?? "-"], ["failedCondition", summary.failedCondition ?? result.degradedReason ?? "-"], ["nextAction", summary.nextAction ?? "-"], ["exitCode", commandResult.exitCode ?? "-"], ["timedOut", commandResult.timedOut ?? commandTimeout.timedOut ?? "-"], ], ), ]; const renderedText = [ webObserveTable( ["WEB_PROBE_RUN", "NODE", "LANE", "STATUS", "OK", "DEGRADED", "URL"], [[result.command ?? "web-probe run", result.node, result.lane, result.status, result.ok, result.degradedReason ?? "-", result.url]], ), "", webObserveTable( ["FIELD", "VALUE"], [ ["mode", result.mode ?? (result.reportLoad ? "async" : "direct")], ["traceId", summary.traceId ?? trace.traceId ?? "-"], ["sessionId", summary.sessionId ?? session.sessionId ?? "-"], ["conversationId", summary.conversationId ?? session.conversationId ?? "-"], ["agentStatus", summary.agentStatus ?? trace.finalAgentStatus ?? "-"], ["traceStatus", summary.traceStatus ?? trace.finalTraceStatus ?? "-"], ["messageCount", summary.messageCount ?? session.messageCount ?? "-"], ["reportSource", reportSource], ["reportPath", reportPath], ["reportSha256", summary.reportSha256 ?? artifacts.reportSha256 ?? "-"], ["timeout", `${commandTimeout.seconds ?? "-"}s timedOut=${commandTimeout.timedOut ?? "-"}`], ["job", job.jobId ?? "-"], ], ), ...blockedRows, "", "NEXT", ` report: ${reportPath}`, ` rerun: ${result.command ?? `web-probe run --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`, ].join("\n"); return withWebObserveRendered(result, renderedText); } export function renderWebObserveStartResult(result: Record): Record { const observer = record(result.observer); const heartbeat = record(observer.heartbeat); const manifest = record(observer.manifest); const commandResult = record(result.result); const credential = record(result.credential); const id = result.id ?? observer.id ?? observer.jobId ?? manifest.jobId ?? "-"; const status = result.status ?? manifest.status ?? heartbeat.status ?? "-"; const stateDir = observer.stateDir ?? manifest.stateDir ?? "-"; const blockedRows = result.ok === true ? [] : [ "", "Blocked detail:", webObserveTable( ["EXIT", "TIMEOUT", "STDOUT", "STDERR"], [[ webObserveText(commandResult.exitCode), webObserveText(commandResult.timedOut), webObserveShort(webObserveText(commandResult.stdoutTail ?? commandResult.stdout), 160), webObserveShort(webObserveText(commandResult.stderrTail ?? commandResult.stderr), 300), ]], ), ]; const renderedText = [ webObserveTable( ["OBSERVER", "NODE", "LANE", "STATUS", "PID", "SAMPLE", "UPDATED"], [[ id, result.node, result.lane, status, observer.pid ?? heartbeat.pid, heartbeat.sampleSeq, heartbeat.updatedAt, ]], ), "", ...renderWebObserveWrapperContract(result), webObserveTable( ["URL", "TARGET_PATH", "STATE_DIR"], [[result.url, result.targetPath, webObserveShort(webObserveText(stateDir), 96)]], ), "", webObserveTable( ["CREDENTIAL", "SOURCE", "VALUES"], [[ credential.username ?? "-", credential.sourceRef ?? "-", credential.valuesRedacted === true ? "redacted" : "unknown", ]], ), ...blockedRows, "", "NEXT", ` status: bun scripts/cli.ts web-probe observe status ${id}`, ` analyze: bun scripts/cli.ts web-probe observe analyze ${id}`, ` command: bun scripts/cli.ts web-probe observe command ${id} --type mark --label checkpoint`, ` stop: bun scripts/cli.ts web-probe observe stop ${id}`, ].join("\n"); return withWebObserveRendered(result, renderedText); } export function renderWebObserveStatusResult(result: Record): Record { const observer = record(result.observer); const commandResult = record(result.result); const manifest = record(observer.manifest); const heartbeat = record(observer.heartbeat); const tails = record(observer.tails); const samples = webObserveArray(tails.samples); const network = webObserveArray(tails.network); const control = webObserveArray(tails.control); const artifacts = webObserveArray(tails.artifacts); const lastSample = record(samples[samples.length - 1]); const failedNetwork = network .map((item) => record(item)) .filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500); const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-"; const resultSection = result.ok === true ? [] : [ "", webObserveTable( ["RESULT", "VALUE"], [ ["exitCode", commandResult.exitCode ?? "-"], ["timedOut", commandResult.timedOut ?? "-"], ["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"], ["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"], ], ), ]; const renderedText = [ webObserveTable( ["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"], [[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]], ), "", webObserveTable( ["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"], [[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]], ), "", webObserveTable( ["TAIL", "COUNT", "DETAIL"], [ ["control", control.length, record(control[control.length - 1]).type ?? "-"], ["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"], ["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`], ["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"], ], ), ...resultSection, "", "NEXT", ` status: bun scripts/cli.ts web-probe observe status ${id}`, ` analyze: bun scripts/cli.ts web-probe observe analyze ${id}`, ` command: bun scripts/cli.ts web-probe observe command ${id} --type mark --label checkpoint`, ].join("\n"); return withWebObserveRendered(result, renderedText); } export function renderWebObserveCommandResult(result: Record): Record { const observerCommand = record(result.observerCommand); const observer = record(result.observer); const id = result.id ?? "-"; const renderedText = [ webObserveTable( ["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"], [[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]], ), "", webObserveTable( ["INPUT", "VALUE"], [ ["label", observerCommand.label ?? "-"], ["path", observerCommand.path ?? "-"], ["provider", observerCommand.provider ?? "-"], ["sessionId", observerCommand.sessionId ?? "-"], ["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`], ], ), "", "NEXT", ` status: bun scripts/cli.ts web-probe observe status ${id}`, ` analyze: bun scripts/cli.ts web-probe observe analyze ${id}`, ].join("\n"); return withWebObserveRendered(result, renderedText); } export function renderWebObserveAnalyzeResult(result: Record): Record { const analysis = record(result.analysis); const failure = record(result.failure); const counts = record(analysis.counts); const sampleMetrics = record(analysis.sampleMetrics); const runtimeAlerts = record(analysis.runtimeAlerts); const pagePerformance = record(analysis.pagePerformance); const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item)); const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item)); const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item)); const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item)); const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item)); const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); const sseStreams = webObserveArray(analysis.pagePerformanceSseStreams).slice(0, 8).map((item) => record(item)); const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); const commandState = record(analysis.commandState); const id = result.id ?? "-"; const blockerRows = String(result.status ?? "") === "blocked" ? [ ["reason", failure.reason], ["exitCode", failure.exitCode], ["timedOut", failure.timedOut], ["parsedJson", failure.parsedJson], ["stdoutBytes", failure.stdoutBytes], ["stderrBytes", failure.stderrBytes], ["stdoutTail", String(failure.stdoutTail ?? "").replace(/\s+/g, " ").slice(0, 220)], ["stderrTail", String(failure.stderrTail ?? "").replace(/\s+/g, " ").slice(0, 220)], ["workspace", failure.workspace], ] : []; const renderedText = [ webObserveTable( ["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"], [[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]], ), "", ...renderWebObserveWrapperContract(result), ...(blockerRows.length > 0 ? ["", webObserveTable(["ANALYZE_BLOCKER", "VALUE"], blockerRows)] : []), "", webObserveTable( ["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"], [[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]], ), "", jsonlReadIssues.length === 0 ? "JSONL_READ_ISSUES\n-" : webObserveTable( ["FILE", "CODE", "MESSAGE"], jsonlReadIssues.map((item) => [item.file, item.code, item.message]), ), "", webObserveTable( ["TURN_TIMING", "VALUE"], [ ["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount], ["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount], ["elapsedForwardJump", sampleMetrics.turnTimingTotalElapsedForwardJumpCount], ["elapsedForwardJumpMaxSec", sampleMetrics.turnTimingTotalElapsedForwardJumpMaxSeconds], ["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount], ["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds], ], ), "", rounds.length === 0 ? "ROUNDS\n-" : webObserveTable( ["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "CARD_DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"], rounds.map((item) => [ item.promptIndex, item.promptCommandId, item.sampleCount, item.maxTotalElapsedSeconds, item.lastTotalElapsedSeconds, item.maxRecentUpdateSeconds, item.lastRecentUpdateSeconds, item.diagnosticSamples, item.terminalSamples, item.turnTimingNonMonotonicCount, item.turnTimingTotalElapsedForwardJumpCount, item.turnTimingRecentUpdateJumpCount, ]), ), "", turnColumns.length === 0 ? "TURN_COLUMNS\n-" : webObserveTable( ["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"], turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]), ), "", webObserveTable( ["ALERT", "COUNT"], [ ["httpError", runtimeAlerts.httpErrorCount], ["requestFailed", runtimeAlerts.requestFailedCount], ["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount], ["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount], ["consoleAlerts", runtimeAlerts.consoleAlertCount], ["executionErrors", runtimeAlerts.executionErrorCount], ], ), "", webObserveTable( ["TOOL_COMMANDS", "COUNT"], [ ["pending", commandState.pendingCount], ["processing", commandState.processingCount], ["abandoned", commandState.abandonedCount], ["failed", commandState.failedCount], ], ), "", domDiagnosticGroups.length === 0 ? "DOM_DIAGNOSTIC_GROUPS\n-" : webObserveTable( ["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"], domDiagnosticGroups.map((item) => [ item.diagnosticCode, item.count, item.firstSeq, item.lastSeq, item.firstAt, item.lastAt, webObserveArray(item.promptIndexes).join(",") || "-", item.traceId, String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140), ]), ), "", domDiagnostics.length === 0 ? "DOM_DIAGNOSTICS\n-" : webObserveTable( ["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"], domDiagnostics.map((item) => [ item.seq, item.ts, item.promptIndex, item.diagnosticCode, item.traceId, item.httpStatus, item.idleSeconds, item.waitingFor, item.lastEventLabel, String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180), ]), ), "", webObserveTable( ["PERF", "VALUE"], [ ["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount], ["longLivedStreamPaths", pagePerformance.longLivedStreamPathCount], ["streamOpenOverBudgetSamples", pagePerformance.longLivedStreamOpenOverBudgetSampleCount ?? pagePerformance.longLivedStreamOpenOverFiveSecondSampleCount], ["slowPathCount", pagePerformance.slowPathCount], ["slowSampleCount", pagePerformance.slowSampleCount], ["worstP95Ms", pagePerformance.worstP95Ms], ], ), "", slowApis.length === 0 ? "SLOW_API\n-" : webObserveTable( ["SLOW_API", "P50", "P75", "P95", "OVER_BUDGET", "COUNT"], slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overBudgetCount ?? item.overFiveSecondCount, item.sampleCount]), ), "", sseStreams.length === 0 ? "SSE_STREAMS\n-" : webObserveTable( ["SSE_STREAM", "OPEN_P95", "OPEN_MAX", "OPEN_OVER_BUDGET", "LIFE>5S", "COUNT"], sseStreams.map((item) => [item.path, item.streamOpenP95Ms, item.streamOpenMaxMs, item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount, item.streamLifetimeOverFiveSecondCount, item.sampleCount]), ), "", findings.length === 0 ? "FINDINGS\n-" : webObserveTable( ["FINDING", "SEVERITY", "COUNT", "TIMING", "SUMMARY"], findings.map((item) => [item.id ?? item.kind ?? item.code, item.severity ?? item.level, item.count ?? item.sampleCount, webObserveFindingTiming(item), item.summary ?? item.message]), ), "", "REPORTS", ` json: ${analysis.reportJsonPath ?? "-"}`, ` md: ${analysis.reportMdPath ?? "-"}`, "", "NEXT", ` status: bun scripts/cli.ts web-probe observe status ${id}`, ].join("\n"); return withWebObserveRendered(result, renderedText); } export function renderWebProbeScriptResult(result: Record): Record { const summary = record(result.summary); const issueEvidence = record(result.issueEvidence); const probe = record(result.probe); const script = record(probe.script); const scriptResult = record(script.result); const reportLoad = record(result.reportLoad); const commandResult = record(result.result); const recoveredArtifacts = record(result.recoveredArtifacts); const recoveredArtifactSummary = record(recoveredArtifacts.artifacts); const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item)); const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item)); const warnings = webObserveArray(result.warnings).slice(0, 6); const hints = webObserveArray(result.hints).slice(0, 6).map((item) => String(item ?? "").trim()).filter(Boolean); const preferredCommands = record(result.preferredCommands); const resultRows = webProbeScriptRecordRows(scriptResult, 12); const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12); const summaryRows = [ ["ok", result.ok], ["status", result.status], ["degradedReason", result.degradedReason ?? "-"], ["failureKind", result.failureKind ?? "-"], ["scriptSource", result.scriptSource ?? "-"], ["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"], ["reportPath", reportLoad.path ?? probe.reportPath ?? "-"], ["reportSha256", probe.reportSha256 ?? "-"], ["scriptSha256", probe.scriptSha256 ?? "-"], ["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"], ["exitCode", commandResult.exitCode ?? "-"], ]; const renderedText = [ webObserveTable( ["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"], [[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]], ), "", webObserveTable(["FIELD", "VALUE"], summaryRows), "", resultRows.length === 0 ? "SCRIPT_RESULT\n-" : webObserveTable(["KEY", "VALUE"], resultRows), "", evidenceRows.length === 0 ? "ISSUE_EVIDENCE\n-" : webObserveTable(["KEY", "VALUE"], evidenceRows), "", steps.length === 0 ? "STEPS\n-" : webObserveTable( ["STEP", "OK", "DETAIL"], steps.map((item) => [ item.name ?? item.step ?? item.label ?? "-", item.ok ?? item.status ?? "-", webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item), ]), ), "", recoveredItems.length === 0 ? "ARTIFACTS\n-" : webObserveTable( ["KIND", "BYTES", "PATH"], recoveredItems.map((item) => [item.kind, item.byteCount, item.path]), ), "", warnings.length === 0 ? "WARNINGS\n-" : [ "WARNINGS", webObserveTable( ["CODE", "SEVERITY", "MESSAGE"], warnings.map((item) => { const warning = record(item); return [warning.code ?? "warning", warning.severity ?? warning.level ?? "warning", warning.message ?? warning.summary ?? webProbeScriptPreview(warning)]; }), ), ].join("\n"), "", hints.length === 0 ? "HINTS\n-" : ["HINTS", ...hints.map((hint) => `- ${hint}`)].join("\n"), "", "NEXT", ` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`, ` rerun: ${result.command ?? `web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`, ...Object.entries(preferredCommands).map(([name, command]) => ` ${name}: ${String(command)}`), ].join("\n"); return withWebObserveRendered(result, renderedText); } export function webProbeScriptRecordRows(value: Record, limit: number): unknown[][] { return Object.entries(value) .slice(0, limit) .map(([key, nested]) => [key, webProbeScriptPreview(nested)]); } export function webProbeScriptPreview(value: unknown): string { if (value === null || value === undefined) return "-"; if (typeof value === "string") return value.replace(/\s+/gu, " ").trim(); if (typeof value === "number" || typeof value === "boolean") return String(value); if (Array.isArray(value)) { const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", "); return `[${value.length}] ${preview}`; } if (typeof value === "object") { return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240); } return String(value); } export function withWebObserveShortcuts(value: Record | null, id: string | null): Record | null { if (value === null || id === null) return value; return { ...value, id, next: webObserveNextCommands(id), }; }