Codex
1716 lines
97 KiB
TypeScript
1716 lines
97 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. web-observe-render module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:9001-10514 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { NodeWebProbeObserveAction, NodeWebProbeObserveOptions, WebObserveIndexEntry } from "./entry";
|
|
import { runTransWorkspaceStdinScript } from "./public-exposure";
|
|
import { parseJsonObject, record, shellQuote } from "./utils";
|
|
import { isSafeWebObserveJobId, isSafeWebObserveStateDir, safeWebObserveSegment } from "./web-observe-scripts";
|
|
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
|
|
|
export function recoverWebObserveAnalyzeTurnDetails(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, analysis: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (analysis === null || typeof analysis !== "object") return analysis;
|
|
const sampleMetrics = record(analysis.sampleMetrics);
|
|
const hasRounds = Array.isArray(sampleMetrics?.rounds) && sampleMetrics.rounds.length > 0;
|
|
const hasColumns = Array.isArray(sampleMetrics?.turnColumns) && sampleMetrics.turnColumns.length > 0;
|
|
const columnsHavePageIdentity = hasColumns && (sampleMetrics?.turnColumns as unknown[]).some((item) => {
|
|
const column = record(item);
|
|
return typeof column?.pageRole === "string" || typeof column?.pageId === "string";
|
|
});
|
|
if (hasRounds && hasColumns && columnsHavePageIdentity) return analysis;
|
|
const reportJsonPath = typeof analysis.reportJsonPath === "string" ? analysis.reportJsonPath : "";
|
|
if (!reportJsonPath) return analysis;
|
|
const archiveMetrics = record(record(analysis.archiveSummary)?.sampleMetrics);
|
|
const archiveHasTurnEvidence = Number(archiveMetrics?.rounds ?? 0) > 0 || Number(archiveMetrics?.turnColumns ?? 0) > 0 || Number(archiveMetrics?.turnTimingRows ?? 0) > 0;
|
|
if (!archiveHasTurnEvidence) return analysis;
|
|
const recoverScript = [
|
|
"set -eu",
|
|
`report_json=${shellQuote(reportJsonPath)}`,
|
|
"node - \"$report_json\" <<'UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS'",
|
|
"const fs = require('fs');",
|
|
"const reportJsonPath = process.argv[2];",
|
|
"const readJson = (path) => { try { return JSON.parse(fs.readFileSync(path, 'utf8')); } catch { return null; } };",
|
|
"const report = readJson(reportJsonPath) || {};",
|
|
"const stdoutJson = readJson(String(reportJsonPath).replace(/\\/analysis\\/report\\.json$/u, '/analysis/analyzer-stdout.json')) || {};",
|
|
"const objectOrNull = (value) => value && typeof value === 'object' && !Array.isArray(value) ? value : null;",
|
|
"const firstNonEmptyArray = (...values) => { for (const value of values) if (Array.isArray(value) && value.length > 0) return value; return []; };",
|
|
"const clip = (value, limit = 160) => value === null || value === undefined ? null : String(value).slice(0, limit);",
|
|
"const slimRound = (item) => { const v = objectOrNull(item) || {}; return { promptIndex: v.promptIndex ?? null, sampleCount: v.sampleCount ?? null, loadingSamples: v.loadingSamples ?? null, maxLoadingCount: v.maxLoadingCount ?? null, loadingOwnerCount: v.loadingOwnerCount ?? null, lastTotalElapsedSeconds: v.lastTotalElapsedSeconds ?? null, lastRecentUpdateSeconds: v.lastRecentUpdateSeconds ?? null, diagnosticSamples: v.diagnosticSamples ?? null, terminalSamples: v.terminalSamples ?? null, turnTimingTotalElapsedForwardJumpCount: v.turnTimingTotalElapsedForwardJumpCount ?? null, turnTimingTotalElapsedForwardJumpMaxSeconds: v.turnTimingTotalElapsedForwardJumpMaxSeconds ?? null, promptTextHash: clip(v.promptTextHash, 80) }; };",
|
|
"const slimTurnColumn = (item) => { const v = objectOrNull(item) || {}; return { label: clip(v.label, 24), pageRole: clip(v.pageRole, 24), pageId: clip(v.pageId, 32), pageEpoch: v.pageEpoch ?? null, promptIndex: v.promptIndex ?? null, lastPromptIndex: v.lastPromptIndex ?? null, traceId: clip(v.traceId, 48), firstSeq: v.firstSeq ?? null, lastSeq: v.lastSeq ?? null, source: clip(v.source, 48) }; };",
|
|
"const rounds = firstNonEmptyArray(stdoutJson.sampleMetrics?.rounds, stdoutJson.sampleMetrics?.roundItems, report.windows?.recent?.sampleMetrics?.rounds, report.sampleMetrics?.rounds).slice(-8).map(slimRound);",
|
|
"const turnColumns = firstNonEmptyArray(stdoutJson.sampleMetrics?.turnColumns, report.windows?.recent?.sampleMetrics?.turnColumns, report.sampleMetrics?.turnColumns).slice(-12).map(slimTurnColumn);",
|
|
"console.log(JSON.stringify({ ok: true, rounds, turnColumns }));",
|
|
"UNIDESK_WEB_OBSERVE_RECOVER_TURN_DETAILS",
|
|
].join("\n");
|
|
const recovered = parseJsonObject(runTransWorkspaceStdinScript(options.node, spec.workspace, recoverScript, Math.min(options.commandTimeoutSeconds, 30)).stdout);
|
|
const recoveredRounds = Array.isArray(recovered.rounds) ? recovered.rounds : [];
|
|
const recoveredColumns = Array.isArray(recovered.turnColumns) ? recovered.turnColumns : [];
|
|
if (recoveredRounds.length === 0 && recoveredColumns.length === 0) return analysis;
|
|
const nextMetrics = { ...(sampleMetrics ?? {}) };
|
|
if (!hasRounds && recoveredRounds.length > 0) {
|
|
nextMetrics.roundItems = recoveredRounds;
|
|
nextMetrics.rounds = recoveredRounds;
|
|
}
|
|
if (!hasColumns && recoveredColumns.length > 0) nextMetrics.turnColumns = recoveredColumns;
|
|
return { ...analysis, sampleMetrics: nextMetrics, recoveredTurnDetails: { rounds: recoveredRounds.length, turnColumns: recoveredColumns.length, source: "analysis-report", valuesRedacted: true } };
|
|
}
|
|
|
|
export function withWebObserveAnalyzeRendered(value: Record<string, unknown>): RenderedCliResult {
|
|
return {
|
|
ok: value.ok !== false,
|
|
command: typeof value.command === "string" ? value.command : "web-probe observe analyze",
|
|
contentType: "text/plain",
|
|
renderedText: renderWebObserveAnalyzeTable(value),
|
|
};
|
|
}
|
|
|
|
export function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
|
|
const analysis = record(value.analysis) ?? value;
|
|
const analyzer = record(analysis?.analyzer);
|
|
const result = record(value.result);
|
|
const counts = record(analysis?.counts);
|
|
const jsonlScope = record(analysis?.jsonlScope);
|
|
const analysisWindow = record(analysis?.analysisWindow);
|
|
const archiveSummary = record(analysis?.archiveSummary);
|
|
const nestedSampleMetrics = record(analysis?.sampleMetrics);
|
|
const directSampleMetrics = record(value.sampleMetrics);
|
|
const hasRenderableTurnMetrics = (item: Record<string, unknown> | null): boolean => {
|
|
if (!item) return false;
|
|
return (Array.isArray(item.roundItems) && item.roundItems.length > 0)
|
|
|| (Array.isArray(item.rounds) && item.rounds.length > 0)
|
|
|| (Array.isArray(item.turnColumns) && item.turnColumns.length > 0);
|
|
};
|
|
const sampleMetrics = hasRenderableTurnMetrics(nestedSampleMetrics)
|
|
? nestedSampleMetrics
|
|
: hasRenderableTurnMetrics(directSampleMetrics)
|
|
? directSampleMetrics
|
|
: nestedSampleMetrics ?? directSampleMetrics;
|
|
const archiveMetrics = record(archiveSummary?.sampleMetrics);
|
|
const archivePagePerformance = record(archiveSummary?.pagePerformance);
|
|
const runtimeAlerts = record(analysis?.runtimeAlerts);
|
|
const pagePerformance = record(analysis?.pagePerformance);
|
|
const pagePerformanceSummary = record(pagePerformance?.summary);
|
|
const nonEmptyRecord = (item: unknown): Record<string, unknown> | null => {
|
|
const valueRecord = record(item);
|
|
return Object.keys(valueRecord).length > 0 ? valueRecord : null;
|
|
};
|
|
const requestRateSummarySource = nonEmptyRecord(analysis?.requestRate);
|
|
const requestRateCurve = nonEmptyRecord(analysis?.requestRateCurve) ?? (requestRateSummarySource?.summary ? requestRateSummarySource : null);
|
|
const requestRateSummary = nonEmptyRecord(requestRateCurve?.summary) ?? requestRateSummarySource;
|
|
const requestRatePeaks = webObserveArray(analysis?.requestRatePeaks ?? requestRateCurve?.peaks ?? requestRateSummarySource?.peaks).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8);
|
|
const requestRatePageCurves = webObserveArray(requestRateCurve?.pageCurves ?? requestRateSummarySource?.pageCurves).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8);
|
|
const requestRateApiPathCurves = webObserveArray(requestRateCurve?.apiPathCurves ?? requestRateSummarySource?.apiPathCurves).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8);
|
|
const projectManagement = record(analysis?.projectManagement) ?? record(value.projectManagement);
|
|
const projectManagementSummary = record(projectManagement?.summary) ?? projectManagement;
|
|
const projectManagementCommandsSource = Array.isArray(projectManagement?.launchCommands) && projectManagement.launchCommands.length > 0
|
|
? projectManagement.launchCommands
|
|
: projectManagement?.commands;
|
|
const projectManagementCommands = webObserveArray(projectManagementCommandsSource).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8);
|
|
const projectManagementSamples = webObserveArray(projectManagement?.samples).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8);
|
|
const projectManagementApiByPath = webObserveArray(projectManagement?.projectApiByPath).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8);
|
|
const projectManagementSlowApis = webObserveArray(projectManagement?.slowProjectApiPerformance).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8);
|
|
const alertThresholds = record(analysis?.alertThresholds ?? pagePerformanceSummary?.alertThresholds ?? value.alertThresholds);
|
|
const budgetLabel = (rawValue: unknown): string => {
|
|
const parsed = Number(rawValue);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) return "unconfigured";
|
|
const ms = parsed;
|
|
return ms % 1000 === 0 ? `${Math.round(ms / 1000)}s` : `${ms}ms`;
|
|
};
|
|
const sameOriginApiBudgetLabel = budgetLabel(alertThresholds?.sameOriginApiSlowMs ?? pagePerformanceSummary?.budgetMs);
|
|
const projectManagementApiBudgetLabel = budgetLabel(projectManagementSummary?.slowApiBudgetMs ?? alertThresholds?.sameOriginApiSlowMs ?? pagePerformanceSummary?.budgetMs);
|
|
const partialApiBudgetLabel = budgetLabel(alertThresholds?.partialApiSlowMs);
|
|
const streamOpenBudgetLabel = budgetLabel(alertThresholds?.longLivedStreamOpenSlowMs);
|
|
const loadingBudgetLabel = budgetLabel(alertThresholds?.visibleLoadingSlowMs);
|
|
const pageLabel = (item: Record<string, unknown>): string => {
|
|
const role = webObserveText(item.pageRole);
|
|
const pageId = webObserveShort(webObserveText(item.pageId), 18);
|
|
const epoch = webObserveText(item.pageEpoch);
|
|
if (role === "-" && pageId === "-") return "-";
|
|
return epoch === "-" ? `${role}/${pageId}` : `${role}/${pageId}#${epoch}`;
|
|
};
|
|
const promptNetwork = record(analysis?.promptNetwork);
|
|
const loading = record(sampleMetrics?.loading);
|
|
const loadingSummary = record(loading?.summary);
|
|
const traceOrder = record(sampleMetrics?.traceOrder);
|
|
const traceOrderSummary = record(traceOrder?.summary);
|
|
const traceOrderAnomalies = Array.isArray(traceOrder?.orderAnomalies) ? traceOrder.orderAnomalies.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const loadingSegments = Array.isArray(loading?.longestSegments) ? loading.longestSegments.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const loadingOwners = Array.isArray(loading?.owners) ? loading.owners.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const sessionRailTitles = record(sampleMetrics?.sessionRailTitles);
|
|
const sessionRailTitleSummary = record(sessionRailTitles?.summary);
|
|
const sessionRailTitleSamples = Array.isArray(sessionRailTitles?.samples) ? sessionRailTitles.samples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const sessionRailTitleExamples = Array.isArray(sessionRailTitles?.examples) ? sessionRailTitles.examples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const roundSource = Array.isArray(sampleMetrics?.roundItems) && sampleMetrics.roundItems.length > 0
|
|
? sampleMetrics.roundItems
|
|
: Array.isArray(sampleMetrics?.rounds)
|
|
? sampleMetrics.rounds
|
|
: sampleMetrics?.roundItems ?? sampleMetrics?.rounds;
|
|
const rounds = webObserveArray(roundSource).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8);
|
|
const turnColumns = webObserveArray(sampleMetrics?.turnColumns).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-12);
|
|
const roundCount = Array.isArray(sampleMetrics?.rounds) ? sampleMetrics.rounds.length : sampleMetrics?.rounds;
|
|
const turnColumnCount = Array.isArray(sampleMetrics?.turnColumns) ? sampleMetrics.turnColumns.length : sampleMetrics?.turnColumns;
|
|
const slowApis = Array.isArray(analysis?.pagePerformanceSlowApi) ? analysis.pagePerformanceSlowApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const archiveSlowApis = Array.isArray(analysis?.archivePagePerformanceSlowApi) ? analysis.archivePagePerformanceSlowApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const partialApis = Array.isArray(analysis?.pagePerformancePartialApi) ? analysis.pagePerformancePartialApi.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const sseStreams = Array.isArray(analysis?.pagePerformanceSseStreams) ? analysis.pagePerformanceSseStreams.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const allFindingRecords = Array.isArray(analysis?.findings) ? analysis.findings.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
|
const explicitToolFindings = Array.isArray(analysis?.toolFindings) ? analysis.toolFindings.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
|
const inferredToolFindings = allFindingRecords.filter((item) => String(item.id ?? item.kind ?? item.code ?? "").startsWith("tool-"));
|
|
const toolFindings = dedupeWebObserveRecords(explicitToolFindings.length > 0 ? explicitToolFindings : inferredToolFindings, (item) => String(item.id ?? item.kind ?? item.code ?? "")).slice(0, 8);
|
|
const commandState = record(analysis?.commandState);
|
|
const findings = allFindingRecords.slice(0, 8);
|
|
const archiveRedFindings = Array.isArray(analysis?.archiveRedFindings)
|
|
? analysis.archiveRedFindings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8)
|
|
: Array.isArray(archiveSummary?.redFindings)
|
|
? archiveSummary.redFindings.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8)
|
|
: [];
|
|
const httpErrorGroups = Array.isArray(analysis?.httpErrorGroups) ? analysis.httpErrorGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const requestFailedGroups = Array.isArray(analysis?.requestFailedGroups) ? analysis.requestFailedGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const domDiagnostics = Array.isArray(analysis?.domDiagnosticGroups) ? analysis.domDiagnosticGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 5) : [];
|
|
const domDiagnosticSamples = Array.isArray(analysis?.domDiagnosticSamples) ? analysis.domDiagnosticSamples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const consoleAlertGroups = Array.isArray(analysis?.consoleAlertGroups) ? analysis.consoleAlertGroups.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const consoleAlertSamples = Array.isArray(analysis?.consoleAlertSamples) ? analysis.consoleAlertSamples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const runnerErrors = Array.isArray(analysis?.runnerErrors) ? analysis.runnerErrors.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
|
|
const commandFailures = Array.isArray(analysis?.commandFailures) ? analysis.commandFailures.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-8) : [];
|
|
const recentUpdateJumps = Array.isArray(analysis?.turnTimingRecentUpdateJumps) ? analysis.turnTimingRecentUpdateJumps.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const elapsedZeroResets = Array.isArray(analysis?.turnTimingElapsedZeroResets) ? analysis.turnTimingElapsedZeroResets.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const elapsedForwardJumps = Array.isArray(analysis?.turnTimingTotalElapsedForwardJumps) ? analysis.turnTimingTotalElapsedForwardJumps.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 8) : [];
|
|
const domDiagnosticRows = domDiagnostics.length > 0
|
|
? domDiagnostics.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveShort(webObserveText(item.firstAt), 24),
|
|
webObserveShort(webObserveText(item.lastAt), 24),
|
|
webObserveShort(webObserveText(item.text), 96),
|
|
])
|
|
: domDiagnosticSamples.map((item) => [
|
|
webObserveText(item.seq),
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort([
|
|
webObserveText(item.source),
|
|
webObserveText(item.diagnosticCode),
|
|
item.traceId ? `trace=${webObserveText(item.traceId)}` : "",
|
|
item.httpStatus !== undefined && item.httpStatus !== null ? `http=${webObserveText(item.httpStatus)}` : "",
|
|
item.idleSeconds !== undefined && item.idleSeconds !== null ? `idle=${webObserveText(item.idleSeconds)}` : "",
|
|
item.waitingFor ? `waitingFor=${webObserveText(item.waitingFor)}` : "",
|
|
item.lastEventLabel ? `lastEvent=${webObserveText(item.lastEventLabel)}` : "",
|
|
].filter((part) => part !== "" && part !== "-").join(" "), 64),
|
|
webObserveShort(webObserveText(item.text), 96),
|
|
]);
|
|
const consoleAlertRows = consoleAlertGroups.length > 0
|
|
? consoleAlertGroups.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveShort(webObserveText(item.type), 14),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.traceIds).join(",") || "-", 28),
|
|
])
|
|
: consoleAlertSamples.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.type), 14),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.traceId), 28),
|
|
webObserveShort(webObserveText(item.text), 72),
|
|
]);
|
|
const recentUpdateJumpRows = recentUpdateJumps.map((item) => [
|
|
webObserveShort(webObserveText(item.columnLabel), 14),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.fromSeq),
|
|
webObserveText(item.toSeq),
|
|
`${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`,
|
|
webObserveText(item.delta),
|
|
webObserveText(item.sampleDeltaSeconds),
|
|
webObserveText(item.allowedIncreaseSeconds),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
]);
|
|
const elapsedZeroResetRows = elapsedZeroResets.map((item) => [
|
|
webObserveShort(webObserveText(item.columnLabel), 14),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.fromSeq),
|
|
webObserveText(item.toSeq),
|
|
`${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`,
|
|
webObserveText(item.delta),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
]);
|
|
const elapsedForwardJumpRows = elapsedForwardJumps.map((item) => [
|
|
webObserveShort(webObserveText(item.columnLabel), 14),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.fromSeq),
|
|
webObserveText(item.toSeq),
|
|
`${webObserveText(item.fromValue)}->${webObserveText(item.toValue)}`,
|
|
webObserveText(item.delta),
|
|
webObserveText(item.sampleDeltaSeconds),
|
|
webObserveText(item.allowedIncreaseSeconds),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
]);
|
|
const httpErrorRows = httpErrorGroups.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveText(item.method),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40),
|
|
]);
|
|
const requestFailedRows = requestFailedGroups.map((item) => [
|
|
webObserveText(item.count),
|
|
webObserveText(item.method),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40),
|
|
]);
|
|
const projectManagementCommandRows = projectManagementCommands.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.phase), 12),
|
|
webObserveShort(webObserveText(item.type), 26),
|
|
webObserveText(item.launchStatus ?? item.status),
|
|
webObserveShort(webObserveText(item.sessionId), 24),
|
|
webObserveShort(webObserveText(item.otelTraceId), 18),
|
|
webObserveShort(webObserveText(item.selectedTaskRefHash ?? item.taskHash), 18),
|
|
webObserveShort(webObserveText(item.workbenchUrl ?? item.afterPath), 44),
|
|
webObserveShort(webObserveText(item.message ?? item.errorMessageHash), 72),
|
|
]);
|
|
const projectManagementSampleRows = projectManagementSamples.map((item) => [
|
|
webObserveText(item.seq),
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.pageRole), 12),
|
|
webObserveShort(webObserveText(item.pageKind), 28),
|
|
webObserveShort(webObserveText(item.path), 36),
|
|
webObserveText(item.sourceCount),
|
|
webObserveText(item.fileCount),
|
|
webObserveText(item.taskCount),
|
|
webObserveShort(webObserveText(item.selectedTaskRefHash), 18),
|
|
webObserveText(item.launchButtonEnabled),
|
|
webObserveText(item.workbenchLinkCount),
|
|
]);
|
|
const projectManagementApiRows = projectManagementApiByPath.map((item) => [
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveText(item.type), 12),
|
|
webObserveText(item.method),
|
|
webObserveText(item.status),
|
|
webObserveShort(webObserveText(item.path ?? item.urlPath), 52),
|
|
webObserveShort(webObserveText(item.lastAt ?? item.firstAt), 24),
|
|
webObserveShort(webObserveArray(item.failureKinds).join(",") || "-", 40),
|
|
]);
|
|
const projectManagementSlowRows = projectManagementSlowApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.p95Ms ?? item.p95),
|
|
webObserveText(item.maxMs ?? item.max),
|
|
webObserveText(item.overBudgetCount ?? item.overFiveSecondCount),
|
|
webObserveShort(webObserveArray(item.slowSamples).map((sample) => webObserveText(record(sample)?.otelTraceId)).filter((text) => text !== "-").join(",") || "-", 36),
|
|
]);
|
|
const frontendPerformance = record(analysis?.frontendPerformance);
|
|
const frontendPerformanceSummary = nonEmptyRecord(frontendPerformance?.summary) ?? frontendPerformance;
|
|
const frontendPerformanceHotspots = record(analysis?.frontendPerformanceHotspots);
|
|
const frontendProfileFunctions = webObserveArray(frontendPerformanceHotspots?.profileFunctions ?? frontendPerformance?.profileFunctions ?? frontendPerformance?.profileHotspots).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 4);
|
|
const frontendProfileStacks = webObserveArray(frontendPerformanceHotspots?.profileStacks ?? frontendPerformance?.profileStacks).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 3);
|
|
const frontendScriptHotspots = webObserveArray(frontendPerformanceHotspots?.scripts ?? frontendPerformance?.scripts ?? frontendPerformance?.scriptHotspots).map(record).filter((item): item is Record<string, unknown> => item !== null).slice(0, 4);
|
|
const highSignalFindings = dedupeWebObserveRecords(
|
|
[...toolFindings, ...findings, ...archiveRedFindings],
|
|
(item) => String(item.kind ?? item.id ?? item.code ?? ""),
|
|
).slice(0, 6);
|
|
const id = webObserveText(value.id);
|
|
const statusLine = value.ok === false
|
|
? `blocked reason=${webObserveText(record(value.failure)?.reason)} exit=${webObserveText(result.exitCode)} timeout=${webObserveText(result.timedOut)} recovered=${webObserveText(record(value.failure)?.recoveredFromArtifacts)}`
|
|
: `ok=${webObserveText(value.ok !== false)} exit=${webObserveText(result.exitCode)} timeout=${webObserveText(result.timedOut)}`;
|
|
const compactLines = [
|
|
`web-probe observe analyze (${webObserveText(value.status)})`,
|
|
"",
|
|
webObserveTable(["ID", "NODE", "LANE", "STATUS", "SCOPE", "STATE_DIR"], [[
|
|
id,
|
|
value.node,
|
|
value.lane,
|
|
statusLine,
|
|
webObserveShort([webObserveText(jsonlScope?.mode ?? "current"), webObserveText(record(jsonlScope?.focus)?.mode), jsonlScope?.archivePrefix ? webObserveText(jsonlScope.archivePrefix) : ""].filter((part) => part && part !== "-").join(":"), 36),
|
|
webObserveShort(webObserveText(analysis?.stateDir), 72),
|
|
]]),
|
|
"",
|
|
"Artifacts:",
|
|
webObserveTable(["KIND", "PATH", "SHA256"], [
|
|
["report.json", webObserveShort(webObserveText(analysis?.reportJsonPath), 72), webObserveShort(webObserveText(analysis?.reportJsonSha256), 24)],
|
|
["report.md", webObserveShort(webObserveText(analysis?.reportMdPath), 72), webObserveShort(webObserveText(analysis?.reportMdSha256), 24)],
|
|
]),
|
|
"",
|
|
"Counts:",
|
|
webObserveTable(["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ARCHIVE_SAMPLES", "ARCHIVE_RED"], [[
|
|
webObserveText(counts?.samples),
|
|
webObserveText(counts?.control),
|
|
webObserveText(counts?.network),
|
|
webObserveText(counts?.console),
|
|
webObserveText(counts?.artifacts),
|
|
webObserveText(record(archiveSummary?.sampleMetrics)?.sampleCount),
|
|
webObserveText(archiveSummary?.redFindingCount),
|
|
]]),
|
|
"",
|
|
"Tool state:",
|
|
webObserveTable(["PENDING", "PROCESSING", "ABANDONED", "FAILED", "RUNNER_ERRORS", "COMMAND_FAILURES"], [[
|
|
webObserveText(commandState?.pendingCount),
|
|
webObserveText(commandState?.processingCount),
|
|
webObserveText(commandState?.abandonedCount),
|
|
webObserveText(commandState?.failedCount),
|
|
webObserveText(runnerErrors.length),
|
|
webObserveText(commandFailures.length),
|
|
]]),
|
|
"",
|
|
"Findings:",
|
|
webObserveTable(["KIND", "SEVERITY", "COUNT", "ROOT_CAUSE", "SUMMARY"], highSignalFindings.length > 0 ? highSignalFindings.map((item) => [
|
|
webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 38),
|
|
webObserveText(item.severity ?? item.level),
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveText(item.rootCause ?? item.rootCauseStatus), 38),
|
|
webObserveShort(webObserveText(item.summary ?? item.message), 104),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Turn/loading:",
|
|
webObserveTable(["ROUNDS", "TURNS", "ROWS", "ELAPSED_JUMP", "RECENT_JUMP", "LOADING_SAMPLES", "MAX_LOADING", `OVER_${loadingBudgetLabel}`], [[
|
|
webObserveText(roundCount),
|
|
webObserveText(turnColumnCount),
|
|
webObserveText(sampleMetrics?.turnTimingRows),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedForwardJumpCount ?? elapsedForwardJumps.length),
|
|
webObserveText(sampleMetrics?.turnTimingRecentUpdateJumpCount ?? sampleMetrics?.turnTimingRecentUpdateSawtoothJumpCount),
|
|
webObserveText(loadingSummary?.loadingSampleCount ?? sampleMetrics?.loadingSampleCount),
|
|
webObserveText(loadingSummary?.maxSimultaneousCount ?? sampleMetrics?.loadingMaxCount),
|
|
webObserveText(loadingSummary?.overBudgetSegmentCount ?? loadingSummary?.overFiveSecondSegmentCount ?? sampleMetrics?.loadingOverFiveSecondSegmentCount),
|
|
]]),
|
|
"",
|
|
"Request/performance:",
|
|
webObserveTable(["REQUESTS", "TOTAL_PEAK", "PAGE_PEAK", "API_PEAK", "SLOW_API", "ARCHIVE_SLOW", "SSE_SLOW"], [[
|
|
webObserveText(requestRateSummary?.requestCount),
|
|
webObserveText(requestRateSummary?.totalPeakPerMinute),
|
|
webObserveText(requestRateSummary?.pagePeakPerMinute),
|
|
webObserveText(requestRateSummary?.apiPathPeakPerMinute),
|
|
webObserveText(slowApis.length),
|
|
webObserveText(archiveSlowApis.length),
|
|
webObserveText(sseStreams.length),
|
|
]]),
|
|
webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER"], (slowApis.length > 0 ? slowApis : archiveSlowApis).slice(0, 4).map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 56),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.p95Ms ?? item.p95),
|
|
webObserveText(item.maxMs ?? item.max),
|
|
webObserveText(item.overBudgetCount ?? item.overFiveSecondCount),
|
|
]).concat((slowApis.length > 0 || archiveSlowApis.length > 0) ? [] : [["-", "-", "-", "-", "-"]])),
|
|
"",
|
|
"Frontend performance:",
|
|
webObserveTable(["EVENTS", "LONGTASK", "LOAF", "GAP", "CAPTURES", "MAX_LONGTASK", "MAX_LOAF", "MAX_GAP"], [[
|
|
webObserveText(frontendPerformanceSummary?.eventCount),
|
|
webObserveText(frontendPerformanceSummary?.longTaskCount),
|
|
webObserveText(frontendPerformanceSummary?.longAnimationFrameCount),
|
|
webObserveText(frontendPerformanceSummary?.eventLoopGapCount),
|
|
webObserveText(frontendPerformanceSummary?.captureCount),
|
|
webObserveText(frontendPerformanceSummary?.maxLongTaskMs),
|
|
webObserveText(frontendPerformanceSummary?.maxLongAnimationFrameMs),
|
|
webObserveText(frontendPerformanceSummary?.maxEventLoopGapMs),
|
|
]]),
|
|
webObserveTable(["CPU_SELF_MS", "TOTAL_MS", "HITS", "FUNCTION", "URL"], frontendProfileFunctions.length > 0 ? frontendProfileFunctions.map((item) => [
|
|
webObserveText(item.selfTimeMs),
|
|
webObserveText(item.totalTimeMs),
|
|
webObserveText(item.hitCount),
|
|
webObserveShort(webObserveText(item.functionName ?? item.name), 42),
|
|
webObserveShort(webObserveText(item.url ?? item.sourceURL ?? item.scriptUrl), 64),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
webObserveTable(["STACK_SELF_MS", "FRAMES", "FUNCTION", "TOP_FRAME"], frontendProfileStacks.length > 0 ? frontendProfileStacks.map((item) => {
|
|
const frames = webObserveArray(item.frames).map(record).filter((frame): frame is Record<string, unknown> => frame !== null);
|
|
const topFrame = frames[0] ?? {};
|
|
return [
|
|
webObserveText(item.selfTimeMs),
|
|
webObserveText(item.frameCount ?? frames.length),
|
|
webObserveShort(webObserveText(item.functionName ?? item.name), 38),
|
|
webObserveShort(`${webObserveText(topFrame.functionName ?? "(anonymous)")}@${webObserveText(topFrame.url ?? topFrame.scriptId)}`, 68),
|
|
];
|
|
}) : [["-", "-", "-", "-"]]),
|
|
webObserveTable(["LOAF_MS", "COUNT", "FUNCTION", "URL"], frontendScriptHotspots.length > 0 ? frontendScriptHotspots.map((item) => [
|
|
webObserveText(item.totalDurationMs ?? item.durationMs),
|
|
webObserveText(item.count ?? item.hitCount),
|
|
webObserveShort(webObserveText(item.sourceFunctionName ?? item.functionName ?? item.invoker), 42),
|
|
webObserveShort(webObserveText(item.sourceURL ?? item.url), 64),
|
|
]) : [["-", "-", "-", "-"]]),
|
|
"",
|
|
"Next:",
|
|
` bun scripts/cli.ts web-probe observe collect ${id} --view turn-summary`,
|
|
` bun scripts/cli.ts web-probe observe collect ${id} --view timeline --command-id <commandId>`,
|
|
` bun scripts/cli.ts web-probe observe collect ${id} --view trace-frame --trace-id <traceId> --sample-seq <seq>`,
|
|
` bun scripts/cli.ts web-probe observe collect ${id} --view workbench-triad --trace-id <traceId>`,
|
|
` bun scripts/cli.ts web-probe observe collect ${id} --view performance-summary`,
|
|
` bun scripts/cli.ts web-probe observe collect ${id} --view files --file analysis/report.json --finding <findingId>`,
|
|
"",
|
|
"Disclosure:",
|
|
" default analyze output is a bounded index; full offline report stays in report.md/report.json artifacts.",
|
|
];
|
|
if (value.full !== true) return compactLines.join("\n");
|
|
|
|
const lines = [
|
|
`web-probe observe analyze (${webObserveText(value.status)})`,
|
|
"",
|
|
...(value.ok === false ? [
|
|
"Blocked detail:",
|
|
webObserveTable(["EXIT", "TIMEOUT", "ANALYZER", "STDOUT", "STDERR"], [[
|
|
webObserveText(result.exitCode),
|
|
webObserveText(result.timedOut),
|
|
webObserveShort([
|
|
analyzer?.recoveredFrom ? `source=${webObserveText(analyzer.recoveredFrom)}` : "",
|
|
analyzer?.exitCode !== undefined && analyzer?.exitCode !== null ? `exit=${webObserveText(analyzer.exitCode)}` : "",
|
|
analyzer?.stdoutBytes !== undefined && analyzer?.stdoutBytes !== null ? `stdout=${webObserveText(analyzer.stdoutBytes)}B` : "",
|
|
analyzer?.stderrBytes !== undefined && analyzer?.stderrBytes !== null ? `stderr=${webObserveText(analyzer.stderrBytes)}B` : "",
|
|
].filter((part) => part !== "").join(" "), 120),
|
|
webObserveShort(webObserveText(result.stdoutTail), 120),
|
|
webObserveShort(webObserveText(analyzer?.stderrTail ?? result.stderr), 500),
|
|
]]),
|
|
"Analyzer artifacts:",
|
|
webObserveTable(["KIND", "PATH"], [
|
|
["stdout", webObserveShort(webObserveText(analyzer?.stdoutPath), 96)],
|
|
["stderr", webObserveShort(webObserveText(analyzer?.stderrPath), 96)],
|
|
]),
|
|
"",
|
|
] : []),
|
|
webObserveTable(["ID", "NODE", "LANE", "SCOPE", "SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS"], [[
|
|
webObserveText(value.id),
|
|
webObserveText(value.node),
|
|
webObserveText(value.lane),
|
|
webObserveShort([webObserveText(jsonlScope?.mode ?? "current"), webObserveText(record(jsonlScope?.focus)?.mode), jsonlScope?.archivePrefix ? webObserveText(jsonlScope.archivePrefix) : ""].filter((part) => part && part !== "-").join(":"), 36),
|
|
webObserveText(counts?.samples),
|
|
webObserveText(counts?.control),
|
|
webObserveText(counts?.network),
|
|
webObserveText(counts?.console),
|
|
webObserveText(counts?.artifacts),
|
|
]]),
|
|
"",
|
|
"Analysis window:",
|
|
webObserveTable(["WINDOW", "FROM", "TO", "SAMPLES", "NETWORK", "CONSOLE", "ARCHIVE_SAMPLES", "ARCHIVE_RED"], [[
|
|
webObserveText(analysisWindow?.name ?? "recent-5m"),
|
|
webObserveShort(webObserveText(analysisWindow?.fromAt), 24),
|
|
webObserveShort(webObserveText(analysisWindow?.toAt), 24),
|
|
webObserveText(analysisWindow?.samples),
|
|
webObserveText(analysisWindow?.network),
|
|
webObserveText(analysisWindow?.console),
|
|
webObserveText(record(archiveSummary?.sampleMetrics)?.sampleCount ?? counts?.samples),
|
|
webObserveText(archiveSummary?.redFindingCount),
|
|
]]),
|
|
"",
|
|
"Reports:",
|
|
webObserveTable(["KIND", "PATH", "SHA256"], [
|
|
["json", webObserveShort(webObserveText(analysis?.reportJsonPath), 96), webObserveShort(webObserveText(analysis?.reportJsonSha256), 24)],
|
|
["md", webObserveShort(webObserveText(analysis?.reportMdPath), 96), webObserveShort(webObserveText(analysis?.reportMdSha256), 24)],
|
|
]),
|
|
"",
|
|
"Project management:",
|
|
webObserveTable(["ENABLED", "SAMPLES", "MDTODO", "LATEST", "SRC", "FILES", "TASKS", "SELECTED_TASK", "LAUNCH", "OTEL", "API", `SLOW>${projectManagementApiBudgetLabel}`], [[
|
|
webObserveText(projectManagementSummary?.enabled),
|
|
webObserveText(projectManagementSummary?.projectSampleCount),
|
|
webObserveText(projectManagementSummary?.mdtodoSampleCount),
|
|
webObserveShort([webObserveText(projectManagementSummary?.latestPageKind), webObserveText(projectManagementSummary?.latestPath)].filter((part) => part !== "-" && part !== "").join(" "), 52),
|
|
webObserveText(projectManagementSummary?.latestSourceCount),
|
|
webObserveText(projectManagementSummary?.latestFileCount),
|
|
webObserveText(projectManagementSummary?.latestTaskCount),
|
|
webObserveShort(webObserveText(projectManagementSummary?.latestSelectedTaskRefHash), 18),
|
|
`ok=${webObserveText(projectManagementSummary?.launchSuccessCount)} fail=${webObserveText(projectManagementSummary?.launchFailureCount)} total=${webObserveText(projectManagementSummary?.launchCommandCount)}`,
|
|
`${webObserveText(projectManagementSummary?.launchWithOtelTraceHeaderCount)}/${webObserveText(projectManagementSummary?.launchSuccessCount)}`,
|
|
`resp=${webObserveText(projectManagementSummary?.projectApiResponseCount)} fail=${webObserveText(projectManagementSummary?.projectApiFailureCount ?? 0)}/${webObserveText(projectManagementSummary?.projectApiRequestFailedCount ?? 0)}`,
|
|
webObserveText(projectManagementSummary?.projectApiSlowPathCount),
|
|
]]),
|
|
"",
|
|
"Project management samples:",
|
|
webObserveTable(["SEQ", "TS", "ROLE", "KIND", "PATH", "SRC", "FILES", "TASKS", "SELECTED", "LAUNCH", "LINKS"], projectManagementSampleRows.length > 0 ? projectManagementSampleRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Project management commands:",
|
|
webObserveTable(["TS", "PHASE", "TYPE", "STATUS", "SESSION", "OTEL", "TASK", "URL", "MESSAGE"], projectManagementCommandRows.length > 0 ? projectManagementCommandRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Project management API:",
|
|
webObserveTable(["COUNT", "TYPE", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], projectManagementApiRows.length > 0 ? projectManagementApiRows : [["-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Project management slow API (>${projectManagementApiBudgetLabel}):`,
|
|
webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET", "OTEL"], projectManagementSlowRows.length > 0 ? projectManagementSlowRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Turn timing:",
|
|
webObserveTable(["ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "ZERO_RESET", "ELAPSED_JUMP", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[
|
|
webObserveText(roundCount),
|
|
webObserveText(turnColumnCount),
|
|
webObserveText(sampleMetrics?.turnTimingRows),
|
|
webObserveText(sampleMetrics?.turnTimingNonMonotonicCount),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedDecreaseCount),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedZeroResetCount ?? elapsedZeroResets.length),
|
|
webObserveText(sampleMetrics?.turnTimingTotalElapsedForwardJumpCount ?? elapsedForwardJumps.length),
|
|
webObserveText(sampleMetrics?.turnTimingTerminalElapsedGrowthCount),
|
|
webObserveText(sampleMetrics?.turnTimingRecentUpdateJumpCount ?? sampleMetrics?.turnTimingRecentUpdateSawtoothJumpCount),
|
|
webObserveText(sampleMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds),
|
|
]]),
|
|
"",
|
|
"Trace row order:",
|
|
webObserveTable(["SAMPLES", "TRACE_ROWS", "ORDER_ANOMALY", "COMPLETION_NOT_LAST"], [[
|
|
webObserveText(traceOrderSummary?.sampleCount),
|
|
webObserveText(traceOrderSummary?.traceRowCount),
|
|
webObserveText(traceOrderSummary?.orderAnomalyCount),
|
|
webObserveText(traceOrderSummary?.completionNotLastCount),
|
|
]]),
|
|
"",
|
|
"Trace row order anomalies:",
|
|
traceOrderAnomalies.length === 0
|
|
? "-"
|
|
: webObserveTable(["SEQ", "PAGE", "TRACE", "ROWS", "REASONS", "TOTAL", "CLOCK", "PREVIEW"], traceOrderAnomalies.map((item) => [
|
|
webObserveText(item.sampleIndex ?? item.seq),
|
|
pageLabel(item),
|
|
webObserveShort(webObserveText(item.traceId), 24),
|
|
`${webObserveText(item.previousRowIndex)}->${webObserveText(item.currentRowIndex)}`,
|
|
webObserveShort(webObserveArray(item.reasons).map(webObserveText).join(",") || "-", 48),
|
|
`${webObserveText(item.previousTotalSeconds)}->${webObserveText(item.currentTotalSeconds)}`,
|
|
`${webObserveText(item.previousClockSeconds)}->${webObserveText(item.currentClockSeconds)}`,
|
|
webObserveShort(`${webObserveText(item.previousPreview)} / ${webObserveText(item.currentPreview)}`, 96),
|
|
])),
|
|
"",
|
|
"Archive turn timing:",
|
|
webObserveTable(["SAMPLES", "ROUNDS", "TURNS", "ROWS", "NON_MONO", "ELAPSED_DEC", "ZERO_RESET", "ELAPSED_JUMP", "TERMINAL_GROWTH", "RECENT_JUMP", "MAX_RECENT_STEP"], [[
|
|
webObserveText(archiveMetrics?.sampleCount),
|
|
webObserveText(archiveMetrics?.rounds),
|
|
webObserveText(archiveMetrics?.turnColumns),
|
|
webObserveText(archiveMetrics?.turnTimingRows),
|
|
webObserveText(archiveMetrics?.turnTimingNonMonotonicCount),
|
|
webObserveText(archiveMetrics?.turnTimingTotalElapsedDecreaseCount),
|
|
webObserveText(archiveMetrics?.turnTimingTotalElapsedZeroResetCount),
|
|
webObserveText(archiveMetrics?.turnTimingTotalElapsedForwardJumpCount),
|
|
webObserveText(archiveMetrics?.turnTimingTerminalElapsedGrowthCount),
|
|
webObserveText(archiveMetrics?.turnTimingRecentUpdateJumpCount ?? archiveMetrics?.turnTimingRecentUpdateSawtoothJumpCount),
|
|
webObserveText(archiveMetrics?.turnTimingRecentUpdateMaxIncreaseSeconds),
|
|
]]),
|
|
"",
|
|
"Rounds:",
|
|
webObserveTable(["ROUND", "SAMPLES", "LOADING", "MAX_LOAD", "TOTAL_LAST", "RECENT_LAST", "CARD_DIAG", "TERMINAL", "PROMPT_HASH"], rounds.length > 0 ? rounds.map((item) => [
|
|
webObserveText(item.promptIndex),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.loadingSamples),
|
|
webObserveText(item.maxLoadingCount),
|
|
webObserveText(item.lastTotalElapsedSeconds),
|
|
webObserveText(item.lastRecentUpdateSeconds),
|
|
webObserveText(item.diagnosticSamples),
|
|
webObserveText(item.terminalSamples),
|
|
webObserveShort(webObserveText(item.promptTextHash), 24),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Loading visibility:",
|
|
webObserveTable(["SAMPLES", "LOADING_SAMPLES", "MAX_COUNT", "MAX_OWNERS", "OWNERS", "LONGEST_S", "CURRENT_S", `OVER_${loadingBudgetLabel}`], [[
|
|
webObserveText(loadingSummary?.sampleCount ?? sampleMetrics?.sampleCount),
|
|
webObserveText(loadingSummary?.loadingSampleCount ?? sampleMetrics?.loadingSampleCount),
|
|
webObserveText(loadingSummary?.maxSimultaneousCount ?? sampleMetrics?.loadingMaxCount),
|
|
webObserveText(loadingSummary?.maxSimultaneousOwnerCount ?? sampleMetrics?.loadingMaxOwnerCount),
|
|
webObserveText(loadingSummary?.ownerCount ?? sampleMetrics?.loadingOwnerCount),
|
|
webObserveText(loadingSummary?.longestContinuousSeconds ?? sampleMetrics?.loadingLongestContinuousSeconds),
|
|
webObserveText(loadingSummary?.currentContinuousSeconds ?? sampleMetrics?.loadingCurrentContinuousSeconds),
|
|
webObserveText(loadingSummary?.overBudgetSegmentCount ?? loadingSummary?.overFiveSecondSegmentCount ?? sampleMetrics?.loadingOverFiveSecondSegmentCount),
|
|
]]),
|
|
"",
|
|
"Session rail titles:",
|
|
webObserveTable(["SAMPLES", "VISIBLE", "MAJORITY", "MAX_RATIO", "MAX_VISIBLE", "MAX_FALLBACK"], [[
|
|
webObserveText(sessionRailTitleSummary?.sampleCount ?? sampleMetrics?.sessionRailSampleCount ?? archiveMetrics?.sessionRailSampleCount),
|
|
webObserveText(sessionRailTitleSummary?.visibleSampleCount ?? sampleMetrics?.sessionRailVisibleSampleCount ?? archiveMetrics?.sessionRailVisibleSampleCount),
|
|
webObserveText(sessionRailTitleSummary?.majorityFallbackSampleCount ?? sampleMetrics?.sessionRailFallbackMajoritySampleCount ?? archiveMetrics?.sessionRailFallbackMajoritySampleCount),
|
|
webObserveText(sessionRailTitleSummary?.maxFallbackRatio ?? sampleMetrics?.sessionRailFallbackMaxRatio ?? archiveMetrics?.sessionRailFallbackMaxRatio),
|
|
webObserveText(sessionRailTitleSummary?.maxVisibleCount ?? sampleMetrics?.sessionRailFallbackMaxVisibleCount ?? archiveMetrics?.sessionRailFallbackMaxVisibleCount),
|
|
webObserveText(sessionRailTitleSummary?.maxFallbackTitleCount ?? sampleMetrics?.sessionRailFallbackMaxCount ?? archiveMetrics?.sessionRailFallbackMaxCount),
|
|
]]),
|
|
sessionRailTitleSamples.length === 0 ? "Session rail fallback samples:\n-" : webObserveTable(["SEQ", "ROLE", "VISIBLE", "FALLBACK", "RATIO", "EXAMPLE"], sessionRailTitleSamples.map((item) => [
|
|
webObserveText(item.seq),
|
|
webObserveText(item.pageRole),
|
|
webObserveText(item.visibleCount),
|
|
webObserveText(item.fallbackTitleCount),
|
|
webObserveText(item.fallbackTitleRatio),
|
|
webObserveShort(webObserveText((Array.isArray(item.examples) ? record(item.examples[0])?.titlePreview : null) ?? ""), 48),
|
|
])),
|
|
sessionRailTitleExamples.length === 0 ? "Session rail fallback examples:\n-" : webObserveTable(["SEQ", "ROLE", "ACTIVE", "SESSION", "TITLE"], sessionRailTitleExamples.map((item) => [
|
|
webObserveText(item.firstSeq),
|
|
webObserveText(item.pageRole),
|
|
webObserveText(item.active),
|
|
webObserveText(item.sessionIdPrefix),
|
|
webObserveShort(webObserveText(item.titlePreview ?? item.titleHash), 64),
|
|
])),
|
|
"",
|
|
"Loading owners:",
|
|
webObserveTable(["SAMPLES", "OCCUR", "MAX", "LONGEST_S", "KIND", "TRACE", "MESSAGE", "SESSION", "OWNER"], loadingOwners.length > 0 ? loadingOwners.map((item) => [
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.occurrenceCount),
|
|
webObserveText(item.maxSimultaneousCount),
|
|
webObserveText(item.longestContinuousSeconds),
|
|
webObserveShort(webObserveText(item.ownerKind), 16),
|
|
webObserveShort(webObserveText(item.ownerTraceId), 24),
|
|
webObserveShort(webObserveText(item.ownerMessageId), 24),
|
|
webObserveShort(webObserveText(item.ownerSessionId), 24),
|
|
webObserveShort(webObserveText(item.ownerLabel ?? item.ownerKey), 64),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Loading segments:",
|
|
webObserveTable(["OBSERVED_S", "UPPER_S", "SAMPLES", "MAX", "OWNERS", "SEQ", "LAST", "STATUS"], loadingSegments.length > 0 ? loadingSegments.map((item) => [
|
|
webObserveText(item.durationSeconds),
|
|
webObserveText(item.upperBoundSeconds ?? item.durationSeconds),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.maxCount),
|
|
webObserveText(item.ownerCount),
|
|
`${webObserveText(item.firstSeq)}..${webObserveText(item.lastSeq)}`,
|
|
webObserveShort(webObserveText(item.lastAt), 24),
|
|
item.ongoing === true ? "ongoing" : webObserveShort(webObserveText(item.endedAt), 24),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Turn columns:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "TRACE", "FIRST_SEQ", "LAST_SEQ", "SOURCE"], turnColumns.length > 0 ? turnColumns.map((item) => [
|
|
webObserveText(item.label),
|
|
pageLabel(item),
|
|
webObserveText(item.promptIndex ?? item.lastPromptIndex),
|
|
webObserveShort(webObserveText(item.traceId), 28),
|
|
webObserveText(item.firstSeq),
|
|
webObserveText(item.lastSeq),
|
|
webObserveShort(webObserveText(item.source), 24),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Recent update jumps:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "SAMPLE_S", "ALLOWED", "TRACE"], recentUpdateJumpRows.length > 0 ? recentUpdateJumpRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Total elapsed zero resets:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "TRACE"], elapsedZeroResetRows.length > 0 ? elapsedZeroResetRows : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Total elapsed forward jumps:",
|
|
webObserveTable(["TURN", "PAGE", "PROMPT", "FROM_SEQ", "TO_SEQ", "VALUE", "DELTA", "SAMPLE_S", "ALLOWED", "TRACE"], elapsedForwardJumpRows.length > 0 ? elapsedForwardJumpRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Runtime alerts:",
|
|
webObserveTable(["HTTP_ERR", "REQUEST_FAILED", "DOM_DIAG", "CONSOLE_ALERT", "PROMPT_SEGMENTS", "PROMPT_NETWORK"], [[
|
|
webObserveText(runtimeAlerts?.httpErrorCount),
|
|
webObserveText(runtimeAlerts?.requestFailedCount),
|
|
webObserveText(runtimeAlerts?.domDiagnosticSampleCount),
|
|
webObserveText(runtimeAlerts?.consoleAlertCount),
|
|
webObserveText(sampleMetrics?.promptSegments),
|
|
webObserveText(promptNetwork?.promptSegments),
|
|
]]),
|
|
"",
|
|
"Request rate:",
|
|
webObserveTable(["BUCKET", "REQUESTS", "TOTAL_PEAK", "PAGE_PEAK", "API_PEAK", "OVER_PEAKS", "THRESHOLDS"], [[
|
|
requestRateSummary?.bucketSeconds ? `${webObserveText(requestRateSummary.bucketSeconds)}s` : webObserveText(requestRateSummary?.bucketMs),
|
|
webObserveText(requestRateSummary?.requestCount),
|
|
webObserveText(requestRateSummary?.totalPeakPerMinute),
|
|
webObserveText(requestRateSummary?.pagePeakPerMinute),
|
|
webObserveText(requestRateSummary?.apiPathPeakPerMinute),
|
|
webObserveText(requestRateSummary?.overThresholdPeakCount),
|
|
`total=${webObserveText(requestRateSummary?.totalRedPerMinute)} page=${webObserveText(requestRateSummary?.pageRedPerMinute)} api=${webObserveText(requestRateSummary?.apiPathRedPerMinute)}`,
|
|
]]),
|
|
requestRatePeaks.length === 0
|
|
? "Request rate peaks:\n-"
|
|
: webObserveTable(["SCOPE", "RPM", "COUNT", "START", "PAGE", "API"], requestRatePeaks.map((item) => [
|
|
webObserveText(item.scope),
|
|
webObserveText(item.requestPerMinute),
|
|
webObserveText(item.count),
|
|
webObserveShort(webObserveText(item.startAt), 24),
|
|
webObserveShort(webObserveText(item.path ?? item.pageKey), 48),
|
|
webObserveShort(webObserveText(item.apiKey), 64),
|
|
])),
|
|
"Request page curves:",
|
|
webObserveTable(["PAGE", "COUNT", "PEAK_RPM", "PEAK_AT"], requestRatePageCurves.length > 0 ? requestRatePageCurves.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.pageKey), 56),
|
|
webObserveText(item.count),
|
|
webObserveText(item.peakRequestPerMinute),
|
|
webObserveShort(webObserveText(record(item.peakBucket)?.startAt), 24),
|
|
]) : [["-", "-", "-", "-"]]),
|
|
"Request API curves:",
|
|
webObserveTable(["API", "COUNT", "PEAK_RPM", "PEAK_AT"], requestRateApiPathCurves.length > 0 ? requestRateApiPathCurves.map((item) => [
|
|
webObserveShort(webObserveText(item.apiKey ?? item.path), 72),
|
|
webObserveText(item.count),
|
|
webObserveText(item.peakRequestPerMinute),
|
|
webObserveShort(webObserveText(record(item.peakBucket)?.startAt), 24),
|
|
]) : [["-", "-", "-", "-"]]),
|
|
"",
|
|
"Runner errors:",
|
|
webObserveTable(["TS", "TYPE", "RETRY", "EXH", "ATTEMPTS", "LAST_KIND", "READY", "MESSAGE"], runnerErrors.length > 0 ? runnerErrors.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.type), 18),
|
|
webObserveText(item.retry),
|
|
webObserveText(item.retryExhausted),
|
|
webObserveText(item.attemptCount),
|
|
webObserveShort(webObserveText(item.lastFailureKind), 24),
|
|
webObserveShort(webObserveText(item.lastReadinessReason), 24),
|
|
webObserveShort(webObserveText(item.message || item.lastError), 96),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Command failures:",
|
|
webObserveTable(["TS", "TYPE", "COMMAND", "DURATION", "SAMPLE", "PATH", "MESSAGE"], commandFailures.length > 0 ? commandFailures.map((item) => [
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
webObserveShort(webObserveText(item.type), 18),
|
|
webObserveShort(webObserveText(item.commandId), 28),
|
|
webObserveText(item.durationMs),
|
|
webObserveText(item.failureSampleOk === true ? item.sampleSeq : "-"),
|
|
webObserveShort([webObserveText(item.beforePath), webObserveText(item.afterPath)].filter((part) => part !== "-" && part !== "").join("->") || "-", 52),
|
|
webObserveShort(webObserveText(item.message || item.failureKind || item.name), 96),
|
|
]) : [["-", "-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Tool findings:",
|
|
webObserveTable(["KIND", "SEVERITY", "COUNT", "SUMMARY"], toolFindings.length > 0 ? toolFindings.map((item) => [
|
|
webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 36),
|
|
webObserveText(item.severity ?? item.level),
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveText(item.summary ?? item.message), 110),
|
|
]) : [["-", "-", "-", "-"]]),
|
|
webObserveTable(["TOOL_COMMANDS", "COUNT"], [[
|
|
`pending=${webObserveText(commandState?.pendingCount)}`,
|
|
`processing=${webObserveText(commandState?.processingCount)} abandoned=${webObserveText(commandState?.abandonedCount)} failed=${webObserveText(commandState?.failedCount)}`,
|
|
]]),
|
|
"",
|
|
"HTTP error groups:",
|
|
webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], httpErrorRows.length > 0 ? httpErrorRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Request failed groups:",
|
|
webObserveTable(["COUNT", "METHOD", "STATUS", "PATH", "LAST", "FAILURE"], requestFailedRows.length > 0 ? requestFailedRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Slow API (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET"], slowApis.length > 0 ? slowApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.p95Ms ?? item.p95),
|
|
webObserveText(item.maxMs ?? item.max),
|
|
webObserveText(item.overBudgetCount ?? item.overFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Slow API samples (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["TS", "SEQ", "PATH", "DURATION", "REQ_WAIT", "RESP_XFER", "TIMING", "INIT", "PROTO", "SERVER", "OTEL"], (() => {
|
|
const rows = slowApis.flatMap((item) => (Array.isArray(item.slowSamples) ? item.slowSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
webObserveText(sample.seq),
|
|
webObserveShort(webObserveText(sample.path), 44),
|
|
webObserveText(sample.durationMs),
|
|
webObserveText(sample.requestToResponseStartMs ?? sample.streamOpenMs),
|
|
webObserveText(sample.responseTransferMs),
|
|
webObserveShort(webObserveText(sample.timingStatus), 12),
|
|
webObserveShort(webObserveText(sample.initiatorType), 12),
|
|
webObserveShort(webObserveText(sample.nextHopProtocol), 12),
|
|
webObserveShort(Array.isArray(sample.serverTimingNames) ? sample.serverTimingNames.join(",") : "-", 24),
|
|
webObserveShort(webObserveText(sample.otelTraceId), 16),
|
|
]);
|
|
return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]];
|
|
})()),
|
|
"",
|
|
`Partial API timing (>${partialApiBudgetLabel}, not counted as completed slow API):`,
|
|
webObserveTable(["PATH", "SAMPLES", "COMPLETE", "PARTIAL", "PARTIAL_OVER_BUDGET"], partialApis.length > 0 ? partialApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.completeTimingSampleCount),
|
|
webObserveText(item.partialTimingSampleCount),
|
|
webObserveText(item.partialOverBudgetCount ?? item.partialOverFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Partial API samples:",
|
|
webObserveTable(["TS", "SEQ", "PATH", "DURATION", "TIMING", "INIT", "PROTO"], (() => {
|
|
const rows = partialApis.flatMap((item) => (Array.isArray(item.partialSamples) ? item.partialSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
webObserveText(sample.seq),
|
|
webObserveShort(webObserveText(sample.path), 44),
|
|
webObserveText(sample.durationMs),
|
|
webObserveShort(webObserveText(sample.timingStatus), 12),
|
|
webObserveShort(webObserveText(sample.initiatorType), 12),
|
|
webObserveShort(webObserveText(sample.nextHopProtocol), 12),
|
|
]);
|
|
return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-"]];
|
|
})()),
|
|
"",
|
|
"Long-lived streams (SSE):",
|
|
webObserveTable(["PATH", "SAMPLES", "OPEN_P95", "OPEN_MAX", `OPEN>${streamOpenBudgetLabel}`, "LIFE>5S"], sseStreams.length > 0 ? sseStreams.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.streamOpenP95Ms),
|
|
webObserveText(item.streamOpenMaxMs),
|
|
webObserveText(item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount),
|
|
webObserveText(item.streamLifetimeOverFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Archive page performance:",
|
|
webObserveTable(["SAME_ORIGIN_API", "SLOW_PATHS", "SLOW_SAMPLES", "WORST_P95"], [[
|
|
webObserveText(archivePagePerformance?.sameOriginApiPathCount ?? archivePagePerformance?.sameOriginApiPaths),
|
|
webObserveText(archivePagePerformance?.slowPathCount),
|
|
webObserveText(archivePagePerformance?.slowSampleCount),
|
|
webObserveText(archivePagePerformance?.worstP95Ms),
|
|
]]),
|
|
"",
|
|
`Archive slow API (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["PATH", "SAMPLES", "P95", "MAX", "OVER_BUDGET"], archiveSlowApis.length > 0 ? archiveSlowApis.map((item) => [
|
|
webObserveShort(webObserveText(item.path ?? item.route), 52),
|
|
webObserveText(item.sampleCount),
|
|
webObserveText(item.p95Ms ?? item.p95),
|
|
webObserveText(item.maxMs ?? item.max),
|
|
webObserveText(item.overBudgetCount ?? item.overFiveSecondCount),
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
"",
|
|
`Archive slow API samples (>${sameOriginApiBudgetLabel}):`,
|
|
webObserveTable(["TS", "SEQ", "PATH", "DURATION", "REQ_WAIT", "TIMING", "OTEL"], (() => {
|
|
const rows = archiveSlowApis.flatMap((item) => (Array.isArray(item.slowSamples) ? item.slowSamples : []).map((sample) => ({ ...sample, path: sample.path ?? item.path ?? item.route }))).slice(0, 8).map((sample) => [
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
webObserveText(sample.seq),
|
|
webObserveShort(webObserveText(sample.path), 44),
|
|
webObserveText(sample.durationMs),
|
|
webObserveText(sample.requestToResponseStartMs ?? sample.streamOpenMs),
|
|
webObserveShort(webObserveText(sample.timingStatus), 12),
|
|
webObserveShort(webObserveText(sample.otelTraceId), 16),
|
|
]);
|
|
return rows.length > 0 ? rows : [["-", "-", "-", "-", "-", "-", "-"]];
|
|
})()),
|
|
"",
|
|
"DOM diagnostics:",
|
|
webObserveTable(domDiagnostics.length > 0 ? ["COUNT", "FIRST", "LAST", "TEXT"] : ["SEQ", "TS", "META", "TEXT"], domDiagnosticRows.length > 0 ? domDiagnosticRows : [["-", "-", "-", "-"]]),
|
|
"",
|
|
"Console alerts:",
|
|
webObserveTable(consoleAlertGroups.length > 0 ? ["COUNT", "TYPE", "STATUS", "PATH", "LAST", "TRACES"] : ["TS", "TYPE", "STATUS", "PATH", "TRACE", "TEXT"], consoleAlertRows.length > 0 ? consoleAlertRows : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Findings:",
|
|
webObserveTable(["KIND", "SEVERITY", "COUNT", "TIMING", "ROOT_CAUSE", "SUMMARY"], findings.length > 0 ? findings.map((item) => [
|
|
webObserveShort(webObserveText(item.kind ?? item.id ?? item.code), 32),
|
|
webObserveText(item.severity ?? item.level),
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveFindingTiming(item), 42),
|
|
webObserveShort(webObserveText(item.rootCause ?? item.rootCauseStatus), 48),
|
|
webObserveShort(webObserveText(item.summary ?? item.message), 96),
|
|
]) : [["-", "-", "-", "-", "-", "-"]]),
|
|
"",
|
|
"Archive red findings:",
|
|
webObserveTable(["KIND", "COUNT", "SUMMARY"], archiveRedFindings.length > 0 ? archiveRedFindings.map((item) => [
|
|
webObserveShort(webObserveText(item.kind ?? item.code), 36),
|
|
webObserveText(item.count ?? item.sampleCount),
|
|
webObserveShort(webObserveText(item.summary ?? item.message), 110),
|
|
]) : [["-", "-", "-"]]),
|
|
];
|
|
if (pagePerformance !== null && Object.keys(pagePerformance).length > 0) {
|
|
lines.push("", "Page performance:", ` ${webObserveShort(JSON.stringify(pagePerformance), 180)}`);
|
|
}
|
|
lines.push(
|
|
"",
|
|
"Next:",
|
|
` cat ${webObserveText(analysis?.reportMdPath)}`,
|
|
` cat ${webObserveText(analysis?.reportJsonPath)}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded report summary; use report.md/report.json paths for full offline analysis.",
|
|
);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
export function dedupeWebObserveRecords(items: Record<string, unknown>[], keyOf: (item: Record<string, unknown>) => string): Record<string, unknown>[] {
|
|
const out: Record<string, unknown>[] = [];
|
|
const seen = new Set<string>();
|
|
for (const item of items) {
|
|
const key = keyOf(item);
|
|
if (!key || key === "-" || seen.has(key)) continue;
|
|
seen.add(key);
|
|
out.push(item);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export function nodeWebObserveResolveStateDirShell(options: Pick<NodeWebProbeObserveOptions, "stateDir" | "jobId" | "node" | "lane">): string {
|
|
if (options.stateDir !== null) {
|
|
return [
|
|
`state_dir=${shellQuote(options.stateDir)}`,
|
|
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"state-dir-missing\",\"stateDir\":\"%s\"}\\n' \"$state_dir\"; exit 2; }",
|
|
].join("\n");
|
|
}
|
|
if (options.jobId === null) throw new Error("web-probe observe requires --state-dir or --job-id");
|
|
const root = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
|
|
return [
|
|
`observe_root=${shellQuote(root)}`,
|
|
`job_id=${shellQuote(options.jobId)}`,
|
|
"state_dir=$(find \"$observe_root\" -type d -name \"*_${job_id}\" 2>/dev/null | sort | tail -n 1 || true)",
|
|
"test -n \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"job-id-not-found\",\"jobId\":\"%s\",\"root\":\"%s\"}\\n' \"$job_id\" \"$observe_root\"; exit 2; }",
|
|
].join("\n");
|
|
}
|
|
|
|
export function webObserveIndexPath(): string {
|
|
return rootPath(".state/web-observe/index.json");
|
|
}
|
|
|
|
export function readWebObserveIndex(): Record<string, WebObserveIndexEntry> {
|
|
const path = webObserveIndexPath();
|
|
if (!existsSync(path)) return {};
|
|
const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`web-probe observe index is invalid: ${path}`);
|
|
const entries: Record<string, WebObserveIndexEntry> = {};
|
|
for (const [id, value] of Object.entries(parsed)) {
|
|
if (!isSafeWebObserveJobId(id)) continue;
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) continue;
|
|
const recordValue = value as Record<string, unknown>;
|
|
const stateDir = typeof recordValue.stateDir === "string" ? recordValue.stateDir : "";
|
|
const node = typeof recordValue.node === "string" ? recordValue.node : "";
|
|
const lane = typeof recordValue.lane === "string" ? recordValue.lane : "";
|
|
const workspace = typeof recordValue.workspace === "string" ? recordValue.workspace : "";
|
|
if (!isSafeWebObserveStateDir(stateDir) || node.length === 0 || lane.length === 0 || workspace.length === 0) continue;
|
|
entries[id] = {
|
|
id,
|
|
node,
|
|
lane,
|
|
workspace,
|
|
stateDir,
|
|
url: typeof recordValue.url === "string" ? recordValue.url : "",
|
|
targetPath: typeof recordValue.targetPath === "string" ? recordValue.targetPath : "",
|
|
status: typeof recordValue.status === "string" ? recordValue.status : null,
|
|
pid: typeof recordValue.pid === "number" ? recordValue.pid : null,
|
|
startedAt: typeof recordValue.startedAt === "string" ? recordValue.startedAt : null,
|
|
updatedAt: typeof recordValue.updatedAt === "string" ? recordValue.updatedAt : "",
|
|
};
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
export function readWebObserveIndexEntry(id: string): WebObserveIndexEntry | null {
|
|
return readWebObserveIndex()[id] ?? null;
|
|
}
|
|
|
|
export function discoverWebObserveIndexEntry(id: string): WebObserveIndexEntry | null {
|
|
const matches: WebObserveIndexEntry[] = [];
|
|
const errors: string[] = [];
|
|
for (const lane of hwlabRuntimeLaneIds()) {
|
|
for (const node of hwlabRuntimeNodeIds()) {
|
|
let spec: HwlabRuntimeLaneSpec;
|
|
try {
|
|
spec = hwlabRuntimeLaneSpecForNode(lane, node);
|
|
} catch {
|
|
continue;
|
|
}
|
|
try {
|
|
const entry = discoverWebObserveIndexEntryOnTarget(id, node, lane, spec);
|
|
if (entry !== null) matches.push(entry);
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
errors.push(`${node}/${lane}: ${message.slice(0, 240)}`);
|
|
}
|
|
}
|
|
}
|
|
if (matches.length > 1) {
|
|
const candidates = matches.map((entry) => `${entry.node}/${entry.lane}:${entry.stateDir}`).join(", ");
|
|
throw new Error(`web-probe observe observer-id-ambiguous: ${id}; candidates=${candidates}`);
|
|
}
|
|
if (matches.length === 0) {
|
|
if (errors.length > 0) {
|
|
throw new Error(`web-probe observe observer-id-unknown: ${id}; discoveryErrors=${errors.join(" | ")}`);
|
|
}
|
|
return null;
|
|
}
|
|
upsertWebObserveIndexEntry(matches[0]!);
|
|
return matches[0]!;
|
|
}
|
|
|
|
export function discoverWebObserveIndexEntryOnTarget(id: string, node: string, lane: HwlabRuntimeLane, spec: HwlabRuntimeLaneSpec): WebObserveIndexEntry | null {
|
|
const observeRoot = `.state/web-observe/${safeWebObserveSegment(node)}/${safeWebObserveSegment(lane)}`;
|
|
const script = [
|
|
"set -eu",
|
|
`observe_root=${shellQuote(observeRoot)}`,
|
|
`job_id=${shellQuote(id)}`,
|
|
"node - \"$observe_root\" \"$job_id\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const [root, jobId] = process.argv.slice(2);",
|
|
"const matches = [];",
|
|
"function readJson(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } }",
|
|
"function walk(dir, depth) {",
|
|
" if (depth > 10 || matches.length > 1) return;",
|
|
" let entries = [];",
|
|
" try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }",
|
|
" for (const entry of entries) {",
|
|
" if (!entry.isDirectory()) continue;",
|
|
" const full = path.join(dir, entry.name);",
|
|
" if (entry.name.endsWith('_' + jobId)) matches.push(full);",
|
|
" else walk(full, depth + 1);",
|
|
" if (matches.length > 1) return;",
|
|
" }",
|
|
"}",
|
|
"walk(root, 0);",
|
|
"if (matches.length === 0) { console.log(JSON.stringify({ ok: true, found: false, root, jobId })); process.exit(0); }",
|
|
"if (matches.length > 1) { console.log(JSON.stringify({ ok: false, found: true, ambiguous: true, root, jobId, matches })); process.exit(3); }",
|
|
"const stateDir = matches[0];",
|
|
"const manifest = readJson(path.join(stateDir, 'manifest.json')) || {};",
|
|
"const heartbeat = readJson(path.join(stateDir, 'heartbeat.json')) || {};",
|
|
"let pid = null;",
|
|
"try { const raw = fs.readFileSync(path.join(stateDir, 'pid'), 'utf8').trim(); const value = Number(raw); if (Number.isFinite(value)) pid = value; } catch {}",
|
|
"console.log(JSON.stringify({",
|
|
" ok: true,",
|
|
" found: true,",
|
|
" ambiguous: false,",
|
|
" entry: {",
|
|
" id: jobId,",
|
|
" stateDir,",
|
|
" url: typeof manifest.baseUrl === 'string' ? manifest.baseUrl : '',",
|
|
" targetPath: typeof manifest.targetPath === 'string' ? manifest.targetPath : '',",
|
|
" status: typeof manifest.status === 'string' ? manifest.status : null,",
|
|
" pid,",
|
|
" startedAt: typeof manifest.startedAt === 'string' ? manifest.startedAt : null,",
|
|
" updatedAt: typeof heartbeat.updatedAt === 'string' ? heartbeat.updatedAt : new Date().toISOString()",
|
|
" }",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runTransWorkspaceStdinScript(node, spec.workspace, script, 55);
|
|
const payload = parseJsonObject(result.stdout);
|
|
if (result.exitCode !== 0 || result.timedOut || payload?.ok !== true) {
|
|
const reason = result.timedOut ? "timeout" : payload?.ambiguous === true ? "ambiguous" : `exit=${result.exitCode}`;
|
|
throw new Error(`observer discovery failed (${reason}): ${result.stderr.trim().slice(-240) || result.stdout.trim().slice(-240)}`);
|
|
}
|
|
if (payload.found !== true) return null;
|
|
const entry = record(payload.entry);
|
|
const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : "";
|
|
if (!isSafeWebObserveStateDir(stateDir)) throw new Error(`observer discovery returned unsafe stateDir: ${stateDir}`);
|
|
return {
|
|
id,
|
|
node,
|
|
lane,
|
|
workspace: spec.workspace,
|
|
stateDir,
|
|
url: typeof entry.url === "string" ? entry.url : "",
|
|
targetPath: typeof entry.targetPath === "string" ? entry.targetPath : "",
|
|
status: typeof entry.status === "string" ? entry.status : null,
|
|
pid: typeof entry.pid === "number" ? entry.pid : null,
|
|
startedAt: typeof entry.startedAt === "string" ? entry.startedAt : null,
|
|
updatedAt: typeof entry.updatedAt === "string" ? entry.updatedAt : new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
export function upsertWebObserveIndexEntry(entry: WebObserveIndexEntry): Record<string, unknown> {
|
|
const path = webObserveIndexPath();
|
|
try {
|
|
const current = readWebObserveIndex();
|
|
mkdirSync(dirname(path), { recursive: true });
|
|
const next = { ...current, [entry.id]: entry };
|
|
const tmp = `${path}.tmp-${process.pid}`;
|
|
writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 });
|
|
renameSync(tmp, path);
|
|
return {
|
|
ok: true,
|
|
id: entry.id,
|
|
path,
|
|
statusCommand: `bun scripts/cli.ts web-probe observe status ${entry.id}`,
|
|
valuesRedacted: true,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
id: entry.id,
|
|
path,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
fallback: `bun scripts/cli.ts web-probe observe status --node ${entry.node} --lane ${entry.lane} --state-dir ${entry.stateDir}`,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
export function webObserveIdFromOptions(options: Pick<NodeWebProbeObserveOptions, "id" | "jobId">): string | null {
|
|
return options.id ?? options.jobId;
|
|
}
|
|
|
|
export function webObserveIdFromStatus(status: Record<string, unknown> | null, options: Pick<NodeWebProbeObserveOptions, "id" | "jobId">): string | null {
|
|
const manifest = record(status?.manifest);
|
|
const heartbeat = record(status?.heartbeat);
|
|
const manifestJobId = typeof manifest.jobId === "string" ? manifest.jobId : null;
|
|
const heartbeatJobId = typeof heartbeat.jobId === "string" ? heartbeat.jobId : null;
|
|
return manifestJobId ?? heartbeatJobId ?? webObserveIdFromOptions(options);
|
|
}
|
|
|
|
export function webObserveIndexEntryFromOptions(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, id: string, status: Record<string, unknown>): WebObserveIndexEntry {
|
|
const manifest = record(status.manifest);
|
|
const heartbeat = record(status.heartbeat);
|
|
return {
|
|
id,
|
|
node: options.node,
|
|
lane: options.lane,
|
|
workspace: spec.workspace,
|
|
stateDir: options.stateDir ?? "",
|
|
url: typeof manifest.baseUrl === "string" ? manifest.baseUrl : options.url,
|
|
targetPath: typeof manifest.targetPath === "string" ? manifest.targetPath : options.targetPath,
|
|
status: typeof manifest.status === "string" ? manifest.status : null,
|
|
pid: typeof status.pid === "number" ? status.pid : null,
|
|
startedAt: typeof manifest.startedAt === "string" ? manifest.startedAt : null,
|
|
updatedAt: typeof heartbeat.updatedAt === "string" ? heartbeat.updatedAt : new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
export function webObserveCommandLabel(action: NodeWebProbeObserveAction, options: Pick<NodeWebProbeObserveOptions, "id" | "jobId" | "node" | "lane">): string {
|
|
const id = webObserveIdFromOptions(options);
|
|
return id === null
|
|
? `web-probe observe ${action} --node ${options.node} --lane ${options.lane}`
|
|
: `web-probe observe ${action} ${id}`;
|
|
}
|
|
|
|
export function webObserveNextCommands(id: string): Record<string, string> {
|
|
return {
|
|
status: `bun scripts/cli.ts web-probe observe status ${id}`,
|
|
command: `bun scripts/cli.ts web-probe observe command ${id} --type mark --label checkpoint`,
|
|
stop: `bun scripts/cli.ts web-probe observe stop ${id}`,
|
|
analyze: `bun scripts/cli.ts web-probe observe analyze ${id}`,
|
|
};
|
|
}
|
|
|
|
export function webObserveCell(value: unknown, maxLength = 96): string {
|
|
if (value === null || value === undefined) return "-";
|
|
const text = typeof value === "string" ? value : String(value);
|
|
const compact = text.replace(/\s+/gu, " ").trim();
|
|
if (compact.length === 0) return "-";
|
|
if (compact.length <= maxLength) return compact;
|
|
return `${compact.slice(0, Math.max(1, maxLength - 1))}...`;
|
|
}
|
|
|
|
function webObserveFindingTiming(item: Record<string, unknown>): string {
|
|
const status = item.timingStatus ?? null;
|
|
const source = item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind ?? null;
|
|
const parts = [status, source].filter((value) => value !== null && value !== undefined && value !== "");
|
|
if (parts.length === 0) return "-";
|
|
return parts.map((value) => webObserveText(value)).join("/");
|
|
}
|
|
|
|
export function webObserveTable(headers: string[], rows: unknown[][]): string {
|
|
const stringRows = rows.map((row) => row.map((value) => webObserveCell(value)));
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0)));
|
|
const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();
|
|
return [renderRow(headers), ...stringRows.map(renderRow)].join("\n");
|
|
}
|
|
|
|
export function webObserveArray(value: unknown): unknown[] {
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
export function withWebObserveRendered(result: Record<string, unknown>, renderedText: string): Record<string, unknown> {
|
|
return {
|
|
...result,
|
|
renderedText,
|
|
contentType: "text/plain",
|
|
};
|
|
}
|
|
|
|
export function renderWebProbeRunResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(result.summary);
|
|
const probe = record(result.probe);
|
|
const trace = record(probe.trace);
|
|
const session = record(probe.session);
|
|
const artifacts = record(probe.artifacts);
|
|
const reportLoad = record(result.reportLoad);
|
|
const commandTimeout = record(result.commandTimeout);
|
|
const job = record(result.job);
|
|
const start = record(result.start);
|
|
const commandResult = record(result.result ?? result.statusResult);
|
|
const reportPath = reportLoad.path ?? summary.reportPath ?? artifacts.reportPath ?? start.reportPath ?? "-";
|
|
const reportSource = reportLoad.source ?? (result.mode === "async" ? "-" : "stdout");
|
|
const blockedRows = result.ok === true ? [] : [
|
|
"",
|
|
webObserveTable(
|
|
["BLOCKED_FIELD", "VALUE"],
|
|
[
|
|
["failureKind", result.failureKind ?? summary.failureKind ?? "-"],
|
|
["failedCondition", summary.failedCondition ?? result.degradedReason ?? "-"],
|
|
["nextAction", summary.nextAction ?? "-"],
|
|
["exitCode", commandResult.exitCode ?? "-"],
|
|
["timedOut", commandResult.timedOut ?? commandTimeout.timedOut ?? "-"],
|
|
],
|
|
),
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["WEB_PROBE_RUN", "NODE", "LANE", "STATUS", "OK", "DEGRADED", "URL"],
|
|
[[result.command ?? "web-probe run", result.node, result.lane, result.status, result.ok, result.degradedReason ?? "-", result.url]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["FIELD", "VALUE"],
|
|
[
|
|
["mode", result.mode ?? (result.reportLoad ? "async" : "direct")],
|
|
["traceId", summary.traceId ?? trace.traceId ?? "-"],
|
|
["sessionId", summary.sessionId ?? session.sessionId ?? "-"],
|
|
["conversationId", summary.conversationId ?? session.conversationId ?? "-"],
|
|
["agentStatus", summary.agentStatus ?? trace.finalAgentStatus ?? "-"],
|
|
["traceStatus", summary.traceStatus ?? trace.finalTraceStatus ?? "-"],
|
|
["messageCount", summary.messageCount ?? session.messageCount ?? "-"],
|
|
["reportSource", reportSource],
|
|
["reportPath", reportPath],
|
|
["reportSha256", summary.reportSha256 ?? artifacts.reportSha256 ?? "-"],
|
|
["timeout", `${commandTimeout.seconds ?? "-"}s timedOut=${commandTimeout.timedOut ?? "-"}`],
|
|
["job", job.jobId ?? "-"],
|
|
],
|
|
),
|
|
...blockedRows,
|
|
"",
|
|
"NEXT",
|
|
` report: ${reportPath}`,
|
|
` rerun: ${result.command ?? `web-probe run --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
export function renderWebObserveStartResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const observer = record(result.observer);
|
|
const heartbeat = record(observer.heartbeat);
|
|
const manifest = record(observer.manifest);
|
|
const commandResult = record(result.result);
|
|
const credential = record(result.credential);
|
|
const id = result.id ?? observer.id ?? observer.jobId ?? manifest.jobId ?? "-";
|
|
const status = result.status ?? manifest.status ?? heartbeat.status ?? "-";
|
|
const stateDir = observer.stateDir ?? manifest.stateDir ?? "-";
|
|
const blockedRows = result.ok === true ? [] : [
|
|
"",
|
|
"Blocked detail:",
|
|
webObserveTable(
|
|
["EXIT", "TIMEOUT", "STDOUT", "STDERR"],
|
|
[[
|
|
webObserveText(commandResult.exitCode),
|
|
webObserveText(commandResult.timedOut),
|
|
webObserveShort(webObserveText(commandResult.stdoutTail ?? commandResult.stdout), 160),
|
|
webObserveShort(webObserveText(commandResult.stderrTail ?? commandResult.stderr), 300),
|
|
]],
|
|
),
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "STATUS", "PID", "SAMPLE", "UPDATED"],
|
|
[[
|
|
id,
|
|
result.node,
|
|
result.lane,
|
|
status,
|
|
observer.pid ?? heartbeat.pid,
|
|
heartbeat.sampleSeq,
|
|
heartbeat.updatedAt,
|
|
]],
|
|
),
|
|
"",
|
|
...renderWebObserveWrapperContract(result),
|
|
webObserveTable(
|
|
["URL", "TARGET_PATH", "STATE_DIR"],
|
|
[[result.url, result.targetPath, webObserveShort(webObserveText(stateDir), 96)]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["CREDENTIAL", "SOURCE", "VALUES"],
|
|
[[
|
|
credential.username ?? "-",
|
|
credential.sourceRef ?? "-",
|
|
credential.valuesRedacted === true ? "redacted" : "unknown",
|
|
]],
|
|
),
|
|
...blockedRows,
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts web-probe observe status ${id}`,
|
|
` analyze: bun scripts/cli.ts web-probe observe analyze ${id}`,
|
|
` command: bun scripts/cli.ts web-probe observe command ${id} --type mark --label checkpoint`,
|
|
` stop: bun scripts/cli.ts web-probe observe stop ${id}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
export function renderWebObserveStatusResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const observer = record(result.observer);
|
|
const commandResult = record(result.result);
|
|
const manifest = record(observer.manifest);
|
|
const heartbeat = record(observer.heartbeat);
|
|
const tails = record(observer.tails);
|
|
const samples = webObserveArray(tails.samples);
|
|
const network = webObserveArray(tails.network);
|
|
const control = webObserveArray(tails.control);
|
|
const artifacts = webObserveArray(tails.artifacts);
|
|
const lastSample = record(samples[samples.length - 1]);
|
|
const failedNetwork = network
|
|
.map((item) => record(item))
|
|
.filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500);
|
|
const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-";
|
|
const resultSection = result.ok === true ? [] : [
|
|
"",
|
|
webObserveTable(
|
|
["RESULT", "VALUE"],
|
|
[
|
|
["exitCode", commandResult.exitCode ?? "-"],
|
|
["timedOut", commandResult.timedOut ?? "-"],
|
|
["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"],
|
|
["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"],
|
|
],
|
|
),
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"],
|
|
[[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"],
|
|
[[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["TAIL", "COUNT", "DETAIL"],
|
|
[
|
|
["control", control.length, record(control[control.length - 1]).type ?? "-"],
|
|
["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"],
|
|
["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`],
|
|
["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"],
|
|
],
|
|
),
|
|
...resultSection,
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts web-probe observe status ${id}`,
|
|
` analyze: bun scripts/cli.ts web-probe observe analyze ${id}`,
|
|
` command: bun scripts/cli.ts web-probe observe command ${id} --type mark --label checkpoint`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
export function renderWebObserveCommandResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const observerCommand = record(result.observerCommand);
|
|
const observer = record(result.observer);
|
|
const id = result.id ?? "-";
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"],
|
|
[[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["INPUT", "VALUE"],
|
|
[
|
|
["label", observerCommand.label ?? "-"],
|
|
["path", observerCommand.path ?? "-"],
|
|
["provider", observerCommand.provider ?? "-"],
|
|
["sessionId", observerCommand.sessionId ?? "-"],
|
|
["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`],
|
|
],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts web-probe observe status ${id}`,
|
|
` analyze: bun scripts/cli.ts web-probe observe analyze ${id}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
export function renderWebObserveAnalyzeResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const analysis = record(result.analysis);
|
|
const failure = record(result.failure);
|
|
const counts = record(analysis.counts);
|
|
const sampleMetrics = record(analysis.sampleMetrics);
|
|
const runtimeAlerts = record(analysis.runtimeAlerts);
|
|
const pagePerformance = record(analysis.pagePerformance);
|
|
const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item));
|
|
const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item));
|
|
const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item));
|
|
const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item));
|
|
const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item));
|
|
const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item));
|
|
const sseStreams = webObserveArray(analysis.pagePerformanceSseStreams).slice(0, 8).map((item) => record(item));
|
|
const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item));
|
|
const commandState = record(analysis.commandState);
|
|
const id = result.id ?? "-";
|
|
const blockerRows = String(result.status ?? "") === "blocked"
|
|
? [
|
|
["reason", failure.reason],
|
|
["exitCode", failure.exitCode],
|
|
["timedOut", failure.timedOut],
|
|
["parsedJson", failure.parsedJson],
|
|
["stdoutBytes", failure.stdoutBytes],
|
|
["stderrBytes", failure.stderrBytes],
|
|
["stdoutTail", String(failure.stdoutTail ?? "").replace(/\s+/g, " ").slice(0, 220)],
|
|
["stderrTail", String(failure.stderrTail ?? "").replace(/\s+/g, " ").slice(0, 220)],
|
|
["workspace", failure.workspace],
|
|
]
|
|
: [];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"],
|
|
[[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]],
|
|
),
|
|
"",
|
|
...renderWebObserveWrapperContract(result),
|
|
...(blockerRows.length > 0
|
|
? ["", webObserveTable(["ANALYZE_BLOCKER", "VALUE"], blockerRows)]
|
|
: []),
|
|
"",
|
|
webObserveTable(
|
|
["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"],
|
|
[[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]],
|
|
),
|
|
"",
|
|
jsonlReadIssues.length === 0
|
|
? "JSONL_READ_ISSUES\n-"
|
|
: webObserveTable(
|
|
["FILE", "CODE", "MESSAGE"],
|
|
jsonlReadIssues.map((item) => [item.file, item.code, item.message]),
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["TURN_TIMING", "VALUE"],
|
|
[
|
|
["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount],
|
|
["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount],
|
|
["elapsedForwardJump", sampleMetrics.turnTimingTotalElapsedForwardJumpCount],
|
|
["elapsedForwardJumpMaxSec", sampleMetrics.turnTimingTotalElapsedForwardJumpMaxSeconds],
|
|
["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount],
|
|
["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds],
|
|
],
|
|
),
|
|
"",
|
|
rounds.length === 0
|
|
? "ROUNDS\n-"
|
|
: webObserveTable(
|
|
["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "CARD_DIAG", "TERM", "NONMONO", "ELAPSED_JUMP", "JUMP"],
|
|
rounds.map((item) => [
|
|
item.promptIndex,
|
|
item.promptCommandId,
|
|
item.sampleCount,
|
|
item.maxTotalElapsedSeconds,
|
|
item.lastTotalElapsedSeconds,
|
|
item.maxRecentUpdateSeconds,
|
|
item.lastRecentUpdateSeconds,
|
|
item.diagnosticSamples,
|
|
item.terminalSamples,
|
|
item.turnTimingNonMonotonicCount,
|
|
item.turnTimingTotalElapsedForwardJumpCount,
|
|
item.turnTimingRecentUpdateJumpCount,
|
|
]),
|
|
),
|
|
"",
|
|
turnColumns.length === 0
|
|
? "TURN_COLUMNS\n-"
|
|
: webObserveTable(
|
|
["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"],
|
|
turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]),
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["ALERT", "COUNT"],
|
|
[
|
|
["httpError", runtimeAlerts.httpErrorCount],
|
|
["requestFailed", runtimeAlerts.requestFailedCount],
|
|
["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount],
|
|
["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount],
|
|
["consoleAlerts", runtimeAlerts.consoleAlertCount],
|
|
["executionErrors", runtimeAlerts.executionErrorCount],
|
|
],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["TOOL_COMMANDS", "COUNT"],
|
|
[
|
|
["pending", commandState.pendingCount],
|
|
["processing", commandState.processingCount],
|
|
["abandoned", commandState.abandonedCount],
|
|
["failed", commandState.failedCount],
|
|
],
|
|
),
|
|
"",
|
|
domDiagnosticGroups.length === 0
|
|
? "DOM_DIAGNOSTIC_GROUPS\n-"
|
|
: webObserveTable(
|
|
["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"],
|
|
domDiagnosticGroups.map((item) => [
|
|
item.diagnosticCode,
|
|
item.count,
|
|
item.firstSeq,
|
|
item.lastSeq,
|
|
item.firstAt,
|
|
item.lastAt,
|
|
webObserveArray(item.promptIndexes).join(",") || "-",
|
|
item.traceId,
|
|
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140),
|
|
]),
|
|
),
|
|
"",
|
|
domDiagnostics.length === 0
|
|
? "DOM_DIAGNOSTICS\n-"
|
|
: webObserveTable(
|
|
["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"],
|
|
domDiagnostics.map((item) => [
|
|
item.seq,
|
|
item.ts,
|
|
item.promptIndex,
|
|
item.diagnosticCode,
|
|
item.traceId,
|
|
item.httpStatus,
|
|
item.idleSeconds,
|
|
item.waitingFor,
|
|
item.lastEventLabel,
|
|
String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180),
|
|
]),
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["PERF", "VALUE"],
|
|
[
|
|
["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount],
|
|
["longLivedStreamPaths", pagePerformance.longLivedStreamPathCount],
|
|
["streamOpenOverBudgetSamples", pagePerformance.longLivedStreamOpenOverBudgetSampleCount ?? pagePerformance.longLivedStreamOpenOverFiveSecondSampleCount],
|
|
["slowPathCount", pagePerformance.slowPathCount],
|
|
["slowSampleCount", pagePerformance.slowSampleCount],
|
|
["worstP95Ms", pagePerformance.worstP95Ms],
|
|
],
|
|
),
|
|
"",
|
|
slowApis.length === 0
|
|
? "SLOW_API\n-"
|
|
: webObserveTable(
|
|
["SLOW_API", "P50", "P75", "P95", "OVER_BUDGET", "COUNT"],
|
|
slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overBudgetCount ?? item.overFiveSecondCount, item.sampleCount]),
|
|
),
|
|
"",
|
|
sseStreams.length === 0
|
|
? "SSE_STREAMS\n-"
|
|
: webObserveTable(
|
|
["SSE_STREAM", "OPEN_P95", "OPEN_MAX", "OPEN_OVER_BUDGET", "LIFE>5S", "COUNT"],
|
|
sseStreams.map((item) => [item.path, item.streamOpenP95Ms, item.streamOpenMaxMs, item.streamOpenOverBudgetCount ?? item.streamOpenOverFiveSecondCount, item.streamLifetimeOverFiveSecondCount, item.sampleCount]),
|
|
),
|
|
"",
|
|
findings.length === 0
|
|
? "FINDINGS\n-"
|
|
: webObserveTable(
|
|
["FINDING", "SEVERITY", "COUNT", "TIMING", "SUMMARY"],
|
|
findings.map((item) => [item.id ?? item.kind ?? item.code, item.severity ?? item.level, item.count ?? item.sampleCount, webObserveFindingTiming(item), item.summary ?? item.message]),
|
|
),
|
|
"",
|
|
"REPORTS",
|
|
` json: ${analysis.reportJsonPath ?? "-"}`,
|
|
` md: ${analysis.reportMdPath ?? "-"}`,
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts web-probe observe status ${id}`,
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
export function renderWebProbeScriptResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(result.summary);
|
|
const issueEvidence = record(result.issueEvidence);
|
|
const probe = record(result.probe);
|
|
const script = record(probe.script);
|
|
const scriptResult = record(script.result);
|
|
const reportLoad = record(result.reportLoad);
|
|
const commandResult = record(result.result);
|
|
const recoveredArtifacts = record(result.recoveredArtifacts);
|
|
const recoveredArtifactSummary = record(recoveredArtifacts.artifacts);
|
|
const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item));
|
|
const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item));
|
|
const warnings = webObserveArray(result.warnings).slice(0, 6);
|
|
const hints = webObserveArray(result.hints).slice(0, 6).map((item) => String(item ?? "").trim()).filter(Boolean);
|
|
const preferredCommands = record(result.preferredCommands);
|
|
const resultRows = webProbeScriptRecordRows(scriptResult, 12);
|
|
const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12);
|
|
const summaryRows = [
|
|
["ok", result.ok],
|
|
["status", result.status],
|
|
["degradedReason", result.degradedReason ?? "-"],
|
|
["failureKind", result.failureKind ?? "-"],
|
|
["scriptSource", result.scriptSource ?? "-"],
|
|
["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"],
|
|
["reportPath", reportLoad.path ?? probe.reportPath ?? "-"],
|
|
["reportSha256", probe.reportSha256 ?? "-"],
|
|
["scriptSha256", probe.scriptSha256 ?? "-"],
|
|
["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"],
|
|
["exitCode", commandResult.exitCode ?? "-"],
|
|
];
|
|
const renderedText = [
|
|
webObserveTable(
|
|
["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"],
|
|
[[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]],
|
|
),
|
|
"",
|
|
webObserveTable(["FIELD", "VALUE"], summaryRows),
|
|
"",
|
|
resultRows.length === 0
|
|
? "SCRIPT_RESULT\n-"
|
|
: webObserveTable(["KEY", "VALUE"], resultRows),
|
|
"",
|
|
evidenceRows.length === 0
|
|
? "ISSUE_EVIDENCE\n-"
|
|
: webObserveTable(["KEY", "VALUE"], evidenceRows),
|
|
"",
|
|
steps.length === 0
|
|
? "STEPS\n-"
|
|
: webObserveTable(
|
|
["STEP", "OK", "DETAIL"],
|
|
steps.map((item) => [
|
|
item.name ?? item.step ?? item.label ?? "-",
|
|
item.ok ?? item.status ?? "-",
|
|
webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item),
|
|
]),
|
|
),
|
|
"",
|
|
recoveredItems.length === 0
|
|
? "ARTIFACTS\n-"
|
|
: webObserveTable(
|
|
["KIND", "BYTES", "PATH"],
|
|
recoveredItems.map((item) => [item.kind, item.byteCount, item.path]),
|
|
),
|
|
"",
|
|
warnings.length === 0
|
|
? "WARNINGS\n-"
|
|
: [
|
|
"WARNINGS",
|
|
webObserveTable(
|
|
["CODE", "SEVERITY", "MESSAGE"],
|
|
warnings.map((item) => {
|
|
const warning = record(item);
|
|
return [warning.code ?? "warning", warning.severity ?? warning.level ?? "warning", warning.message ?? warning.summary ?? webProbeScriptPreview(warning)];
|
|
}),
|
|
),
|
|
].join("\n"),
|
|
"",
|
|
hints.length === 0
|
|
? "HINTS\n-"
|
|
: ["HINTS", ...hints.map((hint) => `- ${hint}`)].join("\n"),
|
|
"",
|
|
"NEXT",
|
|
` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`,
|
|
` rerun: ${result.command ?? `web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`,
|
|
...Object.entries(preferredCommands).map(([name, command]) => ` ${name}: ${String(command)}`),
|
|
].join("\n");
|
|
return withWebObserveRendered(result, renderedText);
|
|
}
|
|
|
|
export function webProbeScriptRecordRows(value: Record<string, unknown>, limit: number): unknown[][] {
|
|
return Object.entries(value)
|
|
.slice(0, limit)
|
|
.map(([key, nested]) => [key, webProbeScriptPreview(nested)]);
|
|
}
|
|
|
|
export function webProbeScriptPreview(value: unknown): string {
|
|
if (value === null || value === undefined) return "-";
|
|
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim();
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
if (Array.isArray(value)) {
|
|
const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", ");
|
|
return `[${value.length}] ${preview}`;
|
|
}
|
|
if (typeof value === "object") {
|
|
return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240);
|
|
}
|
|
return String(value);
|
|
}
|
|
|
|
export function withWebObserveShortcuts(value: Record<string, unknown> | null, id: string | null): Record<string, unknown> | null {
|
|
if (value === null || id === null) return value;
|
|
return {
|
|
...value,
|
|
id,
|
|
next: webObserveNextCommands(id),
|
|
};
|
|
}
|