Codex
289 lines
11 KiB
TypeScript
289 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { spawnSync } from "node:child_process";
|
|
import { test } from "bun:test";
|
|
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
|
|
const alertThresholds = {
|
|
sameOriginApiSlowMs: 60000,
|
|
partialApiSlowMs: 60000,
|
|
longLivedStreamOpenSlowMs: 60000,
|
|
visibleLoadingSlowMs: 60000,
|
|
turnTimingSampleSlackSeconds: 60,
|
|
turnElapsedSevereTimeoutSeconds: 3600,
|
|
domEvaluateTimeoutRedCount: 99,
|
|
domEvaluateTimeoutRedWindowMs: 60000,
|
|
screenshotTimeoutRedCount: 99,
|
|
pageErrorRedCount: 99,
|
|
longTaskRedMs: 60000,
|
|
longAnimationFrameRedMs: 60000,
|
|
eventLoopGapRedMs: 60000,
|
|
browserProcessSampleIntervalMs: 1000,
|
|
requestRateBucketMs: 10000,
|
|
requestRateTotalRedPerMinute: 999999,
|
|
requestRatePageRedPerMinute: 999999,
|
|
requestRateApiPathRedPerMinute: 999999,
|
|
browserTotalRssRedMb: 999999,
|
|
browserProcessRssRedMb: 999999,
|
|
browserRssGrowthRedMb: 999999,
|
|
browserRssGrowthWindowMs: 60000,
|
|
playwrightResponsivenessRedMs: 60000,
|
|
playwrightResponsivenessTimeoutRedCount: 99,
|
|
cdpMetricsTimeoutRedCount: 99,
|
|
uncommandedStateChangeCommandWindowMs: 1000,
|
|
scrollJumpCommandWindowMs: 1000,
|
|
scrollJumpFromY: 999999,
|
|
scrollJumpToY: 999999,
|
|
sessionRailFallbackRatio: 0.5,
|
|
};
|
|
|
|
const browserFreezePolicy = {
|
|
enabled: true,
|
|
blockerWindowMs: 60000,
|
|
memory: {
|
|
totalRssBlockerMb: 999999,
|
|
processRssBlockerMb: 999999,
|
|
growthBlockerMb: 999999,
|
|
},
|
|
responsiveness: {
|
|
latencyBlockerMs: 60000,
|
|
eventBlockerCount: 99,
|
|
},
|
|
cdp: {
|
|
metricsTimeoutBlockerCount: 99,
|
|
},
|
|
kill: {
|
|
enabled: false,
|
|
gracefulSignal: "SIGTERM",
|
|
forceSignal: "SIGKILL",
|
|
graceMs: 1000,
|
|
pollIntervalMs: 100,
|
|
exitCode: 124,
|
|
},
|
|
};
|
|
|
|
async function runObserveAnalyzer(files: Record<string, string>): Promise<Record<string, any>> {
|
|
const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-"));
|
|
const analyzerPath = join(stateDir, "analyze.mjs");
|
|
await writeFile(analyzerPath, nodeWebObserveAnalyzerSource(), { mode: 0o700 });
|
|
for (const [name, content] of Object.entries(files)) {
|
|
await writeFile(join(stateDir, name), content.endsWith("\n") ? content : content + "\n");
|
|
}
|
|
const result = spawnSync("bun", [analyzerPath, stateDir], {
|
|
cwd: join(import.meta.dir, "../../.."),
|
|
env: {
|
|
...process.env,
|
|
UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES: "0",
|
|
UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON: JSON.stringify(alertThresholds),
|
|
UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON: JSON.stringify(browserFreezePolicy),
|
|
},
|
|
encoding: "utf8",
|
|
});
|
|
assert.equal(result.status, 0, result.stderr || result.stdout);
|
|
return JSON.parse(await readFile(join(stateDir, "analysis", "report.json"), "utf8"));
|
|
}
|
|
|
|
test("observe analyzer classifies stale completed session rail when a newer turn is running", async () => {
|
|
const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-"));
|
|
const analyzerPath = join(stateDir, "analyze.mjs");
|
|
const samplesPath = join(stateDir, "samples.jsonl");
|
|
await writeFile(analyzerPath, nodeWebObserveAnalyzerSource(), { mode: 0o700 });
|
|
await writeFile(samplesPath, [
|
|
JSON.stringify({
|
|
seq: 1,
|
|
ts: "2026-07-01T15:21:24.767Z",
|
|
path: "/workbench/sessions/ses_triage",
|
|
url: "https://hwlab.example.test/workbench/sessions/ses_triage",
|
|
pageRole: "control",
|
|
pageId: "control-test",
|
|
routeSessionId: "ses_triage",
|
|
activeSessionId: "ses_triage",
|
|
sessionRail: {
|
|
items: [{
|
|
index: 0,
|
|
active: true,
|
|
status: "completed",
|
|
dataStatus: "completed",
|
|
running: false,
|
|
dataRunning: "false",
|
|
sessionId: "ses_triage",
|
|
sessionIdPrefix: "ses_triage",
|
|
}],
|
|
},
|
|
turns: [
|
|
{
|
|
role: "agent",
|
|
status: "completed",
|
|
traceId: "trc_previous_completed",
|
|
messageId: "msg_previous_completed_agent",
|
|
finalResponsePresent: true,
|
|
finalResponseTextBytes: 12,
|
|
},
|
|
{
|
|
role: "agent",
|
|
status: "running",
|
|
traceId: "trc_running_new_turn",
|
|
messageId: "msg_running_new_turn_agent",
|
|
finalResponsePresent: false,
|
|
finalResponseTextBytes: 0,
|
|
},
|
|
],
|
|
}),
|
|
JSON.stringify({
|
|
seq: 2,
|
|
ts: "2026-07-01T15:22:49.918Z",
|
|
path: "/workbench/sessions/ses_triage",
|
|
url: "https://hwlab.example.test/workbench/sessions/ses_triage",
|
|
pageRole: "control",
|
|
pageId: "control-test",
|
|
routeSessionId: "ses_triage",
|
|
activeSessionId: "ses_triage",
|
|
sessionRail: {
|
|
items: [{
|
|
index: 0,
|
|
active: true,
|
|
status: "canceled",
|
|
dataStatus: "canceled",
|
|
running: false,
|
|
dataRunning: "false",
|
|
sessionId: "ses_triage",
|
|
sessionIdPrefix: "ses_triage",
|
|
}],
|
|
},
|
|
turns: [{
|
|
role: "agent",
|
|
status: "canceled",
|
|
traceId: "trc_canceled_terminal",
|
|
messageId: "msg_canceled_terminal_agent",
|
|
finalResponsePresent: true,
|
|
finalResponseTextBytes: 17,
|
|
}],
|
|
}),
|
|
].join("\n") + "\n");
|
|
|
|
const result = spawnSync("bun", [analyzerPath, stateDir], {
|
|
cwd: join(import.meta.dir, "../../.."),
|
|
env: {
|
|
...process.env,
|
|
UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES: "0",
|
|
UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON: JSON.stringify(alertThresholds),
|
|
UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON: JSON.stringify(browserFreezePolicy),
|
|
},
|
|
encoding: "utf8",
|
|
});
|
|
assert.equal(result.status, 0, result.stderr || result.stdout);
|
|
|
|
const report = JSON.parse(await readFile(join(stateDir, "analysis", "report.json"), "utf8"));
|
|
const finding = report.findings.find((item: Record<string, unknown>) => item.id === "workbench-turn-state-triad-inconsistent");
|
|
assert.equal(finding?.rootCause, "workbench_session_rail_status_stale_after_new_running_turn");
|
|
assert.equal(finding?.dominantMismatchKind, "rail-card-status-mismatch");
|
|
assert.match(String(finding?.summary), /previous completed terminal status/u);
|
|
assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.invalidRowCount, 1);
|
|
assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.cardFinalResponseMismatchCount, 0);
|
|
}, 20_000);
|
|
|
|
test("observe analyzer flags automatic recovery legacy fanout authority", async () => {
|
|
const samples = [
|
|
JSON.stringify({
|
|
seq: 1,
|
|
ts: "2026-07-01T15:21:25.000Z",
|
|
path: "/workbench/sessions/ses_auto",
|
|
url: "https://hwlab.example.test/workbench/sessions/ses_auto",
|
|
pageRole: "control",
|
|
pageId: "control-test",
|
|
routeSessionId: "ses_auto",
|
|
activeSessionId: "ses_auto",
|
|
turns: []
|
|
})
|
|
].join("\n") + "\n";
|
|
const network = [
|
|
JSON.stringify({
|
|
ts: "2026-07-01T15:21:26.000Z",
|
|
type: "response",
|
|
method: "POST",
|
|
status: 202,
|
|
url: "https://hwlab.example.test/v1/web-performance",
|
|
webPerformancePayload: {
|
|
parseStatus: "parsed",
|
|
schemaVersion: "hwlab-web-performance-v2",
|
|
events: [{
|
|
kind: "workbench_ui_event",
|
|
eventType: "runtime_diagnostic",
|
|
module: "workbench-stream-transport",
|
|
diagnosticCode: "workbench_recovery_action",
|
|
recoveryAction: "hydrate-trace-events",
|
|
rootCause: "sse-gap",
|
|
eventCount: 0
|
|
}]
|
|
}
|
|
}),
|
|
JSON.stringify({ ts: "2026-07-01T15:21:27.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/turns/trc_auto", bodySummary: { pathKind: "workbench-turn", terminalEvidenceCount: 1, traceIds: ["trc_auto"] } }),
|
|
JSON.stringify({ ts: "2026-07-01T15:21:28.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/sessions/ses_auto/messages", bodySummary: { pathKind: "workbench-session-messages", terminalEvidenceCount: 1, sessionIds: ["ses_auto"] } }),
|
|
JSON.stringify({ ts: "2026-07-01T15:21:29.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/traces/trc_auto/events", bodySummary: { pathKind: "workbench-trace-events", traceEventLikeCount: 3, traceIds: ["trc_auto"] } }),
|
|
JSON.stringify({ ts: "2026-07-01T15:21:30.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/sync?sessionId=ses_auto&traceId=trc_auto&since=7", bodySummary: { pathKind: "workbench-sync-replay", eventCount: 1, cursorOutboxSeq: 12, realtimeAuthority: "workbench-realtime-authority-v2", terminalSeal: true } })
|
|
].join("\n") + "\n";
|
|
|
|
const report = await runObserveAnalyzer({ "samples.jsonl": samples, "network.jsonl": network });
|
|
const finding = report.findings.find((item: Record<string, unknown>) => item.id === "workbench-automatic-recovery-fanout-authority");
|
|
|
|
assert.equal(finding?.rootCause, "workbench_automatic_recovery_legacy_fanout");
|
|
assert.equal(finding?.recoveryAuthority.summary.forbiddenAutomaticRecoveryCount, 1);
|
|
assert.equal(finding?.recoveryAuthority.summary.legacyFanoutWindowCount, 1);
|
|
assert.equal(finding?.recoveryAuthority.summary.syncReplayEvidenceCount, 1);
|
|
assert.equal(report.runtimeAlerts.webPerformanceRuntimeDiagnostics[0].recoveryAction, "hydrate-trace-events");
|
|
}, 20_000);
|
|
|
|
test("observe analyzer allows explicit detail trace reads when sync authority evidence exists", async () => {
|
|
const samples = JSON.stringify({
|
|
seq: 1,
|
|
ts: "2026-07-01T15:31:25.000Z",
|
|
path: "/workbench/sessions/ses_detail",
|
|
url: "https://hwlab.example.test/workbench/sessions/ses_detail",
|
|
pageRole: "control",
|
|
pageId: "control-test",
|
|
routeSessionId: "ses_detail",
|
|
activeSessionId: "ses_detail",
|
|
turns: []
|
|
}) + "\n";
|
|
const network = [
|
|
JSON.stringify({ ts: "2026-07-01T15:31:26.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/traces/trc_detail/events", bodySummary: { pathKind: "workbench-trace-events", traceEventLikeCount: 2, traceIds: ["trc_detail"], authority: "trace-detail-only", detailProjection: true } }),
|
|
JSON.stringify({
|
|
ts: "2026-07-01T15:31:27.000Z",
|
|
type: "response",
|
|
method: "POST",
|
|
status: 202,
|
|
url: "https://hwlab.example.test/v1/web-performance",
|
|
webPerformancePayload: {
|
|
parseStatus: "parsed",
|
|
schemaVersion: "hwlab-web-performance-v2",
|
|
events: [{
|
|
kind: "workbench_ui_event",
|
|
eventType: "runtime_diagnostic",
|
|
module: "workbench-sync-replay",
|
|
diagnosticCode: "workbench_sync_replay_applied",
|
|
rootCause: "sse-gap",
|
|
eventCount: 1,
|
|
sinceOutboxSeq: 7,
|
|
syncCursorOutboxSeq: 12,
|
|
realtimeAuthority: "workbench-realtime-authority-v2",
|
|
entityFamilyCount: 1,
|
|
entityFamilies: "turn",
|
|
maxEntityVersion: 3,
|
|
projectionRevision: "rev_detail",
|
|
terminalSeal: true,
|
|
detailProjection: false
|
|
}]
|
|
}
|
|
})
|
|
].join("\n") + "\n";
|
|
|
|
const report = await runObserveAnalyzer({ "samples.jsonl": samples, "network.jsonl": network });
|
|
const finding = report.findings.find((item: Record<string, unknown>) => item.id === "workbench-automatic-recovery-fanout-authority");
|
|
|
|
assert.equal(finding, undefined);
|
|
assert.equal(report.runtimeAlerts.webPerformanceRuntimeDiagnostics[0].syncCursorOutboxSeq, 12);
|
|
}, 20_000);
|