Codex
972 lines
53 KiB
TypeScript
972 lines
53 KiB
TypeScript
// SPEC: PJ2026-0104010803 Workbench realtime visibility.
|
|
// Responsibility: Long-running, typed realtime fanout validation inside the persistent Web observer.
|
|
|
|
export function nodeWebObserveRunnerRealtimeSource(): string {
|
|
return String.raw`
|
|
function parseRealtimeFanoutProfiles(raw) {
|
|
if (!raw) return {};
|
|
try {
|
|
const parsed = JSON.parse(raw);
|
|
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
async function validateRealtimeFanout(command) {
|
|
const profileId = String(command.profile || "").trim();
|
|
const profile = realtimeFanoutProfiles[profileId];
|
|
if (!profile || typeof profile !== "object") throw new Error("validateRealtimeFanout profile is not declared by the owning YAML: " + profileId);
|
|
const provider = String(command.provider || "").trim();
|
|
const firstPrompt = String(command.text || "").trim();
|
|
const secondPrompt = String(command.secondText || "").trim();
|
|
if (!provider) throw new Error("validateRealtimeFanout requires provider");
|
|
if (!firstPrompt || !secondPrompt) throw new Error("validateRealtimeFanout requires first and second prompt");
|
|
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
|
|
const reportDir = path.join(stateDir, "artifacts", "realtime-fanout", safeId(command.id));
|
|
await mkdir(reportDir, { recursive: true, mode: 0o700 });
|
|
const network = [];
|
|
const requestListener = (request) => {
|
|
let url;
|
|
try { url = new URL(request.url()); } catch { return; }
|
|
if (url.origin !== new URL(baseUrl).origin) return;
|
|
network.push({
|
|
at: Date.now(),
|
|
method: request.method(),
|
|
path: url.pathname,
|
|
sessionId: url.searchParams.get("sessionId"),
|
|
traceId: url.searchParams.get("traceId"),
|
|
afterSeq: url.searchParams.get("afterSeq"),
|
|
lastEventId: request.headers()["last-event-id"] || null,
|
|
valuesRedacted: true,
|
|
});
|
|
if (network.length > 1000) network.splice(0, network.length - 1000);
|
|
};
|
|
page.on("request", requestListener);
|
|
let subscriberA = null;
|
|
let subscriberB = null;
|
|
let subscriberB2 = null;
|
|
let debug = null;
|
|
let createdSessionId = null;
|
|
const report = {
|
|
ok: false,
|
|
contract: "web-probe-realtime-fanout-v1",
|
|
commandId: command.id,
|
|
profile: profileId,
|
|
provider,
|
|
startedAt: new Date().toISOString(),
|
|
promptEvidence: {
|
|
first: { hash: sha256Text(firstPrompt), bytes: Buffer.byteLength(firstPrompt) },
|
|
second: { hash: sha256Text(secondPrompt), bytes: Buffer.byteLength(secondPrompt) },
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
try {
|
|
const providerSelection = await selectProvider(provider);
|
|
const session = await createSessionFromUi();
|
|
const sessionId = session.sessionId;
|
|
createdSessionId = sessionId;
|
|
report.sessionId = sessionId;
|
|
report.providerSelection = sanitize(providerSelection);
|
|
const storageState = await context.storageState();
|
|
subscriberA = await realtimeNewSubscriber(storageState, sessionId, "subscriber-a", effective);
|
|
subscriberB = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b", effective);
|
|
debug = await realtimeNewDebugContext(storageState);
|
|
const connectedA = await realtimeWaitProductConnected(subscriberA, effective.barrierTimeoutMs);
|
|
const connectedB = await realtimeWaitProductConnected(subscriberB, effective.barrierTimeoutMs);
|
|
realtimeAssertLiveConnected(connectedA, sessionId, "subscriber-a", effective);
|
|
realtimeAssertLiveConnected(connectedB, sessionId, "subscriber-b", effective);
|
|
await appendJsonl(files.control, eventRecord("realtime-fanout-product-ready", {
|
|
profile: profileId,
|
|
sessionId,
|
|
subscriberCount: 2,
|
|
liveOnly: true,
|
|
replay: false,
|
|
valuesRedacted: true,
|
|
}));
|
|
|
|
const first = await realtimeRunTurn({
|
|
command,
|
|
turn: 1,
|
|
prompt: firstPrompt,
|
|
sessionId,
|
|
subscribers: [subscriberA, subscriberB],
|
|
debug,
|
|
profile: effective,
|
|
});
|
|
const bBeforeDisconnect = await realtimeProductSnapshot(subscriberB);
|
|
realtimeAssertStableProductTransport(bBeforeDisconnect, "subscriber-b-before-disconnect");
|
|
if (bBeforeDisconnect.records.some((item) => item.traceId === first.traceId && item.terminal === true)) {
|
|
throw new Error("first turn reached terminal before subscriber B could be disconnected; use a prompt with an observable live interval");
|
|
}
|
|
const bBeforeShot = await realtimeCaptureSubscriberScreenshot(subscriberB.page, reportDir, "subscriber-b-before-disconnect.png", {
|
|
title: "Subscriber B before disconnect",
|
|
sessionId,
|
|
traceId: first.traceId,
|
|
eventCount: bBeforeDisconnect.records.length,
|
|
terminalSeen: bBeforeDisconnect.records.some((item) => item.terminal === true),
|
|
valuesRedacted: true,
|
|
});
|
|
await subscriberB.context.close();
|
|
subscriberB = null;
|
|
const firstTerminal = await realtimeWaitTraceEvent(subscriberA, first.traceId, true, effective.terminalTimeoutMs);
|
|
await realtimeWaitUiTerminal(first.traceId, effective.terminalTimeoutMs);
|
|
const firstStable = await realtimeWaitTurnStable(debug, first.debugKey, first.traceId, [subscriberA], effective);
|
|
const firstDebug = firstStable.debug;
|
|
realtimeEnrichTurnExecutionIds(first, firstDebug);
|
|
const firstProductA = firstStable.products[0];
|
|
realtimeAssertStableProductTransport(firstProductA, "subscriber-a-first-terminal");
|
|
realtimeValidateTurnEvidence({
|
|
turn: 1,
|
|
traceId: first.traceId,
|
|
sessionId,
|
|
runId: first.runId,
|
|
commandId: first.commandId,
|
|
debug: firstDebug,
|
|
terminalSnapshots: [firstProductA],
|
|
preTerminalSnapshots: [bBeforeDisconnect],
|
|
profile: effective,
|
|
});
|
|
await realtimeCloseDebugStreams(debug, first.debugKey);
|
|
|
|
subscriberB2 = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b-reconnected", effective);
|
|
const bReconnected = await realtimeWaitProductConnected(subscriberB2, effective.barrierTimeoutMs);
|
|
realtimeAssertLiveConnected(bReconnected, sessionId, "subscriber-b-reconnected", effective);
|
|
await sleep(effective.reconnectQuietMs);
|
|
const reconnectBeforeSecond = await realtimeProductSnapshot(subscriberB2);
|
|
realtimeAssertStableProductTransport(reconnectBeforeSecond, "subscriber-b-reconnected-before-second-turn");
|
|
if (reconnectBeforeSecond.records.some((item) => item.traceId === first.traceId)) {
|
|
throw new Error("reconnected subscriber replayed first-turn events");
|
|
}
|
|
|
|
const second = await realtimeRunTurn({
|
|
command,
|
|
turn: 2,
|
|
prompt: secondPrompt,
|
|
sessionId,
|
|
subscribers: [subscriberA, subscriberB2],
|
|
debug,
|
|
profile: effective,
|
|
});
|
|
if (second.traceId === first.traceId) throw new Error("second turn reused the first business traceId");
|
|
const secondTerminalA = await realtimeWaitTraceEvent(subscriberA, second.traceId, true, effective.terminalTimeoutMs);
|
|
const secondTerminalB = await realtimeWaitTraceEvent(subscriberB2, second.traceId, true, effective.terminalTimeoutMs);
|
|
await realtimeWaitUiTerminal(second.traceId, effective.terminalTimeoutMs);
|
|
const secondStable = await realtimeWaitTurnStable(debug, second.debugKey, second.traceId, [subscriberA, subscriberB2], effective);
|
|
const secondDebug = secondStable.debug;
|
|
realtimeEnrichTurnExecutionIds(second, secondDebug);
|
|
const [secondProductA, secondProductB] = secondStable.products;
|
|
realtimeAssertStableProductTransport(secondProductA, "subscriber-a-second-terminal");
|
|
realtimeAssertStableProductTransport(secondProductB, "subscriber-b-reconnected-second-terminal");
|
|
if (secondProductB.records.some((item) => item.traceId === first.traceId)) {
|
|
throw new Error("reconnected subscriber received first-turn events after the second submit");
|
|
}
|
|
realtimeValidateTurnEvidence({
|
|
turn: 2,
|
|
traceId: second.traceId,
|
|
sessionId,
|
|
runId: second.runId,
|
|
commandId: second.commandId,
|
|
debug: secondDebug,
|
|
terminalSnapshots: [secondProductA, secondProductB],
|
|
preTerminalSnapshots: [],
|
|
profile: effective,
|
|
});
|
|
await realtimeCloseDebugStreams(debug, second.debugKey);
|
|
|
|
const productA = await realtimeProductSnapshot(subscriberA);
|
|
const productB2 = await realtimeProductSnapshot(subscriberB2);
|
|
realtimeAssertStableProductTransport(productA, "subscriber-a-final");
|
|
realtimeAssertStableProductTransport(productB2, "subscriber-b-reconnected-final");
|
|
const mainStreams = network.filter((item) => item.path === effective.productEventsPath && item.sessionId === sessionId);
|
|
const forbidden = network.filter((item) => realtimeForbiddenRequest(item, effective.forbiddenRequestPaths));
|
|
if (mainStreams.length !== 1) throw new Error("main Workbench session SSE connection count is " + mainStreams.length + ", expected 1");
|
|
if (mainStreams[0].traceId !== null) throw new Error("main Workbench switched to trace-scoped SSE");
|
|
if (mainStreams[0].afterSeq !== null || mainStreams[0].lastEventId !== null) throw new Error("main Workbench requested replay cursor");
|
|
if (forbidden.length > 0) throw new Error("forbidden recovery or polling requests observed: " + forbidden.map((item) => item.path).join(","));
|
|
|
|
const summaryScreenshot = await realtimeCaptureSubscriberScreenshot(debug.page, reportDir, "realtime-fanout-summary.png", {
|
|
title: "Pure Kafka live fanout validated",
|
|
profile: profileId,
|
|
sessionId,
|
|
traceIds: [first.traceId, second.traceId],
|
|
productSubscriberCount: 2,
|
|
debugStreams: effective.debugStreams,
|
|
forbiddenRequestCount: forbidden.length,
|
|
replayedEventCount: 0,
|
|
valuesRedacted: true,
|
|
});
|
|
const subscriberAScreenshot = await realtimeCaptureSubscriberScreenshot(subscriberA.page, reportDir, "subscriber-a-final.png", {
|
|
title: "Subscriber A continuous live connection",
|
|
sessionId,
|
|
traceIds: [first.traceId, second.traceId],
|
|
eventCount: productA.records.length,
|
|
terminalCount: productA.records.filter((item) => item.terminal === true).length,
|
|
valuesRedacted: true,
|
|
});
|
|
const subscriberB2Screenshot = await realtimeCaptureSubscriberScreenshot(subscriberB2.page, reportDir, "subscriber-b-reconnected-final.png", {
|
|
title: "Subscriber B reconnected live-only",
|
|
sessionId,
|
|
traceIds: [second.traceId],
|
|
eventCount: productB2.records.length,
|
|
firstTraceReplayCount: productB2.records.filter((item) => item.traceId === first.traceId).length,
|
|
valuesRedacted: true,
|
|
});
|
|
const turns = [
|
|
realtimeTurnSummary(first, firstDebug, productA, firstTerminal),
|
|
realtimeTurnSummary(second, secondDebug, productB2, secondTerminalB),
|
|
];
|
|
Object.assign(report, {
|
|
ok: true,
|
|
status: "passed",
|
|
completedAt: new Date().toISOString(),
|
|
turns,
|
|
productSse: {
|
|
mainSessionConnectionCount: mainStreams.length,
|
|
sessionScoped: true,
|
|
afterSeqCount: 0,
|
|
lastEventIdCount: 0,
|
|
subscriberAEventCount: productA.records.length,
|
|
subscriberBReconnectEventCount: productB2.records.length,
|
|
subscriberBFirstTraceReplayCount: 0,
|
|
valuesRedacted: true,
|
|
},
|
|
liveContract: {
|
|
expectedKafka: effective.expectedKafka,
|
|
subscriberA: realtimeConnectedSummary(connectedA),
|
|
subscriberBBeforeDisconnect: realtimeConnectedSummary(connectedB),
|
|
subscriberBAfterReconnect: realtimeConnectedSummary(bReconnected),
|
|
valuesRedacted: true,
|
|
},
|
|
forbiddenRequests: { count: forbidden.length, rows: forbidden, valuesRedacted: true },
|
|
screenshots: [bBeforeShot, summaryScreenshot, subscriberAScreenshot, subscriberB2Screenshot],
|
|
warmRunnerReused: first.runnerId && second.runnerId ? first.runnerId === second.runnerId : null,
|
|
valuesRedacted: true,
|
|
});
|
|
const artifacts = await realtimeWriteEvidenceArtifacts(reportDir, network, [firstDebug, secondDebug]);
|
|
report.artifacts = artifacts;
|
|
const reportArtifact = await realtimeWriteReport(reportDir, report);
|
|
await appendJsonl(files.control, eventRecord("realtime-fanout-validated", {
|
|
profile: profileId,
|
|
sessionId,
|
|
traceIds: turns.map((item) => item.traceId),
|
|
runIds: turns.map((item) => item.runId),
|
|
commandIds: turns.map((item) => item.commandId),
|
|
reportPath: reportArtifact.path,
|
|
reportSha256: reportArtifact.sha256,
|
|
valuesRedacted: true,
|
|
}));
|
|
const observer = await syncObserverPageToControlSession("validateRealtimeFanout", sessionId);
|
|
return {
|
|
ok: true,
|
|
status: "passed",
|
|
profile: profileId,
|
|
sessionId,
|
|
traceIds: turns.map((item) => item.traceId),
|
|
runIds: turns.map((item) => item.runId),
|
|
commandIds: turns.map((item) => item.commandId),
|
|
warmRunnerReused: report.warmRunnerReused,
|
|
forbiddenRequestCount: 0,
|
|
replayedEventCount: 0,
|
|
reportPath: reportArtifact.path,
|
|
reportSha256: reportArtifact.sha256,
|
|
screenshots: report.screenshots,
|
|
observer,
|
|
valuesRedacted: true,
|
|
};
|
|
} catch (error) {
|
|
Object.assign(report, {
|
|
ok: false,
|
|
status: "failed",
|
|
completedAt: new Date().toISOString(),
|
|
failure: errorSummary(error),
|
|
valuesRedacted: true,
|
|
});
|
|
const partialDebug = debug ? await realtimeAllDebugSnapshots(debug).catch(() => []) : [];
|
|
const partialProducts = await realtimePartialProductEvidence([subscriberA, subscriberB, subscriberB2]);
|
|
const partialArtifacts = await realtimeWriteEvidenceArtifacts(reportDir, network, partialDebug).catch(() => null);
|
|
report.partialEvidence = { products: partialProducts, debug: partialDebug.map(realtimeDebugSnapshotSummary), valuesRedacted: true };
|
|
if (partialArtifacts) report.artifacts = partialArtifacts;
|
|
const reportArtifact = await realtimeWriteReport(reportDir, report).catch(() => null);
|
|
if (createdSessionId) {
|
|
await syncObserverPageToControlSession("validateRealtimeFanout-failed", createdSessionId)
|
|
.catch((syncError) => appendJsonl(files.errors, eventRecord("realtime-fanout-observer-sync-error", { commandId: command.id, error: errorSummary(syncError), valuesRedacted: true })));
|
|
}
|
|
const wrapped = error instanceof Error ? error : new Error(String(error));
|
|
wrapped.details = {
|
|
...(wrapped.details || {}),
|
|
profile: profileId,
|
|
reportPath: reportArtifact?.path || null,
|
|
reportSha256: reportArtifact?.sha256 || null,
|
|
valuesRedacted: true,
|
|
};
|
|
throw wrapped;
|
|
} finally {
|
|
page.off("request", requestListener);
|
|
if (debug) await realtimeCloseDebugStreams(debug, null).catch(() => {});
|
|
if (debug) await debug.context.close().catch(() => {});
|
|
if (subscriberA) await subscriberA.context.close().catch(() => {});
|
|
if (subscriberB) await subscriberB.context.close().catch(() => {});
|
|
if (subscriberB2) await subscriberB2.context.close().catch(() => {});
|
|
}
|
|
}
|
|
|
|
function realtimeFanoutEffectiveProfile(profile, durationMs) {
|
|
const debugStreams = Array.isArray(profile.debugStreams) ? profile.debugStreams.map(String) : [];
|
|
if (Number(profile.subscriberCount) !== 2) throw new Error("realtime fanout profile subscriberCount must be 2");
|
|
if (debugStreams.length !== 3 || new Set(debugStreams).size !== 3 || !["stdio", "agentrun", "hwlab"].every((item) => debugStreams.includes(item))) {
|
|
throw new Error("realtime fanout profile debugStreams must be stdio, agentrun, and hwlab");
|
|
}
|
|
if (profile.fromBeginning !== false) throw new Error("realtime fanout profile fromBeginning must be false");
|
|
const expectedKafka = profile.expectedKafka && typeof profile.expectedKafka === "object" ? profile.expectedKafka : null;
|
|
if (!expectedKafka?.topics || !expectedKafka?.groups || !expectedKafka?.capabilities) throw new Error("realtime fanout profile lacks YAML-derived Kafka expectations");
|
|
if (expectedKafka.capabilities.directPublish !== true || expectedKafka.capabilities.liveKafkaSse !== true) {
|
|
throw new Error("realtime fanout profile requires YAML-enabled directPublish and liveKafkaSse capabilities");
|
|
}
|
|
const terminalTimeoutMs = Number.isFinite(Number(durationMs))
|
|
? Math.min(Number(profile.terminalTimeoutMs), Math.max(1000, Number(durationMs)))
|
|
: Number(profile.terminalTimeoutMs);
|
|
return {
|
|
...profile,
|
|
debugStreams,
|
|
barrierTimeoutMs: boundedInteger(profile.barrierTimeoutMs, 45000, 1000, 120000),
|
|
terminalTimeoutMs: boundedInteger(terminalTimeoutMs, 480000, 1000, 600000),
|
|
terminalQuietMs: boundedInteger(profile.terminalQuietMs, 2000, 250, 10000),
|
|
reconnectQuietMs: boundedInteger(profile.reconnectQuietMs, 3000, 250, 30000),
|
|
forbiddenRequestPaths: Array.isArray(profile.forbiddenRequestPaths) ? profile.forbiddenRequestPaths.map(String) : [],
|
|
requiredEventTypes: profile.requiredEventTypes && typeof profile.requiredEventTypes === "object" ? profile.requiredEventTypes : {},
|
|
expectedKafka: sanitize(expectedKafka),
|
|
};
|
|
}
|
|
|
|
async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
|
const subscriberContext = await browser.newContext({ storageState, viewport });
|
|
const subscriberPage = await subscriberContext.newPage();
|
|
await subscriberPage.goto(new URL("/health/live", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: profile.barrierTimeoutMs });
|
|
const key = label + "-" + sessionId;
|
|
await subscriberPage.evaluate(({ key: streamKey, sessionId: scopedSessionId, profile: inputProfile }) => {
|
|
const root = window.__unideskRealtimeFanoutProducts ??= {};
|
|
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null };
|
|
root[streamKey] = state;
|
|
const firstText = (...values) => {
|
|
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
|
return null;
|
|
};
|
|
const fingerprint = (text) => {
|
|
const bytes = new TextEncoder().encode(String(text));
|
|
let hash = 2166136261;
|
|
for (const byte of bytes) hash = Math.imul((hash ^ byte) >>> 0, 16777619) >>> 0;
|
|
return "fnv1a32:" + hash.toString(16).padStart(8, "0") + ":" + bytes.length;
|
|
};
|
|
const source = new EventSource(inputProfile.productEventsPath + "?sessionId=" + encodeURIComponent(scopedSessionId), { withCredentials: true });
|
|
state.source = source;
|
|
source.onopen = () => { state.openCount += 1; };
|
|
const record = (eventName, raw) => {
|
|
let value;
|
|
try { value = JSON.parse(raw); } catch { return; }
|
|
const nested = value?.event && typeof value.event === "object" ? value.event : {};
|
|
const context = value?.context && typeof value.context === "object" ? value.context : {};
|
|
const eventType = firstText(nested.eventType, nested.type, value.eventType, value.type);
|
|
state.records.push({
|
|
eventName,
|
|
at: Date.now(),
|
|
schema: firstText(value.schema),
|
|
eventId: firstText(value.eventId, nested.id),
|
|
sourceEventId: firstText(value.sourceEventId, nested.sourceEventId),
|
|
eventType,
|
|
sourceSeq: Number.isFinite(Number(value.sourceSeq)) ? Number(value.sourceSeq) : Number.isFinite(Number(nested.sourceSeq)) ? Number(nested.sourceSeq) : null,
|
|
traceId: firstText(value.traceId, nested.traceId, context.traceId),
|
|
hwlabSessionId: firstText(value.hwlabSessionId, value.sessionId, nested.hwlabSessionId, nested.sessionId, context.sessionId),
|
|
runId: firstText(value.runId, nested.runId, context.runId),
|
|
commandId: firstText(value.commandId, nested.commandId, context.commandId),
|
|
terminal: value.terminal === true || nested.terminal === true || eventType === "terminal" || eventType === "terminal_status",
|
|
status: firstText(nested.status, value.status),
|
|
envelopeFingerprint: fingerprint(raw),
|
|
valuesRedacted: true,
|
|
});
|
|
if (state.records.length > 800) state.records.splice(0, state.records.length - 800);
|
|
};
|
|
source.addEventListener(inputProfile.productReadyEvent, (event) => {
|
|
state.connectedEventCount += 1;
|
|
try { state.connected = JSON.parse(event.data); } catch { state.connected = { parseError: true }; }
|
|
});
|
|
source.addEventListener(inputProfile.businessEvent, (event) => record(inputProfile.businessEvent, event.data));
|
|
source.addEventListener("workbench.error", (event) => state.errors.push({ at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
|
|
source.onerror = () => state.errors.push({ at: Date.now(), event: "transport.error" });
|
|
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent } });
|
|
return { label, key, context: subscriberContext, page: subscriberPage };
|
|
}
|
|
|
|
async function realtimeNewDebugContext(storageState) {
|
|
const debugContext = await browser.newContext({ storageState, viewport });
|
|
const debugPage = await debugContext.newPage();
|
|
await debugPage.goto(new URL("/health/live", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: 30000 });
|
|
return { context: debugContext, page: debugPage };
|
|
}
|
|
|
|
async function realtimeRunTurn(input) {
|
|
const turn = input.turn;
|
|
const prompt = input.prompt;
|
|
await page.locator("#command-input").fill(prompt);
|
|
await page.locator("#command-send").waitFor({ state: "visible", timeout: input.profile.barrierTimeoutMs });
|
|
await page.waitForFunction(() => {
|
|
const button = document.querySelector("#command-send");
|
|
return Boolean(button && !button.disabled && button.getAttribute("data-action") === "turn");
|
|
}, null, { timeout: input.profile.barrierTimeoutMs });
|
|
let capturedResolve;
|
|
let releaseResolve;
|
|
let releasedToServer = false;
|
|
const capturedPromise = new Promise((resolve) => { capturedResolve = resolve; });
|
|
const releasePromise = new Promise((resolve) => { releaseResolve = resolve; });
|
|
const handler = async (route) => {
|
|
const request = route.request();
|
|
let body = {};
|
|
try { body = request.postDataJSON(); } catch {}
|
|
capturedResolve({ traceId: String(body?.traceId || ""), sessionId: String(body?.sessionId || ""), provider: String(body?.providerProfile || "") });
|
|
const releaseAction = await releasePromise;
|
|
if (releaseAction === "continue") await route.continue();
|
|
else await route.abort("aborted");
|
|
};
|
|
await page.route("**/v1/agent/chat", handler);
|
|
try {
|
|
const admissionPromise = page.waitForResponse((response) => {
|
|
let url;
|
|
try { url = new URL(response.url()); } catch { return false; }
|
|
return response.request().method() === "POST" && url.pathname === "/v1/agent/chat";
|
|
}, { timeout: Math.min(input.profile.terminalTimeoutMs, input.profile.barrierTimeoutMs + 30000) })
|
|
.then((response) => ({ response, error: null }))
|
|
.catch((error) => ({ response: null, error }));
|
|
await page.locator("#command-send").click();
|
|
const captured = await withHardTimeout(capturedPromise, input.profile.barrierTimeoutMs, "realtime turn request capture timed out");
|
|
if (!/^trc_[A-Za-z0-9_-]+$/u.test(captured.traceId)) throw new Error("realtime turn did not pre-generate a traceId");
|
|
if (captured.sessionId !== input.sessionId) throw new Error("realtime turn sessionId mismatch");
|
|
if (captured.provider !== String(input.command.provider || "")) throw new Error("realtime turn provider mismatch");
|
|
const debugKey = "turn-" + turn + "-" + captured.traceId;
|
|
await realtimeOpenDebugStreams(input.debug, debugKey, captured.traceId, input.profile);
|
|
const debugConnected = await realtimeWaitDebugConnected(input.debug, debugKey, input.profile);
|
|
for (const stream of input.profile.debugStreams) {
|
|
const connected = debugConnected[stream];
|
|
if (connected?.consumerReady !== true) throw new Error(stream + " debug consumer is not ready");
|
|
if (!String(connected?.groupId || "").includes("-debug-sse-")) throw new Error(stream + " debug group is not isolated");
|
|
if (connected?.topic !== input.profile.expectedKafka.topics[stream]) throw new Error(stream + " debug topic does not match the owning YAML");
|
|
}
|
|
for (const subscriber of input.subscribers) {
|
|
const connected = await realtimeWaitProductConnected(subscriber, input.profile.barrierTimeoutMs);
|
|
realtimeAssertLiveConnected(connected, input.sessionId, subscriber.label, input.profile);
|
|
}
|
|
await appendJsonl(files.control, eventRecord("realtime-fanout-ready-barrier", {
|
|
turn,
|
|
traceId: captured.traceId,
|
|
productSubscriberCount: input.subscribers.length,
|
|
debugStreams: input.profile.debugStreams,
|
|
topics: Object.fromEntries(input.profile.debugStreams.map((stream) => [stream, debugConnected[stream]?.topic || null])),
|
|
groups: Object.fromEntries(input.profile.debugStreams.map((stream) => [stream, debugConnected[stream]?.groupId || null])),
|
|
valuesRedacted: true,
|
|
}));
|
|
releasedToServer = true;
|
|
releaseResolve("continue");
|
|
const admissionOutcome = await admissionPromise;
|
|
if (!admissionOutcome.response) throw new Error("realtime turn admission response failed: " + String(admissionOutcome.error?.message || admissionOutcome.error || "unknown"));
|
|
const admission = admissionOutcome.response;
|
|
if (admission.status() < 200 || admission.status() >= 300) throw new Error("realtime turn admission returned HTTP " + admission.status());
|
|
let admissionBody = {};
|
|
try { admissionBody = await admission.json(); } catch {}
|
|
const canonicalTraceId = realtimeFirstText(admissionBody?.traceId, admissionBody?.result?.traceId, captured.traceId);
|
|
if (canonicalTraceId !== captured.traceId) throw new Error("canonical trace changed after ready barrier");
|
|
const nonTerminal = [];
|
|
for (const subscriber of input.subscribers) {
|
|
nonTerminal.push(await realtimeWaitTraceEvent(subscriber, canonicalTraceId, false, input.profile.terminalTimeoutMs));
|
|
}
|
|
const ids = await realtimeWaitTurnIds(input.debug, debugKey, canonicalTraceId, input.profile.terminalTimeoutMs);
|
|
await appendJsonl(files.control, eventRecord("realtime-fanout-pre-terminal", {
|
|
turn,
|
|
traceId: canonicalTraceId,
|
|
runId: ids.runId,
|
|
commandId: ids.commandId,
|
|
subscriberCount: nonTerminal.length,
|
|
valuesRedacted: true,
|
|
}));
|
|
return { turn, traceId: canonicalTraceId, sessionId: input.sessionId, runId: ids.runId, commandId: ids.commandId, runnerId: ids.runnerId, attemptId: ids.attemptId, debugKey, admissionStatus: admission.status(), valuesRedacted: true };
|
|
} finally {
|
|
if (!releasedToServer) releaseResolve("abort");
|
|
await page.unroute("**/v1/agent/chat", handler).catch(() => {});
|
|
}
|
|
}
|
|
|
|
async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
|
await debug.page.evaluate(({ key: streamKey, traceId: scopedTraceId, profile: inputProfile }) => {
|
|
const root = window.__unideskRealtimeFanoutDebug ??= {};
|
|
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {} };
|
|
root[streamKey] = state;
|
|
const firstText = (...values) => {
|
|
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
|
return null;
|
|
};
|
|
const finite = (...values) => {
|
|
for (const value of values) {
|
|
const number = Number(value);
|
|
if (Number.isFinite(number)) return number;
|
|
}
|
|
return null;
|
|
};
|
|
const fingerprint = (text) => {
|
|
const bytes = new TextEncoder().encode(String(text));
|
|
let hash = 2166136261;
|
|
for (const byte of bytes) hash = Math.imul((hash ^ byte) >>> 0, 16777619) >>> 0;
|
|
return "fnv1a32:" + hash.toString(16).padStart(8, "0") + ":" + bytes.length;
|
|
};
|
|
for (const stream of inputProfile.debugStreams) {
|
|
state.records[stream] = [];
|
|
state.errors[stream] = [];
|
|
state.connectedEventCount[stream] = 0;
|
|
state.openCount[stream] = 0;
|
|
const params = new URLSearchParams({ stream, traceId: scopedTraceId });
|
|
if (inputProfile.fromBeginning === true) params.set("fromBeginning", "true");
|
|
const source = new EventSource(inputProfile.debugEventsPath + "?" + params.toString(), { withCredentials: true });
|
|
state.sources[stream] = source;
|
|
source.onopen = () => { state.openCount[stream] += 1; };
|
|
source.addEventListener(inputProfile.debugReadyEvent, (event) => {
|
|
state.connectedEventCount[stream] += 1;
|
|
try { state.connected[stream] = JSON.parse(event.data); } catch { state.connected[stream] = { parseError: true }; }
|
|
});
|
|
source.addEventListener("hwlab.kafka.event", (event) => {
|
|
let outer;
|
|
try { outer = JSON.parse(event.data); } catch { return; }
|
|
const value = outer?.value && typeof outer.value === "object" ? outer.value : {};
|
|
const nested = value?.event && typeof value.event === "object" ? value.event : {};
|
|
const payload = nested?.payload && typeof nested.payload === "object" ? nested.payload : {};
|
|
const context = value?.context && typeof value.context === "object" ? value.context : {};
|
|
const run = value?.run && typeof value.run === "object" ? value.run : {};
|
|
const command = value?.command && typeof value.command === "object" ? value.command : {};
|
|
const eventType = stream === "agentrun"
|
|
? firstText(nested.type, value.eventType)
|
|
: stream === "hwlab"
|
|
? firstText(nested.eventType, nested.type, value.eventType)
|
|
: firstText(value.eventType, value.stdio?.method);
|
|
state.records[stream].push({
|
|
stream,
|
|
at: Date.now(),
|
|
topic: firstText(outer.topic),
|
|
partition: finite(outer.partition),
|
|
offset: firstText(outer.offset),
|
|
schema: firstText(value.schema),
|
|
eventId: firstText(value.eventId, nested.id),
|
|
sourceEventId: firstText(value.sourceEventId, nested.sourceEventId),
|
|
eventType,
|
|
sourceSeq: finite(value.sourceSeq, nested.seq, nested.sourceSeq),
|
|
frameSeq: finite(value.frameSeq, value.stdio?.frameSeq),
|
|
traceId: firstText(value.traceId, context.traceId, payload.traceId, nested.traceId),
|
|
hwlabSessionId: firstText(value.hwlabSessionId, value.sessionId, run.hwlabSessionId, run.sessionId, context.sessionId, payload.hwlabSessionId, payload.sessionId, nested.hwlabSessionId, nested.sessionId),
|
|
runId: firstText(value.runId, run.runId, context.runId, nested.runId, payload.runId),
|
|
commandId: firstText(value.commandId, command.commandId, context.commandId, payload.commandId, nested.commandId),
|
|
runnerId: firstText(value.runnerId, run.runnerId, command.runnerId, context.runnerId, payload.runnerId, nested.runnerId),
|
|
attemptId: firstText(value.attemptId, run.attemptId, command.attemptId, context.attemptId, payload.attemptId, nested.attemptId),
|
|
terminal: value.terminal === true || nested.terminal === true || eventType === "terminal" || eventType === "terminal_status",
|
|
direction: firstText(value.stdio?.direction),
|
|
method: firstText(value.stdio?.method),
|
|
envelopeFingerprint: stream === "hwlab" ? fingerprint(JSON.stringify(value)) : null,
|
|
valuesRedacted: true,
|
|
});
|
|
if (state.records[stream].length > 1000) state.records[stream].splice(0, state.records[stream].length - 1000);
|
|
});
|
|
source.addEventListener("hwlab.kafka.error", (event) => state.errors[stream].push({ at: Date.now(), dataBytes: String(event.data || "").length }));
|
|
source.onerror = () => state.errors[stream].push({ at: Date.now(), transport: true });
|
|
}
|
|
}, { key, traceId, profile: {
|
|
debugStreams: profile.debugStreams,
|
|
debugEventsPath: profile.debugEventsPath,
|
|
debugReadyEvent: profile.debugReadyEvent,
|
|
fromBeginning: profile.fromBeginning,
|
|
} });
|
|
}
|
|
|
|
async function realtimeWaitProductConnected(subscriber, timeoutMs) {
|
|
return realtimePoll(async () => (await realtimeProductSnapshot(subscriber)).connected, timeoutMs, "product connected");
|
|
}
|
|
|
|
async function realtimeWaitDebugConnected(debug, key, profile) {
|
|
return realtimePoll(async () => {
|
|
const snapshot = await realtimeDebugSnapshot(debug, key);
|
|
return profile.debugStreams.every((stream) => snapshot.connected[stream]?.consumerReady === true) ? snapshot.connected : null;
|
|
}, profile.barrierTimeoutMs, "debug consumers ready");
|
|
}
|
|
|
|
async function realtimeWaitTurnIds(debug, key, traceId, timeoutMs) {
|
|
return realtimePoll(async () => {
|
|
const snapshot = await realtimeDebugSnapshot(debug, key);
|
|
const rows = Object.values(snapshot.records).flat().filter((item) => item.traceId === traceId);
|
|
const runId = rows.map((item) => item.runId).find(Boolean) || null;
|
|
const commandId = rows.map((item) => item.commandId).find(Boolean) || null;
|
|
const runnerId = rows.map((item) => item.runnerId).find(Boolean) || null;
|
|
const attemptId = rows.map((item) => item.attemptId).find(Boolean) || null;
|
|
return runId && commandId ? { runId, commandId, runnerId, attemptId } : null;
|
|
}, timeoutMs, "realtime run and command ids");
|
|
}
|
|
|
|
async function realtimeWaitDebugTurnComplete(debug, key, traceId, profile) {
|
|
return realtimePoll(async () => {
|
|
const snapshot = await realtimeDebugSnapshot(debug, key);
|
|
if (!profile.debugStreams.every((stream) => (snapshot.records[stream] || []).some((item) => item.traceId === traceId))) return null;
|
|
const agentrunTypes = new Set((snapshot.records.agentrun || []).filter((item) => item.traceId === traceId).map((item) => item.eventType));
|
|
const hwlabTypes = new Set((snapshot.records.hwlab || []).filter((item) => item.traceId === traceId).map((item) => item.eventType));
|
|
if (!(profile.requiredEventTypes.agentrun || []).every((type) => agentrunTypes.has(type))) return null;
|
|
if (!(profile.requiredEventTypes.hwlab || []).every((type) => hwlabTypes.has(type))) return null;
|
|
return snapshot;
|
|
}, profile.terminalTimeoutMs, "realtime debug terminal event families");
|
|
}
|
|
|
|
async function realtimeWaitTurnStable(debug, key, traceId, subscribers, profile) {
|
|
await realtimeWaitDebugTurnComplete(debug, key, traceId, profile);
|
|
const deadline = Date.now() + profile.terminalTimeoutMs;
|
|
let previousSignature = null;
|
|
let stableSince = Date.now();
|
|
while (Date.now() < deadline) {
|
|
const debugSnapshot = await realtimeDebugSnapshot(debug, key);
|
|
const productSnapshots = await Promise.all(subscribers.map((subscriber) => realtimeProductSnapshot(subscriber)));
|
|
const signature = JSON.stringify({
|
|
debug: Object.fromEntries(profile.debugStreams.map((stream) => {
|
|
const rows = (debugSnapshot.records[stream] || []).filter((item) => item.traceId === traceId);
|
|
return [stream, { count: rows.length, lastOffset: rows.at(-1)?.offset || null, lastFingerprint: rows.at(-1)?.envelopeFingerprint || null }];
|
|
})),
|
|
products: productSnapshots.map((snapshot) => {
|
|
const rows = snapshot.records.filter((item) => item.traceId === traceId);
|
|
return { count: rows.length, lastSourceSeq: rows.at(-1)?.sourceSeq || null, lastFingerprint: rows.at(-1)?.envelopeFingerprint || null };
|
|
}),
|
|
});
|
|
if (signature !== previousSignature) {
|
|
previousSignature = signature;
|
|
stableSince = Date.now();
|
|
} else if (Date.now() - stableSince >= profile.terminalQuietMs) {
|
|
return { debug: debugSnapshot, products: productSnapshots };
|
|
}
|
|
await sleep(250);
|
|
}
|
|
throw new Error("realtime debug/product streams did not become quiet after terminal");
|
|
}
|
|
|
|
async function realtimeWaitTraceEvent(subscriber, traceId, terminal, timeoutMs) {
|
|
return realtimePoll(async () => {
|
|
const snapshot = await realtimeProductSnapshot(subscriber);
|
|
return snapshot.records.find((item) => item.traceId === traceId && item.terminal === terminal) || null;
|
|
}, timeoutMs, terminal ? "terminal product event" : "pre-terminal product event");
|
|
}
|
|
|
|
async function realtimeWaitUiTerminal(traceId, timeoutMs) {
|
|
await page.waitForFunction((expectedTraceId) => {
|
|
const card = document.querySelector(".message-card[data-role='agent'][data-trace-id='" + expectedTraceId + "']");
|
|
if (!card) return false;
|
|
const status = card.getAttribute("data-status");
|
|
const body = card.querySelector(".message-text")?.textContent?.trim() || "";
|
|
return status === "completed" && body.length > 0;
|
|
}, traceId, { timeout: timeoutMs });
|
|
}
|
|
|
|
async function realtimeProductSnapshot(subscriber) {
|
|
return subscriber.page.evaluate((key) => {
|
|
const state = window.__unideskRealtimeFanoutProducts?.[key];
|
|
if (!state) return { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [{ missing: true }] };
|
|
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
|
|
}, subscriber.key);
|
|
}
|
|
|
|
async function realtimeDebugSnapshot(debug, key) {
|
|
return debug.page.evaluate((streamKey) => {
|
|
const state = window.__unideskRealtimeFanoutDebug?.[streamKey];
|
|
if (!state) return { connected: {}, connectedEventCount: {}, openCount: {}, records: { stdio: [], agentrun: [], hwlab: [] }, errors: { stdio: [], agentrun: [], hwlab: [] } };
|
|
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
|
|
}, key);
|
|
}
|
|
|
|
async function realtimeAllDebugSnapshots(debug) {
|
|
return debug.page.evaluate(() => {
|
|
const root = window.__unideskRealtimeFanoutDebug || {};
|
|
return Object.entries(root).map(([key, state]) => ({
|
|
key,
|
|
connected: JSON.parse(JSON.stringify(state.connected || {})),
|
|
connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount || {})),
|
|
openCount: JSON.parse(JSON.stringify(state.openCount || {})),
|
|
records: JSON.parse(JSON.stringify(state.records || {})),
|
|
errors: JSON.parse(JSON.stringify(state.errors || {})),
|
|
}));
|
|
});
|
|
}
|
|
|
|
async function realtimeCloseDebugStreams(debug, key) {
|
|
if (!debug?.page || debug.page.isClosed()) return;
|
|
await debug.page.evaluate((streamKey) => {
|
|
const root = window.__unideskRealtimeFanoutDebug || {};
|
|
const keys = streamKey ? [streamKey] : Object.keys(root);
|
|
for (const current of keys) {
|
|
for (const source of Object.values(root[current]?.sources || {})) source?.close?.();
|
|
}
|
|
}, key);
|
|
}
|
|
|
|
function realtimeAssertLiveConnected(connected, sessionId, label, profile) {
|
|
if (connected?.realtimeSource !== profile.expectedKafka.topics.hwlab) throw new Error(label + " realtime source mismatch");
|
|
if (connected?.deliverySemantics !== "live-only" || connected?.liveOnly !== true) throw new Error(label + " is not live-only");
|
|
if (connected?.replay !== false || connected?.replaySupported !== false || connected?.lossPossible !== true) throw new Error(label + " replay/loss disclosure mismatch");
|
|
if (connected?.filters?.sessionId !== sessionId) throw new Error(label + " session filter mismatch");
|
|
const capabilities = connected?.capabilities || {};
|
|
for (const [name, expected] of Object.entries(profile.expectedKafka.capabilities)) {
|
|
if (capabilities[name] !== expected) throw new Error(label + " capability " + name + " differs from the owning YAML");
|
|
}
|
|
}
|
|
|
|
function realtimeConnectedSummary(connected) {
|
|
return {
|
|
realtimeSource: connected?.realtimeSource || null,
|
|
deliverySemantics: connected?.deliverySemantics || null,
|
|
liveOnly: connected?.liveOnly === true,
|
|
replay: connected?.replay === true,
|
|
replaySupported: connected?.replaySupported === true,
|
|
lossPossible: connected?.lossPossible === true,
|
|
filters: { sessionId: connected?.filters?.sessionId || null, traceId: connected?.filters?.traceId || null },
|
|
capabilities: sanitize(connected?.capabilities || {}),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function realtimeAssertStableProductTransport(snapshot, label) {
|
|
if (snapshot.openCount !== 1) throw new Error(label + " product SSE open count is " + snapshot.openCount + ", expected 1");
|
|
if (snapshot.connectedEventCount !== 1) throw new Error(label + " product connected event count is " + snapshot.connectedEventCount + ", expected 1");
|
|
if ((snapshot.errors || []).length !== 0) throw new Error(label + " product SSE observed a transport or business error");
|
|
}
|
|
|
|
function realtimeEnrichTurnExecutionIds(turn, debug) {
|
|
const rows = Object.values(debug.records || {}).flat().filter((item) => item.traceId === turn.traceId);
|
|
turn.runnerId ||= rows.map((item) => item.runnerId).find(Boolean) || null;
|
|
turn.attemptId ||= rows.map((item) => item.attemptId).find(Boolean) || null;
|
|
}
|
|
|
|
function realtimeValidateTurnEvidence(input) {
|
|
const profile = input.profile;
|
|
for (const stream of profile.debugStreams) {
|
|
if (input.debug.connected[stream]?.consumerReady !== true) throw new Error("turn " + input.turn + " " + stream + " consumer not ready");
|
|
if (input.debug.connected[stream]?.topic !== profile.expectedKafka.topics[stream]) throw new Error("turn " + input.turn + " " + stream + " topic mismatch");
|
|
if (input.debug.openCount[stream] !== 1 || input.debug.connectedEventCount[stream] !== 1) throw new Error("turn " + input.turn + " " + stream + " debug SSE reconnected");
|
|
const rows = (input.debug.records[stream] || []).filter((item) => item.traceId === input.traceId);
|
|
if (rows.length === 0) throw new Error("turn " + input.turn + " " + stream + " has no trace records");
|
|
if (!realtimeMonotonicOffsets(rows)) throw new Error("turn " + input.turn + " " + stream + " offsets are not monotonic");
|
|
if (rows.some((item) => item.hwlabSessionId !== input.sessionId)) throw new Error("turn " + input.turn + " " + stream + " session missing or mismatched");
|
|
if (rows.some((item) => item.runId !== input.runId)) throw new Error("turn " + input.turn + " " + stream + " runId missing or mismatched");
|
|
if (rows.some((item) => item.commandId && item.commandId !== input.commandId)) throw new Error("turn " + input.turn + " " + stream + " commandId mismatched");
|
|
const commandBoundRows = stream === "stdio"
|
|
? rows
|
|
: rows.filter((item) => (profile.requiredEventTypes[stream] || []).includes(item.eventType));
|
|
if (commandBoundRows.length === 0 || commandBoundRows.some((item) => item.commandId !== input.commandId)) {
|
|
throw new Error("turn " + input.turn + " " + stream + " command-bound event lacks the current commandId");
|
|
}
|
|
if ((input.debug.errors[stream] || []).length !== 0) throw new Error("turn " + input.turn + " " + stream + " debug SSE error");
|
|
}
|
|
const stdio = input.debug.records.stdio.filter((item) => item.traceId === input.traceId);
|
|
const frameSeqs = stdio.map((item) => item.frameSeq).filter(Number.isFinite);
|
|
if (frameSeqs.length === 0 || !realtimeStrictlyIncreasing(frameSeqs)) throw new Error("turn " + input.turn + " stdio frameSeq is not strictly increasing");
|
|
const agentrunTypes = new Set(input.debug.records.agentrun.filter((item) => item.traceId === input.traceId).map((item) => item.eventType));
|
|
const hwlabTypes = new Set(input.debug.records.hwlab.filter((item) => item.traceId === input.traceId).map((item) => item.eventType));
|
|
for (const type of profile.requiredEventTypes.agentrun || []) if (!agentrunTypes.has(type)) throw new Error("turn " + input.turn + " AgentRun lacks " + type);
|
|
for (const type of profile.requiredEventTypes.hwlab || []) if (!hwlabTypes.has(type)) throw new Error("turn " + input.turn + " HWLAB lacks " + type);
|
|
const identityRows = Object.values(input.debug.records).flat().filter((item) => item.traceId === input.traceId);
|
|
if (!input.runId || new Set(identityRows.map((item) => item.runId)).size !== 1) throw new Error("turn " + input.turn + " runId missing or mismatched across streams");
|
|
const commandIds = new Set(identityRows.map((item) => item.commandId).filter(Boolean));
|
|
if (!input.commandId || commandIds.size !== 1 || !commandIds.has(input.commandId)) throw new Error("turn " + input.turn + " commandId missing or mismatched across command-bound events");
|
|
const hwlabRows = input.debug.records.hwlab.filter((item) => item.traceId === input.traceId);
|
|
for (const snapshot of input.terminalSnapshots) {
|
|
const rows = snapshot.records.filter((item) => item.traceId === input.traceId);
|
|
if (!rows.some((item) => item.terminal !== true) || !rows.some((item) => item.terminal === true)) throw new Error("turn " + input.turn + " product subscriber lacks pre-terminal or terminal event");
|
|
realtimeAssertEnvelopePassthrough(hwlabRows, rows, true, "turn " + input.turn + " terminal subscriber");
|
|
}
|
|
for (const snapshot of input.preTerminalSnapshots) {
|
|
const rows = snapshot.records.filter((item) => item.traceId === input.traceId);
|
|
if (!rows.some((item) => item.terminal !== true)) throw new Error("turn " + input.turn + " disconnected subscriber lacks pre-terminal event");
|
|
realtimeAssertEnvelopePassthrough(hwlabRows, rows, false, "turn " + input.turn + " pre-terminal subscriber");
|
|
}
|
|
}
|
|
|
|
function realtimeAssertEnvelopePassthrough(kafkaRows, productRows, requireAllKafkaRows, label) {
|
|
const sameEnvelope = (left, right) => left.envelopeFingerprint && left.envelopeFingerprint === right.envelopeFingerprint
|
|
&& left.eventId === right.eventId
|
|
&& left.sourceEventId === right.sourceEventId
|
|
&& left.sourceSeq === right.sourceSeq
|
|
&& left.eventType === right.eventType
|
|
&& left.traceId === right.traceId
|
|
&& left.hwlabSessionId === right.hwlabSessionId
|
|
&& left.runId === right.runId
|
|
&& left.commandId === right.commandId
|
|
&& left.terminal === right.terminal;
|
|
for (const productRow of productRows) {
|
|
if (!kafkaRows.some((kafkaRow) => sameEnvelope(kafkaRow, productRow))) throw new Error(label + " received a product SSE envelope absent from hwlab.event.v1");
|
|
}
|
|
if (requireAllKafkaRows) {
|
|
if (productRows.length !== kafkaRows.length) throw new Error(label + " envelope count differs from hwlab.event.v1");
|
|
for (const kafkaRow of kafkaRows) {
|
|
if (!productRows.some((productRow) => sameEnvelope(kafkaRow, productRow))) throw new Error(label + " did not receive an unchanged hwlab.event.v1 envelope");
|
|
}
|
|
}
|
|
}
|
|
|
|
function realtimeMonotonicOffsets(rows) {
|
|
const partitions = new Map();
|
|
for (const row of rows) {
|
|
let offset;
|
|
try { offset = BigInt(String(row.offset)); } catch { return false; }
|
|
if (!Number.isFinite(row.partition)) return false;
|
|
const partition = row.partition;
|
|
const previous = partitions.get(partition);
|
|
if (previous !== undefined && offset <= previous) return false;
|
|
partitions.set(partition, offset);
|
|
}
|
|
return partitions.size > 0;
|
|
}
|
|
|
|
function realtimeStrictlyIncreasing(values) {
|
|
for (let index = 1; index < values.length; index += 1) if (!(values[index] > values[index - 1])) return false;
|
|
return true;
|
|
}
|
|
|
|
function realtimeForbiddenRequest(item, forbiddenPaths) {
|
|
const pathValue = String(item.path || "").toLowerCase();
|
|
return forbiddenPaths.some((entry) => pathValue.includes(String(entry).toLowerCase()));
|
|
}
|
|
|
|
function realtimeTurnSummary(turn, debug, product, terminal) {
|
|
const streams = {};
|
|
for (const [stream, rowsRaw] of Object.entries(debug.records)) {
|
|
const rows = rowsRaw.filter((item) => item.traceId === turn.traceId);
|
|
streams[stream] = {
|
|
topic: debug.connected[stream]?.topic || rows[0]?.topic || null,
|
|
groupId: debug.connected[stream]?.groupId || null,
|
|
count: rows.length,
|
|
partitions: [...new Set(rows.map((item) => item.partition).filter(Number.isFinite))],
|
|
firstOffset: rows[0]?.offset || null,
|
|
lastOffset: rows.at(-1)?.offset || null,
|
|
eventTypes: [...new Set(rows.map((item) => item.eventType).filter(Boolean))],
|
|
firstFrameSeq: rows.map((item) => item.frameSeq).find(Number.isFinite) || null,
|
|
lastFrameSeq: rows.map((item) => item.frameSeq).filter(Number.isFinite).at(-1) || null,
|
|
openCount: debug.openCount?.[stream] || 0,
|
|
connectedEventCount: debug.connectedEventCount?.[stream] || 0,
|
|
errorCount: (debug.errors?.[stream] || []).length,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
const productRows = product.records.filter((item) => item.traceId === turn.traceId);
|
|
return {
|
|
turn: turn.turn,
|
|
traceId: turn.traceId,
|
|
runId: turn.runId,
|
|
commandId: turn.commandId,
|
|
runnerId: turn.runnerId,
|
|
attemptId: turn.attemptId,
|
|
sessionId: turn.sessionId,
|
|
admissionStatus: turn.admissionStatus,
|
|
terminalStatus: terminal?.status || null,
|
|
streams,
|
|
product: { count: productRows.length, openCount: product.openCount, connectedEventCount: product.connectedEventCount, errorCount: product.errors.length, preTerminalSeen: productRows.some((item) => item.terminal !== true), terminalSeen: productRows.some((item) => item.terminal === true), valuesRedacted: true },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name, summary) {
|
|
await targetPage.evaluate((value) => {
|
|
document.documentElement.style.background = "#0b1020";
|
|
document.body.innerHTML = "";
|
|
const pre = document.createElement("pre");
|
|
pre.style.cssText = "color:#d7e3ff;background:#111a30;padding:32px;border-radius:16px;font:18px/1.55 ui-monospace,monospace;white-space:pre-wrap;margin:40px;";
|
|
pre.textContent = JSON.stringify(value, null, 2);
|
|
document.body.appendChild(pre);
|
|
}, summary);
|
|
const file = path.join(reportDir, name);
|
|
await targetPage.screenshot({ path: file, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
|
const meta = await fileMeta(file);
|
|
const artifact = { kind: "realtime-fanout-screenshot", path: file, name, byteCount: meta.byteCount, sha256: meta.sha256, commandId: activeCommandId, valuesRedacted: true };
|
|
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...artifact });
|
|
return artifact;
|
|
}
|
|
|
|
async function realtimePartialProductEvidence(subscribers) {
|
|
const out = [];
|
|
for (const subscriber of subscribers.filter(Boolean)) {
|
|
const snapshot = await realtimeProductSnapshot(subscriber).catch(() => null);
|
|
if (!snapshot) continue;
|
|
out.push({
|
|
label: subscriber.label,
|
|
openCount: snapshot.openCount,
|
|
connectedEventCount: snapshot.connectedEventCount,
|
|
errorCount: snapshot.errors.length,
|
|
eventCount: snapshot.records.length,
|
|
traceIds: [...new Set(snapshot.records.map((item) => item.traceId).filter(Boolean))].slice(0, 4),
|
|
terminalCount: snapshot.records.filter((item) => item.terminal === true).length,
|
|
connected: realtimeConnectedSummary(snapshot.connected),
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function realtimeDebugSnapshotSummary(snapshot) {
|
|
return {
|
|
key: snapshot.key || null,
|
|
streams: Object.fromEntries(Object.entries(snapshot.records || {}).map(([stream, rows]) => [stream, {
|
|
topic: snapshot.connected?.[stream]?.topic || null,
|
|
groupId: snapshot.connected?.[stream]?.groupId || null,
|
|
openCount: snapshot.openCount?.[stream] || 0,
|
|
connectedEventCount: snapshot.connectedEventCount?.[stream] || 0,
|
|
errorCount: (snapshot.errors?.[stream] || []).length,
|
|
eventCount: rows.length,
|
|
traceIds: [...new Set(rows.map((item) => item.traceId).filter(Boolean))].slice(0, 4),
|
|
valuesRedacted: true,
|
|
}])),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
async function realtimeWriteEvidenceArtifacts(reportDir, network, debugSnapshots) {
|
|
const requestsFile = path.join(reportDir, "request-ledger.jsonl");
|
|
const eventsFile = path.join(reportDir, "events.jsonl");
|
|
await writeFile(requestsFile, network.map((item) => JSON.stringify(item)).join("\n") + "\n", { mode: 0o600 });
|
|
const eventRows = [];
|
|
for (const snapshot of debugSnapshots) {
|
|
for (const [stream, rows] of Object.entries(snapshot.records || {})) {
|
|
for (const row of rows) eventRows.push({ ...row, stream, valuesRedacted: true });
|
|
}
|
|
}
|
|
await writeFile(eventsFile, eventRows.map((item) => JSON.stringify(item)).join("\n") + "\n", { mode: 0o600 });
|
|
const requestMeta = await fileMeta(requestsFile);
|
|
const eventMeta = await fileMeta(eventsFile);
|
|
const artifacts = {
|
|
requestLedger: { path: requestsFile, ...requestMeta, valuesRedacted: true },
|
|
events: { path: eventsFile, ...eventMeta, count: eventRows.length, valuesRedacted: true },
|
|
valuesRedacted: true,
|
|
};
|
|
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), kind: "realtime-fanout-request-ledger", commandId: activeCommandId, ...artifacts.requestLedger });
|
|
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), kind: "realtime-fanout-events", commandId: activeCommandId, ...artifacts.events });
|
|
return artifacts;
|
|
}
|
|
|
|
async function realtimeWriteReport(reportDir, report) {
|
|
const reportFile = path.join(reportDir, "report.json");
|
|
await writeFile(reportFile, JSON.stringify(sanitize(report), null, 2) + "\n", { mode: 0o600 });
|
|
const meta = await fileMeta(reportFile);
|
|
const artifact = { kind: "realtime-fanout-report", path: reportFile, byteCount: meta.byteCount, sha256: meta.sha256, commandId: activeCommandId, valuesRedacted: true };
|
|
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...artifact });
|
|
return artifact;
|
|
}
|
|
|
|
async function realtimePoll(read, timeoutMs, label) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
const value = await read();
|
|
if (value) return value;
|
|
await sleep(250);
|
|
}
|
|
throw new Error(label + " timed out after " + timeoutMs + "ms");
|
|
}
|
|
|
|
function realtimeFirstText(...values) {
|
|
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
|
return "";
|
|
}
|
|
`;
|
|
}
|