feat: add web sentinel session invariance checks

2026-06-26 12:00:13 +00:00
parent ad7043f3f4
commit dee36326b9
9 changed files with 622 additions and 25 deletions
@@ -97,7 +97,7 @@ const runnerErrors = errors.slice(-8).map((item) => {
 });
 const commandFailures = summarizeCommandFailures(control);
 const toolFindings = buildToolFindings({ manifest, heartbeat, commandState });
-const findings = [...toolFindings, ...buildProjectManagementFindings(projectManagement), ...buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures)];
+const findings = [...toolFindings, ...buildProjectManagementFindings(projectManagement), ...buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures, manifest)];
 if (jsonlReadIssues.length > 0) findings.unshift({ id: "jsonl-read-issues", severity: "red", summary: "observer analyzer hit JSONL read/parse issues", count: jsonlReadIssues.length, issues: jsonlReadIssues.slice(0, 20) });
 const recentWindow = buildRecentAnalysisWindow({ samples, control, network, consoleEvents, errors, manifest });
 const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
@@ -1264,18 +1264,140 @@ function maxNumber(values) {
  return numeric.length > 0 ? Math.max(...numeric) : 0;
 }

-function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = []) {
+function buildSessionInvariantFindings(control, manifest = {}) {
+  const findings = [];
+  for (const row of control || []) {
+    if (row?.type !== "assertSessionInvariant" || row?.phase !== "completed") continue;
+    const detail = objectValue(row.detail);
+    const messageOrder = objectValue(detail.messageOrder);
+    if (messageOrder.userClustered !== true) continue;
+    const afterRound = numberOrNull(detail.afterRound ?? row.input?.afterRound);
+    const consecutiveUserMessageCount = numberOrNull(messageOrder.consecutiveUserMessageCount) ?? 0;
+    const sentinelRange = stringOrNull(messageOrder.sentinelRange) ?? stringOrNull(detail.expectedSentinelRange);
+    const traceIds = arrayStrings(messageOrder.traceIds).slice(0, 12);
+    const findingId = stringOrNull(detail.findingId) ?? "workbench-message-order-user-clustered-after-navigation";
+    const severity = stringOrNull(detail.severity) ?? "amber";
+    findings.push({
+      id: findingId,
+      severity,
+      summary: "controlled refresh/switch-back afterRound=" + (afterRound ?? "-") + " left consecutive user message cards without interleaved assistant/code-agent terminal cards" + (sentinelRange ? " (" + sentinelRange + ")" : ""),
+      count: Math.max(1, consecutiveUserMessageCount),
+      blocking: detail.blocking === true ? true : false,
+      afterRound,
+      canarySessionId: stringOrNull(detail.canarySessionId),
+      routeSessionId: stringOrNull(detail.routeSessionId),
+      activeSessionId: stringOrNull(detail.activeSessionId),
+      consecutiveUserMessageCount,
+      sentinelRange,
+      sampleSeq: numberOrNull(detail.sampleSeq),
+      traceIds,
+      pageRole: stringOrNull(detail.pageRole) ?? "control",
+      pageId: stringOrNull(detail.pageId),
+      observerId: stringOrNull(manifest.jobId),
+      stateDir: stringOrNull(manifest.stateDir),
+      commandId: stringOrNull(row.commandId),
+      commandTs: stringOrNull(row.ts),
+      messageOrder: {
+        sequence: Array.isArray(messageOrder.sequence) ? messageOrder.sequence.slice(-20) : [],
+        clusters: Array.isArray(messageOrder.clusters) ? messageOrder.clusters.slice(0, 8) : [],
+        valuesRedacted: true,
+      },
+      valuesRedacted: true,
+    });
+  }
+  return findings;
+}
+
+function sessionInvariantNavigationWindows(control) {
+  const started = new Map();
+  const windows = [];
+  for (const row of control || []) {
+    if (row?.type !== "switchAwayAndBack") continue;
+    const commandId = stringOrNull(row.commandId) ?? String(row.seq ?? "");
+    if (row.phase === "started") {
+      started.set(commandId, row);
+      continue;
+    }
+    if (row.phase !== "completed") continue;
+    const detail = objectValue(row.detail);
+    const canarySessionId = stringOrNull(detail.canarySessionId);
+    const alternateSessionId = stringOrNull(detail.alternateSessionId);
+    const startRow = started.get(commandId);
+    const startMs = timestampMs(startRow?.ts ?? row.ts);
+    const endMs = timestampMs(row.ts);
+    if (!canarySessionId || !Number.isFinite(startMs) || !Number.isFinite(endMs)) continue;
+    windows.push({
+      commandId,
+      afterRound: numberOrNull(detail.afterRound ?? row.input?.afterRound),
+      startMs: Math.max(0, startMs - 1000),
+      endMs: endMs + 5000,
+      startAt: new Date(startMs).toISOString(),
+      endAt: new Date(endMs).toISOString(),
+      canarySessionId,
+      alternateSessionId,
+      routeOk: detail.routeOk === true,
+      activeOk: detail.activeOk === true,
+      valuesRedacted: true,
+    });
+  }
+  return windows;
+}
+
+function sessionChangeSamplesOutsideControlledNavigation(samples, key, windows) {
+  const canaryIds = new Set((windows || []).map((item) => item.canarySessionId).filter(Boolean));
+  if (canaryIds.size === 0) return samples.filter((item) => item?.[key]);
+  return (samples || []).filter((sample) => {
+    const value = stringOrNull(sample?.[key]);
+    if (!value) return false;
+    if (canaryIds.has(value)) return false;
+    return !sampleInControlledNavigationWindow(sample, windows);
+  });
+}
+
+function sampleInControlledNavigationWindow(sample, windows) {
+  const ms = timestampMs(sample?.ts);
+  if (!Number.isFinite(ms)) return false;
+  return (windows || []).some((window) => ms >= window.startMs && ms <= window.endMs);
+}
+
+function objectValue(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+
+function stringOrNull(value) {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function numberOrNull(value) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function arrayStrings(value) {
+  return Array.isArray(value) ? value.map((item) => String(item || "")).filter(Boolean) : [];
+}
+
+function timestampMs(value) {
+  const parsed = Date.parse(String(value || ""));
+  return Number.isFinite(parsed) ? parsed : NaN;
+}
+
+function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, commandFailures = [], manifest = {}) {
  const findings = [];
  if (commandFailures.length > 0) findings.push({ id: "observer-command-failed", severity: "red", summary: "observer control commands failed; analyze must surface command failure instead of hiding it in command artifacts", count: commandFailures.length, commands: commandFailures.slice(0, 20) });
+  findings.push(...buildSessionInvariantFindings(control, manifest));
  const commandTimes = control
    .filter((item) => item.phase === "completed" || item.phase === "started" || item.type === "observer-periodic-refresh")
    .map((item) => Date.parse(item.ts))
    .filter(Number.isFinite);
+  const controlledNavigationWindows = sessionInvariantNavigationWindows(control);
  const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
  const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean));
-  if (routeSessions.size > 1) findings.push({ id: "session-route-changed", severity: "amber", summary: "route session changed during observation", routeSessionCount: routeSessions.size, samples: sampleRefs(samples, (item) => item.routeSessionId) });
-  if (activeSessions.size > 1) findings.push({ id: "active-session-changed", severity: "amber", summary: "active session changed during observation", activeSessionCount: activeSessions.size, samples: sampleRefs(samples, (item) => item.activeSessionId) });
-  const mismatches = samples.filter((item) => item.routeSessionId && item.activeSessionId && item.routeSessionId !== item.activeSessionId);
+  const routeSessionUnexpected = sessionChangeSamplesOutsideControlledNavigation(samples, "routeSessionId", controlledNavigationWindows);
+  const activeSessionUnexpected = sessionChangeSamplesOutsideControlledNavigation(samples, "activeSessionId", controlledNavigationWindows);
+  if (routeSessions.size > 1 && routeSessionUnexpected.length > 0) findings.push({ id: "session-route-changed", severity: "amber", summary: "route session changed outside controlled session-invariance navigation windows", routeSessionCount: routeSessions.size, samples: sampleRefs(routeSessionUnexpected, (item) => item.routeSessionId) });
+  if (activeSessions.size > 1 && activeSessionUnexpected.length > 0) findings.push({ id: "active-session-changed", severity: "amber", summary: "active session changed outside controlled session-invariance navigation windows", activeSessionCount: activeSessions.size, samples: sampleRefs(activeSessionUnexpected, (item) => item.activeSessionId) });
+  const mismatches = samples.filter((item) => item.routeSessionId && item.activeSessionId && item.routeSessionId !== item.activeSessionId && !sampleInControlledNavigationWindow(item, controlledNavigationWindows));
  if (mismatches.length > 0) findings.push({ id: "route-active-session-mismatch", severity: "red", summary: "routeSessionId and activeSessionId diverged", count: mismatches.length, samples: mismatches.slice(0, 10).map(ref) });
  const uncommandedChanges = [];
  const commandedPromptSeqs = new Set((sampleMetrics?.timeline ?? []).filter((item) => Number(item?.promptIndex) > 0).map((item) => item.seq).filter((seq) => seq !== null && seq !== undefined));