fix: improve sentinel observer evidence

2026-07-06 23:48:33 +00:00
parent 51058f4635
commit e0199e5515
6 changed files with 171 additions and 15 deletions
@@ -13,6 +13,7 @@ export function nodeWebObserveAnalyzerFindingsSource(): string {
    summary: "observer control commands failed; analyze must surface the first failed command with phase/status/stdout/stderr evidence.",
    count: commandFailures.length,
    firstCommandFailure,
+    evidenceSummary: firstCommandFailure ? "command=" + (firstCommandFailure.commandId || "-") + " type=" + (firstCommandFailure.type || "-") + " phase=" + (firstCommandFailure.phase || "-") + " status=" + (firstCommandFailure.status || "-") + " resultOk=" + (firstCommandFailure.resultOk ?? "-") + " failedReason=" + (firstCommandFailure.failedReason || firstCommandFailure.failureKind || firstCommandFailure.message || "-") : "commandFailures>0",
    commands: compactCommandFailures.slice(0, 20)
  });
  findings.push(...buildFrontendFreezeFindings(errors, control));
@@ -429,18 +430,36 @@ export function nodeWebObserveAnalyzerFindingsSource(): string {
 function compactObserverCommandFailure(item) {
  const result = item && typeof item.result === "object" ? item.result : {};
  const output = item && typeof item.output === "object" ? item.output : {};
+  const error = item && typeof item.error === "object" ? item.error : result && typeof result.error === "object" ? result.error : {};
  const stdout = item?.stdoutPreview ?? item?.stdoutTail ?? result.stdoutPreview ?? result.stdoutTail ?? output.stdoutPreview ?? output.stdoutTail ?? null;
  const stderr = item?.stderrPreview ?? item?.stderrTail ?? result.stderrPreview ?? result.stderrTail ?? output.stderrPreview ?? output.stderrTail ?? null;
+  const commandId = item?.commandId ?? item?.id ?? item?.command_id ?? null;
+  const phase = item?.phase ?? item?.statusPhase ?? (item?.failedAt ? "failed" : item?.abandonedAt ? "abandoned" : item?.completedAt ? "completed" : null);
+  const resultOk = item?.resultOk ?? item?.ok ?? result.ok ?? null;
+  const status = item?.status ?? item?.resultStatus ?? result.status ?? (phase === "failed" || resultOk === false ? "failed" : phase === "abandoned" ? "abandoned" : phase === "completed" ? "completed" : null);
+  const failureKind = item?.failureKind ?? result.failureKind ?? error.failureKind ?? null;
+  const message = item?.message ?? result.message ?? error.message ?? item?.reason ?? result.reason ?? null;
+  const failedReason = item?.failedReason ?? item?.reason ?? result.failedReason ?? result.reason ?? failureKind ?? message ?? null;
+  const artifactBucket = item?.artifactBucket ?? (phase === "failed" ? "failed" : phase === "abandoned" ? "abandoned" : phase === "completed" ? "done" : null);
+  const artifactRef = item?.artifactRef ?? item?.artifactPath ?? item?.path ?? (commandId && artifactBucket ? "commands/" + artifactBucket + "/" + commandId + ".json" : null);
  return {
-    commandId: item?.commandId ?? item?.id ?? item?.command_id ?? null,
+    commandId,
    type: item?.type ?? item?.commandType ?? item?.command_type ?? null,
-    phase: item?.phase ?? item?.statusPhase ?? null,
-    status: item?.status ?? item?.resultStatus ?? null,
+    phase,
+    status,
+    resultOk: resultOk === true ? true : resultOk === false ? false : null,
+    failureKind: failureKind == null ? null : String(failureKind).slice(0, 160),
+    failedReason: failedReason == null ? null : String(failedReason).slice(0, 240),
+    message: message == null ? null : String(message).slice(0, 240),
    exitCode: item?.exitCode ?? result.exitCode ?? null,
    timedOut: item?.timedOut === true || result.timedOut === true,
    stdoutPreview: stdout == null ? null : String(stdout).slice(0, 240),
    stderrPreview: stderr == null ? null : String(stderr).slice(0, 240),
-    artifactRef: item?.artifactRef ?? item?.artifactPath ?? item?.path ?? null,
+    artifactBucket,
+    artifactRef,
+    failedAt: item?.failedAt ?? null,
+    completedAt: item?.completedAt ?? null,
+    abandonedAt: item?.abandonedAt ?? null,
    valuesRedacted: true
  };
 }
@@ -838,6 +838,8 @@ function summarizeCommandFailures(control) {
      ts: item.ts ?? null,
      commandId: item.commandId ?? null,
      type: item.type ?? item.input?.type ?? null,
+      phase: "failed",
+      status: "failed",
      source: item.source ?? null,
      durationMs: detail.durationMs ?? null,
      beforePath: urlPath(detail.beforeUrl || item.beforeUrl),
@@ -845,6 +847,11 @@ function summarizeCommandFailures(control) {
      name: error?.name ?? null,
      message: limitText(error?.message ?? detail?.message ?? "", 240),
      failureKind: error?.failureKind ?? detail?.failureKind ?? null,
+      failedReason: limitText(error?.message ?? detail?.message ?? error?.failureKind ?? detail?.failureKind ?? "observer-command-failed", 240),
+      resultOk: false,
+      artifactBucket: "failed",
+      artifactRef: item.commandId ? "commands/failed/" + item.commandId + ".json" : null,
+      failedAt: item.ts ?? null,
      failureSampleOk: detail?.failureSample?.ok === true,
      sampleSeq: detail?.failureSample?.sampleSeq ?? null,
      valuesRedacted: true
@@ -43,6 +43,35 @@ test("quick verify cleanup downgrades abandoned graceful stop when force stop co
  assert.equal(findings[0]?.severity, "warning");
 });

+test("quick verify cleanup reads force-stop evidence from top-level CLI payload", () => {
+  const step = {
+    phase: "observe-stop-after-terminal",
+    ok: false,
+    payload: {
+      ok: true,
+      command: "web-probe-observe force-stop",
+      forced: true,
+      reason: "graceful-stop-not-consumed",
+      aliveBefore: true,
+      aliveAfter: false,
+      termSent: true,
+      killSent: true,
+      pendingAbandoned: 1,
+      processingAbandoned: 0,
+      valuesRedacted: true,
+    },
+    valuesRedacted: true,
+  };
+
+  const cleanup = classifyQuickVerifyCleanupStep(step);
+  const findings = quickVerifyCleanupFindings(step);
+  assert.equal(cleanup.cleanupClass, "bounded-force-stop");
+  assert.equal(cleanup.forceStopOk, true);
+  assert.equal(cleanup.aliveAfter, false);
+  assert.deepEqual(cleanup.evidenceSources, ["payload"]);
+  assert.equal(findings[0]?.id, "observer-graceful-stop-timeout-force-stop-succeeded");
+});
+
 test("quick verify cleanup emits process-alive code when force stop leaves observer alive", () => {
  const step = forceStopStep({
    aliveBefore: true,
@@ -76,6 +105,8 @@ test("quick verify cleanup emits evidence-gap code when stopped state cannot be
  const findings = quickVerifyCleanupFindings(step);
  assert.equal(cleanup.cleanupClass, "observer-stop-failed");
  assert.equal(cleanup.aliveAfter, null);
+  assert.deepEqual(cleanup.missingEvidenceFields, ["aliveAfter"]);
  assert.equal(findings.length, 1);
  assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing");
+  assert.match(String(findings[0]?.evidenceSummary), /missing=aliveAfter/u);
 });
@@ -683,19 +683,40 @@ export function classifyQuickVerifyCleanupStep(cleanupStep: Record<string, unkno
  if (stringAtNullable(explicit, "cleanupClass") !== null) return explicit;
  const payload = quickVerifyCleanupPayload(cleanupStep);
  const observer = record(payload.observer);
+  const forceStop = record(payload.forceStop);
+  const detail = record(payload.detail);
  const forceResult = record(payload.forceResult);
-  const forceReason = stringAtNullable(payload, "forceReason");
-  const status = stringAtNullable(payload, "status");
+  const sources = cleanupEvidenceSources([
+    ["payload.observer", observer],
+    ["payload.forceStop", forceStop],
+    ["payload.detail", detail],
+    ["payload", payload],
+    ["payload.forceResult", forceResult],
+  ]);
+  const forceReason = firstStringFromCleanupSources(sources, ["forceReason", "reason"]);
+  const status = firstStringFromCleanupSources(sources, ["status"]);
+  const command = firstStringFromCleanupSources(sources, ["command"]);
+  const evidenceSourceNames = cleanupEvidenceSourceNames(sources, [
+    "aliveBefore",
+    "aliveAfter",
+    "termSent",
+    "killSent",
+    "pendingAbandoned",
+    "processingAbandoned",
+  ]);
  const forceStopAttempted = forceReason !== null
    || status === "forced-stopped"
-    || stringAtNullable(observer, "command") === "web-probe-observe force-stop"
-    || numberAtNullable(forceResult, "exitCode") !== null;
-  const aliveBefore = booleanAtNullable(observer, "aliveBefore");
-  const aliveAfter = booleanAtNullable(observer, "aliveAfter");
-  const termSent = booleanAtNullable(observer, "termSent");
-  const killSent = booleanAtNullable(observer, "killSent");
-  const pendingAbandoned = numberAtNullable(observer, "pendingAbandoned");
-  const processingAbandoned = numberAtNullable(observer, "processingAbandoned");
+    || status === "force-stopped"
+    || command === "web-probe-observe force-stop"
+    || payload.forced === true
+    || numberAtNullable(forceResult, "exitCode") !== null
+    || evidenceSourceNames.length > 0;
+  const aliveBefore = firstBooleanFromCleanupSources(sources, "aliveBefore");
+  const aliveAfter = firstBooleanFromCleanupSources(sources, "aliveAfter");
+  const termSent = firstBooleanFromCleanupSources(sources, "termSent");
+  const killSent = firstBooleanFromCleanupSources(sources, "killSent");
+  const pendingAbandoned = firstNumberFromCleanupSources(sources, "pendingAbandoned");
+  const processingAbandoned = firstNumberFromCleanupSources(sources, "processingAbandoned");
  const forceStopOk = forceStopAttempted && aliveAfter === false;
  const gracefulStopOk = !forceStopAttempted && (cleanupStep.ok === true || payload.ok === true);
  const cleanupClass = gracefulStopOk
@@ -703,6 +724,15 @@ export function classifyQuickVerifyCleanupStep(cleanupStep: Record<string, unkno
    : forceStopOk
      ? "bounded-force-stop"
      : "observer-stop-failed";
+  const missingEvidenceFields = forceStopAttempted && cleanupClass === "observer-stop-failed"
+    ? [
+      aliveAfter === null ? "aliveAfter" : null,
+      termSent === null ? "termSent" : null,
+      killSent === null ? "killSent" : null,
+      pendingAbandoned === null ? "pendingAbandoned" : null,
+      processingAbandoned === null ? "processingAbandoned" : null,
+    ].filter((value): value is string => value !== null)
+    : [];
  return {
    cleanupClass,
    phase: stringAtNullable(cleanupStep, "phase"),
@@ -717,6 +747,8 @@ export function classifyQuickVerifyCleanupStep(cleanupStep: Record<string, unkno
    killSent,
    pendingAbandoned,
    processingAbandoned,
+    evidenceSources: evidenceSourceNames,
+    missingEvidenceFields,
    valuesRedacted: true,
  };
 }
@@ -733,6 +765,46 @@ function quickVerifyCleanupPayload(cleanupStep: Record<string, unknown>): Record
  return {};
 }

+function cleanupEvidenceSources(entries: readonly (readonly [string, Record<string, unknown>])[]): readonly { name: string; value: Record<string, unknown> }[] {
+  return entries
+    .filter((entry) => Object.keys(entry[1]).length > 0)
+    .map(([name, value]) => ({ name, value }));
+}
+
+function cleanupEvidenceSourceNames(sources: readonly { name: string; value: Record<string, unknown> }[], keys: readonly string[]): string[] {
+  const names: string[] = [];
+  for (const source of sources) {
+    if (keys.some((key) => source.value[key] !== undefined && source.value[key] !== null)) names.push(source.name);
+  }
+  return [...new Set(names)];
+}
+
+function firstStringFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], keys: readonly string[]): string | null {
+  for (const source of sources) {
+    for (const key of keys) {
+      const value = stringAtNullable(source.value, key);
+      if (value !== null) return value;
+    }
+  }
+  return null;
+}
+
+function firstBooleanFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], key: string): boolean | null {
+  for (const source of sources) {
+    const value = booleanAtNullable(source.value, key);
+    if (value !== null) return value;
+  }
+  return null;
+}
+
+function firstNumberFromCleanupSources(sources: readonly { value: Record<string, unknown> }[], key: string): number | null {
+  for (const source of sources) {
+    const value = numberAtNullable(source.value, key);
+    if (value !== null) return value;
+  }
+  return null;
+}
+
 function quickVerifyCleanupFromSteps(steps: readonly Record<string, unknown>[]): Record<string, unknown> | null {
  for (const step of steps.slice().reverse()) {
    const phase = stringAtNullable(step, "phase");
@@ -765,6 +837,7 @@ function formatQuickVerifyCleanupLine(cleanup: Record<string, unknown> | null):
    ["processingAbandoned", cleanup.processingAbandoned],
    ["gracefulStopOk", cleanup.gracefulStopOk],
    ["forceStopOk", cleanup.forceStopOk],
+    ["missing", Array.isArray(cleanup.missingEvidenceFields) && cleanup.missingEvidenceFields.length > 0 ? cleanup.missingEvidenceFields.join(",") : null],
  ];
  return fields.map(([key, value]) => `${key}=${value ?? "-"}`).join(" ");
 }
@@ -1263,6 +1336,8 @@ function compactQuickVerifyCleanupEvidence(value: unknown): Record<string, unkno
    killSent: booleanAtNullable(item, "killSent"),
    pendingAbandoned: numberAtNullable(item, "pendingAbandoned"),
    processingAbandoned: numberAtNullable(item, "processingAbandoned"),
+    evidenceSources: Array.isArray(item.evidenceSources) ? item.evidenceSources.filter((value): value is string => typeof value === "string").slice(0, 6) : [],
+    missingEvidenceFields: Array.isArray(item.missingEvidenceFields) ? item.missingEvidenceFields.filter((value): value is string => typeof value === "string").slice(0, 8) : [],
    valuesRedacted: true,
  };
 }
@@ -456,6 +456,10 @@ function hasCommandFailureEvidence(item: Record<string, unknown>): boolean {
  return stringAtNullable(first, "type") !== null
    || stringAtNullable(first, "phase") !== null
    || stringAtNullable(first, "status") !== null
+    || stringAtNullable(first, "failedReason") !== null
+    || stringAtNullable(first, "failureKind") !== null
+    || stringAtNullable(first, "message") !== null
+    || stringAtNullable(first, "artifactRef") !== null
    || stringAtNullable(first, "stdoutPreview") !== null
    || stringAtNullable(first, "stderrPreview") !== null
    || numberAtNullable(first, "exitCode") !== null;
@@ -616,11 +620,19 @@ function compactSentinelReportCommandFailure(value: unknown): Record<string, unk
    type: stringAtNullable(item, "type"),
    phase: stringAtNullable(item, "phase"),
    status: stringAtNullable(item, "status"),
+    resultOk: item.resultOk === true ? true : item.resultOk === false ? false : null,
+    failureKind: reportText(item.failureKind, 160),
+    failedReason: reportText(item.failedReason, 240),
+    message: reportText(item.message, 240),
    exitCode: numberAtNullable(item, "exitCode"),
    timedOut: item.timedOut === true ? true : item.timedOut === false ? false : null,
    stdoutPreview: reportText(item.stdoutPreview, 160),
    stderrPreview: reportText(item.stderrPreview, 160),
+    artifactBucket: stringAtNullable(item, "artifactBucket"),
    artifactRef: stringAtNullable(item, "artifactRef"),
+    failedAt: stringAtNullable(item, "failedAt"),
+    completedAt: stringAtNullable(item, "completedAt"),
+    abandonedAt: stringAtNullable(item, "abandonedAt"),
    valuesRedacted: true,
  };
 }
@@ -645,7 +657,7 @@ function readSentinelReportArtifactSummary(state: SentinelCicdState, body: Recor
    "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;",
    "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);",
    "const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};",
-    "const compactCommandFailure=(value)=>{const v=rec(value); if(Object.keys(v).length===0)return null; return {commandId:clip(v.commandId,80),type:clip(v.type,48),phase:clip(v.phase,80),status:clip(v.status,80),exitCode:Number.isFinite(Number(v.exitCode))?Number(v.exitCode):null,timedOut:v.timedOut===true?true:v.timedOut===false?false:null,stdoutPreview:clip(v.stdoutPreview,160),stderrPreview:clip(v.stderrPreview,160),artifactRef:clip(v.artifactRef,160),valuesRedacted:true};};",
+    "const compactCommandFailure=(value)=>{const v=rec(value); if(Object.keys(v).length===0)return null; return {commandId:clip(v.commandId,80),type:clip(v.type,48),phase:clip(v.phase,80),status:clip(v.status,80),resultOk:v.resultOk===true?true:v.resultOk===false?false:null,failureKind:clip(v.failureKind,160),failedReason:clip(v.failedReason,240),message:clip(v.message,240),exitCode:Number.isFinite(Number(v.exitCode))?Number(v.exitCode):null,timedOut:v.timedOut===true?true:v.timedOut===false?false:null,stdoutPreview:clip(v.stdoutPreview,160),stderrPreview:clip(v.stderrPreview,160),artifactBucket:clip(v.artifactBucket,48),artifactRef:clip(v.artifactRef,160),failedAt:clip(v.failedAt,48),completedAt:clip(v.completedAt,48),abandonedAt:clip(v.abandonedAt,48),valuesRedacted:true};};",
    "const sleep=(ms)=>{try{Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms)}catch{}};",
    "let jsonBuf=read(reportPath); let report=null; let reportParseError=null; const deadline=Date.now()+waitMs;",
    "while(true){ if(jsonBuf){try{report=JSON.parse(jsonBuf.toString('utf8')); break}catch(err){reportParseError=String(err&&err.message?err.message:err)}} if(Date.now()>=deadline)break; sleep(Math.min(500, Math.max(50, deadline-Date.now()))); jsonBuf=read(reportPath); }",
@@ -1896,6 +1896,10 @@ function hasCommandFailureEvidence(item: Record<string, unknown>): boolean {
  return stringOrNull(first.type) !== null
    || stringOrNull(first.phase) !== null
    || stringOrNull(first.status) !== null
+    || stringOrNull(first.failedReason) !== null
+    || stringOrNull(first.failureKind) !== null
+    || stringOrNull(first.message) !== null
+    || stringOrNull(first.artifactRef) !== null
    || stringOrNull(first.stdoutPreview) !== null
    || stringOrNull(first.stderrPreview) !== null
    || numberOrNull(first.exitCode) !== null;
@@ -2061,11 +2065,19 @@ function compactCommandFailureEvidence(value: unknown): Record<string, unknown>
    type: stringOrNull(item.type),
    phase: stringOrNull(item.phase),
    status: stringOrNull(item.status),
+    resultOk: item.resultOk === true ? true : item.resultOk === false ? false : null,
+    failureKind: stringOrNull(item.failureKind)?.slice(0, 160) ?? null,
+    failedReason: stringOrNull(item.failedReason)?.slice(0, 240) ?? null,
+    message: stringOrNull(item.message)?.slice(0, 240) ?? null,
    exitCode: numberOrNull(item.exitCode),
    timedOut: item.timedOut === true ? true : item.timedOut === false ? false : null,
    stdoutPreview: stringOrNull(item.stdoutPreview)?.slice(0, 240) ?? null,
    stderrPreview: stringOrNull(item.stderrPreview)?.slice(0, 240) ?? null,
+    artifactBucket: stringOrNull(item.artifactBucket),
    artifactRef: stringOrNull(item.artifactRef),
+    failedAt: stringOrNull(item.failedAt),
+    completedAt: stringOrNull(item.completedAt),
+    abandonedAt: stringOrNull(item.abandonedAt),
    valuesRedacted: true,
  };
 }