fix: restore web sentinel recovery diagnostics (#980)

Co-authored-by: Codex <codex@noreply.local>
2026-06-26 14:04:37 +08:00
parent fc6d3bdaf9
commit bdec05729d
8 changed files with 409 additions and 161 deletions
@@ -1,4 +1,5 @@
 // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
+// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p8-web-probe-sentinel-recovery.
 // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
@@ -1353,10 +1354,28 @@ function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
  return {};
 }

-function sentinelElapsedWarnings(value: unknown): string[] {
+function sentinelElapsedWarnings(value: unknown, subject = "sentinel confirmed operation"): string[] {
  const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
  if (elapsedMs === null || elapsedMs <= 120_000) return [];
-  return [`sentinel confirmed operation exceeded 120s (${Math.round(elapsedMs / 1000)}s); investigate env-reuse/git mirror/source build path before treating this as normal.`];
+  return [`${subject} exceeded 120s (${Math.round(elapsedMs / 1000)}s); treat this as a severe timeout and investigate env-reuse/git mirror/source build path plus the current wait stage before retrying.`];
+}
+
+function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] {
+  const warnings: string[] = [];
+  for (const item of items) {
+    const values = Array.isArray(item) ? item : [item];
+    for (const value of values) {
+      if (value === undefined || value === null || value === "") continue;
+      const warning = text(value).trim();
+      if (warning.length > 0 && warning !== "-" && !warnings.includes(warning)) warnings.push(warning);
+    }
+  }
+  return warnings;
+}
+
+function withWarnings(payload: Record<string, unknown>, warnings: readonly unknown[]): Record<string, unknown> {
+  const merged = mergeWarnings(payload.warnings, warnings);
+  return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true };
 }

 function sentinelProgressEvent(event: string, payload: Record<string, unknown>): void {
@@ -1469,6 +1488,7 @@ function runSentinelMaintenance(state: SentinelCicdState, options: Extract<WebPr

 function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "validate" }>): RenderedCliResult {
  const command = "web-probe sentinel validate";
+  const startedAt = Date.now();
  const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds);
  let quickVerify: Record<string, unknown> | null = null;
  if (options.quickVerify) {
@@ -1508,6 +1528,9 @@ function runSentinelValidate(state: SentinelCicdState, options: Extract<WebProbe
  const report = callSentinelService(state, "GET", "/api/report?view=summary", null, options.timeoutSeconds);
  const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds);
  const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds);
+  if (quickVerify !== null) {
+    quickVerify = withWarnings(quickVerify, sentinelElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait"));
+  }
  const ok = health.ok
    && record(health.bodyJson).ok === true
    && metrics.ok
@@ -1572,6 +1595,9 @@ function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[
 }

 function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
+  const startedAt = Date.now();
+  const elapsedMs = () => Date.now() - startedAt;
+  const elapsedWarnings = () => sentinelElapsedWarnings(elapsedMs(), "quick verify confirm-wait");
  const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
  const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
  const scenario = findScenario(state, scenarioId);
@@ -1595,6 +1621,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
  steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
  const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
  if (!started.ok || observerId === null) {
+    const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null);
    return recordQuickVerify(state, {
      ok: false,
      runId,
@@ -1602,8 +1629,12 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
      reason,
      status: "blocked",
      observerId,
+      elapsedMs: elapsedMs(),
      steps,
      failure: "observe-start-failed",
+      findingCount: findings.length,
+      findings,
+      warnings: elapsedWarnings(),
      valuesRedacted: true,
    });
  }
@@ -1621,7 +1652,8 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
          promptIndex,
          steps,
          failure: "quick-verify-timeout-over-120s",
-          warnings: ["quick verify exceeded the configured 120s targetValidation budget; investigate env-reuse/git mirror/source build path before retrying."],
+          elapsedMs: elapsedMs(),
+          warnings: mergeWarnings("quick verify exceeded the configured 120s targetValidation budget; investigate env-reuse/git mirror/source build path before retrying.", elapsedWarnings()),
          promptSource: prompts.summary,
        }));
      }
@@ -1642,6 +1674,8 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
          promptIndex,
          steps,
          failure: `observe-command-${type}-failed`,
+          elapsedMs: elapsedMs(),
+          warnings: elapsedWarnings(),
          promptSource: prompts.summary,
        }));
      }
@@ -1658,7 +1692,8 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
            steps,
            failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"),
            promptSource: prompts.summary,
-            warnings: Array.isArray(waitResult.warnings) ? waitResult.warnings : [],
+            elapsedMs: elapsedMs(),
+            warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()),
          }));
        }
      }
@@ -1670,7 +1705,11 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
  const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30));
  const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
  const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30));
-  const ok = analysis.ok && record(artifactSummary).ok === true;
+  const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame);
+  const artifactSummaryRecord = record(artifactSummary);
+  const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
+  const findings = mergeFindingRecords(artifactFindings, controlFindings);
+  const ok = analysis.ok && record(artifactSummary).ok === true && controlFindings.length === 0;
  return recordQuickVerify(state, {
    ok,
    runId,
@@ -1678,10 +1717,12 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
    reason,
    status: ok ? "analyzed" : "blocked",
    observerId,
+    elapsedMs: elapsedMs(),
    stateDir: indexEntry?.stateDir ?? null,
    reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
-    findingCount: numberAtNullable(artifactSummary, "findingCount") ?? 0,
+    findingCount: findings.length,
    artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
+    failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : null,
    promptSource: prompts.summary,
    steps,
    analysis: artifactSummary,
@@ -1690,9 +1731,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
      "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
      "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
    },
-    findings: Array.isArray(record(artifactSummary).findings) ? record(artifactSummary).findings : [],
+    findings,
    screenshot: record(artifactSummary).screenshot,
    publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
+    warnings: elapsedWarnings(),
    valuesRedacted: true,
  });
 }
@@ -1706,6 +1748,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
  readonly steps: readonly Record<string, unknown>[];
  readonly failure: string;
  readonly promptSource?: Record<string, unknown>;
+  readonly elapsedMs?: number;
  readonly warnings?: readonly unknown[];
 }): Record<string, unknown> {
  const cleanupSteps: Record<string, unknown>[] = [];
@@ -1741,6 +1784,10 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
    : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30);
  const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30);
  const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30);
+  const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame);
+  const artifactSummaryRecord = record(artifactSummary);
+  const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
+  const findings = mergeFindingRecords(artifactFindings, controlFindings);
  return {
    ok: false,
    runId: input.runId,
@@ -1748,9 +1795,10 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
    reason: input.reason,
    status: "blocked",
    observerId: input.observerId,
+    elapsedMs: input.elapsedMs ?? null,
    stateDir: indexEntry?.stateDir ?? null,
    reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
-    findingCount: numberAtNullable(artifactSummary, "findingCount") ?? 0,
+    findingCount: findings.length,
    artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
    failure: input.failure,
    promptSource: input.promptSource,
@@ -1761,10 +1809,10 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
      "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
      "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
    },
-    findings: Array.isArray(record(artifactSummary).findings) ? record(artifactSummary).findings : [],
+    findings,
    screenshot: record(artifactSummary).screenshot,
    publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
-    warnings: Array.isArray(input.warnings) ? input.warnings.map(text) : [],
+    warnings: mergeWarnings(Array.isArray(input.warnings) ? input.warnings : [], sentinelElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait")),
    valuesRedacted: true,
  };
 }
@@ -1782,6 +1830,9 @@ function recordQuickVerify(state: SentinelCicdState, payload: Record<string, unk
    summary: {
      reason: payload.reason,
      status: payload.status,
+      elapsedMs: payload.elapsedMs,
+      failure: payload.failure,
+      warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
      analysis: payload.analysis,
      promptSource: payload.promptSource,
      steps: payload.steps,
@@ -2464,6 +2515,40 @@ function displayPath(pathValue: string): string {
  return pathValue;
 }

+function mergeFindingRecords(primary: readonly Record<string, unknown>[], extra: readonly Record<string, unknown>[]): Record<string, unknown>[] {
+  const merged: Record<string, unknown>[] = [];
+  const seen = new Set<string>();
+  for (const item of [...primary, ...extra]) {
+    const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? stringAtNullable(item, "finding_id") ?? "finding";
+    const severity = stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "unknown";
+    const key = `${id}\0${severity}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    merged.push(item);
+  }
+  return merged;
+}
+
+function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null): Record<string, unknown>[] {
+  const rendered = [
+    typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "",
+    typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "",
+  ].join("\n");
+  const noPrompt = promptIndex <= 0 || /无\s*sendPrompt|no\s+sendPrompt/iu.test(rendered);
+  const noTrace = /无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
+  const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered);
+  if (!noPrompt && !noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
+  return [{
+    id: "quick-verify-no-business-turn",
+    severity: "red",
+    count: 1,
+    summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
+    failure: failure ?? null,
+    promptIndex,
+    valuesRedacted: true,
+  }];
+}
+
 function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } {
  return {
    ...compactCommand(result),