fix: split quick verify control findings

2026-07-02 07:33:19 +00:00
parent a5644314cd
commit 1c9132e7c6
5 changed files with 403 additions and 33 deletions
@@ -342,6 +342,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
  );
  const ok = analysisReadable && controlFindings.length === 0 && blockingFindings.length === 0;
  const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds);
+  const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
  printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), analysisReadable, findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
  return recordQuickVerify(state, {
    ok,
@@ -356,7 +357,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
    reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
    findingCount: findings.length,
    artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
-    failure: !analysisReadable ? "quick-verify-analysis-missing" : controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null,
+    failure: !analysisReadable ? "quick-verify-analysis-missing" : primaryControlFailure ?? (blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null),
    promptSource: prompts.summary,
    accountEnv: accountEnv.summary,
    steps: [...steps, cleanupStep],
@@ -481,6 +482,44 @@ function inlinePromptText(item: Record<string, unknown>): string | null {
  return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value");
 }

+export function reclassifyQuickVerifyControlFindings(state: SentinelCicdState, input: {
+  readonly runId: string | null;
+  readonly scenarioId: string | null;
+  readonly observerId: string | null;
+  readonly failure?: string | null;
+  readonly timeoutSeconds?: number | null;
+}): Record<string, unknown> {
+  if (input.observerId === null || input.observerId.length === 0) {
+    return { ok: false, reason: "observer-id-missing", runId: input.runId, valuesRedacted: true };
+  }
+  const scenarioId = input.scenarioId ?? stringAt(state.cicd, "targetValidation.scenarioId");
+  const scenario = findScenario(state, scenarioId);
+  if (scenario === null) {
+    return { ok: false, reason: "scenario-not-found", runId: input.runId, scenarioId, observerId: input.observerId, valuesRedacted: true };
+  }
+  const commandSequence = arrayAt(scenario, "commandSequence").map(record);
+  const promptIndex = commandSequence.reduce((count, item) => {
+    const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
+    return stringAtNullable(item, "type") === "sendPrompt" ? count + repeat : count;
+  }, 0);
+  const timeoutSeconds = Math.max(5, Math.min(Math.trunc(input.timeoutSeconds ?? 55), 55));
+  const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, timeoutSeconds);
+  const traceFrame = collectObserveView(state, input.observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, timeoutSeconds);
+  const findings = quickVerifyControlFindings(input.failure ?? null, promptIndex, turnSummary, traceFrame);
+  return {
+    ok: true,
+    runId: input.runId,
+    scenarioId,
+    observerId: input.observerId,
+    promptIndex,
+    findingCount: findings.length,
+    findings,
+    turnSummary: { ok: turnSummary.ok === true, collectShape: turnSummary.collectShape, valuesRedacted: true },
+    traceFrame: { ok: traceFrame.ok === true, collectShape: traceFrame.collectShape, valuesRedacted: true },
+    valuesRedacted: true,
+  };
+}
+
 function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
  readonly runId: string;
  readonly scenarioId: string;
@@ -542,6 +581,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
    && controlFindings.length === 0
    && blockingFindings.length === 0;
  const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds);
+  const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
  return {
    ok: recoveredWaitFailure,
    runId: input.runId,
@@ -558,7 +598,8 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
    reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
    findingCount: findings.length,
    artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
-    failure: recoveredWaitFailure ? null : input.failure,
+    failure: recoveredWaitFailure ? null : primaryControlFailure ?? input.failure,
+    runnerFailure: recoveredWaitFailure ? null : input.failure,
    promptSource: input.promptSource,
    steps: [...input.steps, ...cleanupSteps],
    analysis: artifactSummary,
@@ -1811,6 +1852,13 @@ function isQuickVerifyBlockingFinding(item: Record<string, unknown>): boolean {
  if (id === "observer-command-failed") return observerCommandFailureBlocks(item);
  return [
    "quick-verify-no-business-turn",
+    "quick-verify-target-turn-missing",
+    "quick-verify-target-turn-trace-id-missing",
+    "quick-verify-trace-projection-missing",
+    "quick-verify-final-response-empty-incomplete",
+    "quick-verify-final-response-empty-terminal",
+    "quick-verify-final-response-empty-status-unknown",
+    "quick-verify-diagnostics-inconclusive",
    "quick-verify-analysis-summary-unreadable",
    "quick-verify-command-sequence-failed",
    "quick-verify-observer-start-failed",
@@ -1836,14 +1884,105 @@ function observerCommandFailureBlocks(item: Record<string, unknown>): boolean {
  });
 }

+function quickVerifyPrimaryControlFailure(controlFindings: Record<string, unknown>[]): string | null {
+  const first = controlFindings[0];
+  return first === undefined ? null : stringAtNullable(first, "id") ?? stringAtNullable(first, "kind") ?? stringAtNullable(first, "code");
+}
+
+function quickVerifyControlEvidenceSummary(turnDiagnostics: Record<string, unknown>, traceDiagnostics: Record<string, unknown>, targetStatus: Record<string, unknown> | null): string {
+  const targetStatusText = targetStatus === null ? "-" : stringAtNullable(targetStatus, "status") ?? "-";
+  const targetMarks = targetStatus === null ? "-" : stringAtNullable(targetStatus, "marks") ?? "-";
+  const targetTrace = targetStatus === null ? "-" : targetStatus.traceIdPresent === true ? "present" : "missing";
+  const targetFinalBytes = targetStatus === null ? "-" : String(targetStatus.finalResponseBytes ?? "-");
+  const traceBlocker = stringAtNullable(traceDiagnostics, "blocker") ?? "-";
+  const traceRowCount = traceDiagnostics.traceRowCount === null || traceDiagnostics.traceRowCount === undefined ? "-" : String(traceDiagnostics.traceRowCount);
+  return [
+    `turn-summary rows=${String(turnDiagnostics.rowCount ?? 0)}`,
+    `scoped=${String(turnDiagnostics.scopedRowCount ?? 0)}`,
+    `targetStatus=${targetStatusText}`,
+    `targetMarks=${targetMarks}`,
+    `targetTrace=${targetTrace}`,
+    `targetFinalBytes=${targetFinalBytes}`,
+    `traceFrameTrace=${traceDiagnostics.traceIdPresent === true ? "present" : "missing"}`,
+    `traceFrameBlocker=${traceBlocker}`,
+    `traceRows=${traceRowCount}`,
+    `traceFinalBytes=${String(traceDiagnostics.finalResponseBytes ?? "-")}`,
+  ].join(" ");
+}
+
+function quickVerifyScopedStatuses(turnDiagnostics: Record<string, unknown>): Record<string, unknown>[] {
+  return Array.isArray(turnDiagnostics.scopedStatuses) ? turnDiagnostics.scopedStatuses.map(record) : [];
+}
+
+function quickVerifyLatestScopedStatus(turnDiagnostics: Record<string, unknown>): Record<string, unknown> | null {
+  const rows = quickVerifyScopedStatuses(turnDiagnostics);
+  return rows.length === 0 ? null : rows[rows.length - 1] ?? null;
+}
+
+function quickVerifyFinalResponseEmpty(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
+  return targetStatus?.finalResponseEmpty === true || traceDiagnostics.finalResponseEmpty === true;
+}
+
+function quickVerifyTargetTracePresent(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
+  return targetStatus?.traceIdPresent === true || traceDiagnostics.traceIdPresent === true;
+}
+
+function quickVerifyTraceProjectionMissing(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
+  if (!quickVerifyTargetTracePresent(targetStatus, traceDiagnostics)) return false;
+  const blocker = stringAtNullable(traceDiagnostics, "blocker");
+  if (blocker === "trace-rows-missing" || blocker === "sample-not-found") return true;
+  if (traceDiagnostics.collectOk !== true) return true;
+  const traceRowCount = numberAtNullable(traceDiagnostics, "traceRowCount");
+  return traceRowCount !== null && traceRowCount === 0;
+}
+
+function quickVerifyTargetStatusKind(targetStatus: Record<string, unknown> | null): "incomplete" | "terminal" | "unknown" {
+  const status = (targetStatus === null ? "" : stringAtNullable(targetStatus, "status") ?? "").toLowerCase();
+  const marks = (targetStatus === null ? "" : stringAtNullable(targetStatus, "marks") ?? "").toLowerCase();
+  if (/running|pending|queued|starting|submitting|processing|in[-_ ]?progress|waiting/u.test(status)) return "incomplete";
+  if (/cancel/u.test(status) || /cancel/u.test(marks)) return "incomplete";
+  if (/completed|succeeded|success|done|failed|failure|error|terminal|command-failed|timeout|stopped|aborted/u.test(status)) return "terminal";
+  return "unknown";
+}
+
+function quickVerifyControlFinding(
+  id: string,
+  titleZh: string,
+  summary: string,
+  rootCause: string,
+  nextAction: string,
+  promptIndex: number,
+  failure: string | null,
+  turnDiagnostics: Record<string, unknown>,
+  traceDiagnostics: Record<string, unknown>,
+  targetStatus: Record<string, unknown> | null,
+  display: Record<string, unknown> | null,
+): Record<string, unknown> {
+  return {
+    id,
+    severity: "red",
+    count: 1,
+    summary,
+    displayTitleZh: titleZh,
+    errorTitleZh: titleZh,
+    timeoutDisplay: display,
+    rootCause,
+    rootCauseStatus: "confirmed",
+    rootCauseConfidence: "high",
+    evidenceSummary: quickVerifyControlEvidenceSummary(turnDiagnostics, traceDiagnostics, targetStatus),
+    nextAction,
+    turnSummaryDiagnostics: turnDiagnostics,
+    traceFrameDiagnostics: traceDiagnostics,
+    failure: failure ?? null,
+    promptIndex,
+    valuesRedacted: true,
+  };
+}
+
 function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null, display: Record<string, unknown> | null = null): Record<string, unknown>[] {
  if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return [];
  const turnDiagnostics = quickVerifyTurnSummaryDiagnostics(promptIndex, turnSummary);
  const traceDiagnostics = quickVerifyTraceFrameDiagnostics(traceFrame);
-  const rendered = [
-    typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "",
-    typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "",
-  ].join("\n");
  const noPromptScenario = promptIndex <= 0;
  if (noPromptScenario && failure === null) return [];
  if (noPromptScenario && failure !== null) {
@@ -1864,29 +2003,135 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number,
      valuesRedacted: true,
    }];
  }
-  const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
-  const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered);
-  if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
-  const displayTitleZh = stringAtNullable(display ?? {}, "titleZh");
-  return [{
-    id: "quick-verify-no-business-turn",
-    severity: "red",
-    count: 1,
-    summary: displayTitleZh ?? "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
-    displayTitleZh,
-    errorTitleZh: displayTitleZh,
-    timeoutDisplay: display,
-    rootCause: `quick verify could not confirm a durable completed turn: turn-summary scopedRows=${String(turnDiagnostics.scopedRowCount ?? 0)} rowCount=${String(turnDiagnostics.rowCount ?? 0)}, traceFrame traceIdPresent=${traceDiagnostics.traceIdPresent === true} finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
-    rootCauseStatus: "confirmed",
-    rootCauseConfidence: "high",
-    evidenceSummary: `turn-summary rows=${String(turnDiagnostics.rowCount ?? 0)} scoped=${String(turnDiagnostics.scopedRowCount ?? 0)} traceFrameTrace=${traceDiagnostics.traceIdPresent === true ? "present" : "missing"} finalResponseBytes=${String(traceDiagnostics.finalResponseBytes ?? "-")}`,
-    nextAction: "Inspect the structured turnSummary/traceFrame diagnostics first; if rows exist with completed non-empty final responses, fix sentinel interpretation instead of treating HWLAB Web as blocked.",
-    turnSummaryDiagnostics: turnDiagnostics,
-    traceFrameDiagnostics: traceDiagnostics,
-    failure: failure ?? null,
+  const rowCount = numberAtNullable(turnDiagnostics, "rowCount") ?? 0;
+  const scopedRowCount = numberAtNullable(turnDiagnostics, "scopedRowCount") ?? 0;
+  const targetStatus = quickVerifyLatestScopedStatus(turnDiagnostics);
+  if (rowCount === 0) {
+    return [quickVerifyControlFinding(
+      "quick-verify-no-business-turn",
+      "快速验证没有采集到业务轮次",
+      "quick verify collected zero business turn rows; this is not a trace/final-response failure.",
+      `turn-summary rowCount=0 for promptIndex=${promptIndex}.`,
+      "Inspect observe command execution and turn-summary collection before checking trace-frame or final response.",
+      promptIndex,
+      failure,
+      turnDiagnostics,
+      traceDiagnostics,
+      targetStatus,
+      display,
+    )];
+  }
+  if (scopedRowCount === 0) {
+    return [quickVerifyControlFinding(
+      "quick-verify-target-turn-missing",
+      "快速验证缺少目标业务轮次",
+      "quick verify collected turn rows, but none matched the target prompt index.",
+      `turn-summary rowCount=${rowCount} but scopedRowCount=0 for promptIndex=${promptIndex}.`,
+      "Check promptIndex/command segmentation and rerun the bounded turn-summary view for the observer.",
+      promptIndex,
+      failure,
+      turnDiagnostics,
+      traceDiagnostics,
+      targetStatus,
+      display,
+    )];
+  }
+
+  const findings: Record<string, unknown>[] = [];
+  if (!quickVerifyTargetTracePresent(targetStatus, traceDiagnostics)) {
+    findings.push(quickVerifyControlFinding(
+      "quick-verify-target-turn-trace-id-missing",
+      "目标业务轮次缺少 traceId",
+      "quick verify found the target business turn, but the turn has no traceId.",
+      `target turn exists for promptIndex=${promptIndex}, but traceIdPresent=false.`,
+      "Fix Workbench/Code Agent traceId projection for the target turn before inspecting trace rows.",
+      promptIndex,
+      failure,
+      turnDiagnostics,
+      traceDiagnostics,
+      targetStatus,
+      display,
+    ));
+    return findings;
+  }
+
+  if (quickVerifyTraceProjectionMissing(targetStatus, traceDiagnostics)) {
+    findings.push(quickVerifyControlFinding(
+      "quick-verify-trace-projection-missing",
+      "目标 trace 投影缺失",
+      "quick verify found a target turn traceId, but trace-frame did not contain trace rows/projection for it.",
+      `target turn has traceId, but trace-frame collectOk=${traceDiagnostics.collectOk === true} blocker=${stringAtNullable(traceDiagnostics, "blocker") ?? "-"} traceRowCount=${String(traceDiagnostics.traceRowCount ?? "-")}.`,
+      "Inspect trace-frame for the target traceId, then check Workbench trace projection/read-model hydration before treating the run as a business failure.",
+      promptIndex,
+      failure,
+      turnDiagnostics,
+      traceDiagnostics,
+      targetStatus,
+      display,
+    ));
+  }
+
+  if (quickVerifyFinalResponseEmpty(targetStatus, traceDiagnostics)) {
+    const statusKind = quickVerifyTargetStatusKind(targetStatus);
+    if (statusKind === "incomplete") {
+      findings.push(quickVerifyControlFinding(
+        "quick-verify-final-response-empty-incomplete",
+        "目标轮次未完成且最终回复为空",
+        "quick verify found the target business turn, but it is still incomplete/cancel-marked and Final Response is empty.",
+        `target turn status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} marks=${stringAtNullable(targetStatus ?? {}, "marks") ?? "-"} finalResponseEmpty=true.`,
+        "Inspect the same traceId in Workbench and Code Agent runtime; this is an unfinished/cancelled turn, not a zero-dialog case.",
+        promptIndex,
+        failure,
+        turnDiagnostics,
+        traceDiagnostics,
+        targetStatus,
+        display,
+      ));
+    } else if (statusKind === "terminal") {
+      findings.push(quickVerifyControlFinding(
+        "quick-verify-final-response-empty-terminal",
+        "目标轮次已终止但最终回复为空",
+        "quick verify found a terminal target business turn, but Final Response is empty.",
+        `target turn terminal status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} finalResponseEmpty=true.`,
+        "Inspect terminal event persistence and final-response projection for the target traceId; do not classify this as no dialog.",
+        promptIndex,
+        failure,
+        turnDiagnostics,
+        traceDiagnostics,
+        targetStatus,
+        display,
+      ));
+    } else {
+      findings.push(quickVerifyControlFinding(
+        "quick-verify-final-response-empty-status-unknown",
+        "目标轮次状态未知且最终回复为空",
+        "quick verify found the target business turn and empty Final Response, but the turn status is not classifiable.",
+        `target turn status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} finalResponseEmpty=true.`,
+        "Fix the turn-summary status projection so quick verify can classify incomplete versus terminal final-response failures.",
+        promptIndex,
+        failure,
+        turnDiagnostics,
+        traceDiagnostics,
+        targetStatus,
+        display,
+      ));
+    }
+  }
+
+  if (findings.length > 0) return findings;
+  return [quickVerifyControlFinding(
+    "quick-verify-diagnostics-inconclusive",
+    "快速验证诊断信息不足",
+    "quick verify did not prove a durable completed turn, but structured diagnostics did not match any specific failure code.",
+    `quick verify has rowCount=${rowCount} scopedRowCount=${scopedRowCount}, traceIdPresent=${traceDiagnostics.traceIdPresent === true}, finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
+    "Improve turn-summary/trace-frame diagnostics before making a business recovery decision.",
    promptIndex,
-    valuesRedacted: true,
-  }];
+    failure,
+    turnDiagnostics,
+    traceDiagnostics,
+    targetStatus,
+    display,
+  )];
 }

 function enrichObserveStartFailureFinding(finding: Record<string, unknown>, result: Record<string, unknown>): Record<string, unknown> {
@@ -2009,6 +2254,7 @@ function quickVerifyTurnSummaryDiagnostics(promptIndex: number, turnSummary: Rec
    return {
      round: numberAtNullable(row, "round"),
      status: stringAtNullable(row, "status"),
+      marks: stringAtNullable(row, "marks"),
      traceIdPresent: stringAtNullable(row, "traceId") !== null,
      finalResponseEmpty: finalResponse.empty === true,
      finalResponseBytes: numberAtNullable(finalResponse, "textBytes"),