fix: split quick verify control findings

This commit is contained in:
Codex
2026-07-02 07:33:19 +00:00
parent a5644314cd
commit 1c9132e7c6
5 changed files with 403 additions and 33 deletions
+2
View File
@@ -19,6 +19,7 @@ description: UniDesk monitoring and Web sentinel operations. Use when working on
- 诊断可用 `curl` 或一次性 `web-probe script` 采证,但重复 dashboard 验证必须沉淀为受控 `web-probe sentinel dashboard verify|screenshot` 或等价入口。
- `web-probe sentinel dashboard screenshot` 必须作为远程浏览器截图入口使用,PNG 默认下载到调用者 `/tmp`issue/PR 证据引用 `localPath``sha256`、HTTP status、DOM 摘要和 overflow 结果。`VERIFIED=true` 只证明 PNG 回传和哈希校验通过,收口前仍必须打开截图或用 DOM 摘要确认不是 Chrome 网络错误页、登录页或空壳页。
- monitor-web 的“监测项”默认必须跟随选中 run;曲线点、运行详情和监测项摘要必须区分类型数与样本数,历史聚合只能作为明确标注的历史口径展示。
- Web 哨兵 check code 必须语义单一且确定:一个 code/id 只能表达一种处置路径;如果同一 finding 可能表示多种根因或状态,必须拆成多个固定 code/id,而不是用动态标题或摘要在同一 code 下区分。
## Quick Commands
@@ -66,6 +67,7 @@ bun scripts/cli.ts web-probe observe analyze <observerId>
12. Any new analyzer finding id emitted by quick verify must be registered in the selected check catalog before rollout. A missing catalog entry can make `/api/health` return 503 and leave the new runner pod unhealthy even when the image is otherwise correct.
13. If a dashboard screenshot artifact is small or visually shows `ERR_NETWORK_CHANGED`/browser error chrome while CLI status is otherwise pass, discard it as evidence and rerun after checking the public URL/API status. Treat this as a web-probe evidence-quality issue if repeated; do not close visibility issues from such a screenshot alone.
14. Request-rate curve acceptance uses `/api/runs/{id}.requestRate` plus dashboard screenshot/DOM evidence that the request chart is above the memory chart with aligned time axis. Until `dashboard verify` exposes request-rate-specific fields, do not treat legacy `API_PAGES` / `API_SAMPLES` columns as request curve counts; see `docs/reference/observability.md`.
15. Check code 设计必须先拆语义再实现展示:例如“没有业务轮次”“目标轮次缺 traceId”“trace rows/projection 缺失”“Final Response 为空且仍在运行/取消”“Final Response 为空且已失败/终止”应是不同固定 code,而不是一个 WBC-003 下的动态解释。
## Architecture Preference
@@ -38,8 +38,8 @@ sentinel:
- code: WBC-003
id: quick-verify-no-business-turn
level: error
titleZh: 快速验证没有有效对话
summaryZh: 快速验证没有看到稳定会话、过程记录或最终回复
titleZh: 快速验证没有采集到业务轮次
summaryZh: 快速验证的 turn-summary 没有任何业务轮次记录
actionZh: 查看详情后处理。
blocking: true
order: 30
@@ -867,3 +867,59 @@ sentinel:
actionZh: 先修复性能探针采集可见性,再继续性能结论。
blocking: false
order: 1060
- code: WBC-107
id: quick-verify-target-turn-missing
level: error
titleZh: 快速验证缺少目标业务轮次
summaryZh: 快速验证采集到了业务轮次,但没有采集到目标 prompt 对应的轮次。
actionZh: 检查 promptIndex、command 分段和 turn-summary 采集。
blocking: true
order: 1070
- code: WBC-108
id: quick-verify-target-turn-trace-id-missing
level: error
titleZh: 目标业务轮次缺少 traceId
summaryZh: 目标业务轮次存在,但缺少可定位过程记录的 traceId。
actionZh: 检查 Workbench/Code Agent 的 traceId 投影和消息绑定。
blocking: true
order: 1080
- code: WBC-109
id: quick-verify-trace-projection-missing
level: error
titleZh: 目标 trace 投影缺失
summaryZh: 目标业务轮次已有 traceId,但 trace-frame 没有对应 trace rows 或投影。
actionZh: 检查 trace-frame、Workbench trace read-model 和投影水位。
blocking: true
order: 1090
- code: WBC-110
id: quick-verify-final-response-empty-incomplete
level: error
titleZh: 目标轮次未完成且最终回复为空
summaryZh: 目标业务轮次仍在运行或已取消标记,Final Response 为空。
actionZh: 检查同一 traceId 的运行状态、取消事件和 Code Agent 终态。
blocking: true
order: 1100
- code: WBC-111
id: quick-verify-final-response-empty-terminal
level: error
titleZh: 目标轮次已终止但最终回复为空
summaryZh: 目标业务轮次已进入终态,但 Final Response 为空。
actionZh: 检查终态事件持久化和最终回复投影。
blocking: true
order: 1110
- code: WBC-112
id: quick-verify-final-response-empty-status-unknown
level: error
titleZh: 目标轮次状态未知且最终回复为空
summaryZh: 目标业务轮次存在且 Final Response 为空,但状态字段无法判定为运行中或终态。
actionZh: 补齐 turn-summary 状态投影后重新分类。
blocking: true
order: 1120
- code: WBC-113
id: quick-verify-diagnostics-inconclusive
level: error
titleZh: 快速验证诊断信息不足
summaryZh: 快速验证不能证明业务轮次完成,但结构化诊断不足以归入具体失败语义。
actionZh: 先改进 turn-summary/trace-frame 诊断字段,再判断业务恢复。
blocking: true
order: 1130
+2
View File
@@ -35,6 +35,8 @@ Web 哨兵 dashboard/API 展示问题的第一事实源是 sentinel runner 的 `
Web 哨兵 findings 可见性要同时核对 runner API 和已有 observe artifact。若某个 run 的公开 `/api/report?view=findings&run=<id>` 只显示 WBC-003,但 `web-probe sentinel report --run <id> --view findings --raw` 能从 `analysis/report.json` 读出 red/amber analyzer findings,根因是索引或 artifact 可见性遮盖,不是业务没有产生 warning/error。此时应回填或重建这条既有 run 的 report index,并保留原有 report views;不要通过启动新的哨兵 run 来解释旧记录。
Web 哨兵 check code 是排障和验收合同,必须保持语义单一且确定。一个 code/id 只能对应一种可执行处置路径;如果一个 finding 可能同时表示“没有采集到业务轮次”“目标轮次缺 traceId”“trace rows/projection 缺失”“Final Response 为空但仍在运行/取消”“Final Response 为空且已失败/终止”等多种状态,就必须拆成多个固定 code/id。不得用同一个 code 的动态标题、动态 summary 或 rootCause 文案来承载多种互斥语义;动态字段只能补充证据,不能改变 code 的语义。
Web 哨兵请求频率曲线的验收事实源是 runner `/api/runs/{id}.requestRate`、已有 observe artifact 中的 request-rate summary,以及 `web-probe sentinel dashboard screenshot` 的远程浏览器证据。阈值、采样间隔、bucket 大小和红黄线只从 YAML/source-of-truth 读取,长期文档只记录字段族与验证入口。验收时应核对 `bucketSeconds`、总请求曲线、页面曲线、API path 曲线、峰值每分钟计数、数据来源和 chart/DOM 是否显示在内存曲线上方并共享时间轴。若 quick-verify 的业务链路失败,但同一 run 的 `requestRate` API 和截图已经有曲线数据,应把请求频率能力验收与业务阻塞分开记录;反之,`requestRate.source=unavailable` 或曲线为空时要继续检查 analyzer compact 输出、artifact summary、索引回填和 report fallback。除非 `dashboard verify` 已显式输出 request-rate 专用字段,不得把旧的 `API_PAGES` / `API_SAMPLES` 列当作请求频率曲线验收结果。
## Workbench Request Storm And Freeze
+274 -28
View File
@@ -342,6 +342,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
);
const ok = analysisReadable && controlFindings.length === 0 && blockingFindings.length === 0;
const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds);
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), analysisReadable, findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
return recordQuickVerify(state, {
ok,
@@ -356,7 +357,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
findingCount: findings.length,
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
failure: !analysisReadable ? "quick-verify-analysis-missing" : controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null,
failure: !analysisReadable ? "quick-verify-analysis-missing" : primaryControlFailure ?? (blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null),
promptSource: prompts.summary,
accountEnv: accountEnv.summary,
steps: [...steps, cleanupStep],
@@ -481,6 +482,44 @@ function inlinePromptText(item: Record<string, unknown>): string | null {
return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value");
}
export function reclassifyQuickVerifyControlFindings(state: SentinelCicdState, input: {
readonly runId: string | null;
readonly scenarioId: string | null;
readonly observerId: string | null;
readonly failure?: string | null;
readonly timeoutSeconds?: number | null;
}): Record<string, unknown> {
if (input.observerId === null || input.observerId.length === 0) {
return { ok: false, reason: "observer-id-missing", runId: input.runId, valuesRedacted: true };
}
const scenarioId = input.scenarioId ?? stringAt(state.cicd, "targetValidation.scenarioId");
const scenario = findScenario(state, scenarioId);
if (scenario === null) {
return { ok: false, reason: "scenario-not-found", runId: input.runId, scenarioId, observerId: input.observerId, valuesRedacted: true };
}
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
const promptIndex = commandSequence.reduce((count, item) => {
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
return stringAtNullable(item, "type") === "sendPrompt" ? count + repeat : count;
}, 0);
const timeoutSeconds = Math.max(5, Math.min(Math.trunc(input.timeoutSeconds ?? 55), 55));
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, timeoutSeconds);
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, timeoutSeconds);
const findings = quickVerifyControlFindings(input.failure ?? null, promptIndex, turnSummary, traceFrame);
return {
ok: true,
runId: input.runId,
scenarioId,
observerId: input.observerId,
promptIndex,
findingCount: findings.length,
findings,
turnSummary: { ok: turnSummary.ok === true, collectShape: turnSummary.collectShape, valuesRedacted: true },
traceFrame: { ok: traceFrame.ok === true, collectShape: traceFrame.collectShape, valuesRedacted: true },
valuesRedacted: true,
};
}
function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
readonly runId: string;
readonly scenarioId: string;
@@ -542,6 +581,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
&& controlFindings.length === 0
&& blockingFindings.length === 0;
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds);
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
return {
ok: recoveredWaitFailure,
runId: input.runId,
@@ -558,7 +598,8 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
findingCount: findings.length,
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
failure: recoveredWaitFailure ? null : input.failure,
failure: recoveredWaitFailure ? null : primaryControlFailure ?? input.failure,
runnerFailure: recoveredWaitFailure ? null : input.failure,
promptSource: input.promptSource,
steps: [...input.steps, ...cleanupSteps],
analysis: artifactSummary,
@@ -1811,6 +1852,13 @@ function isQuickVerifyBlockingFinding(item: Record<string, unknown>): boolean {
if (id === "observer-command-failed") return observerCommandFailureBlocks(item);
return [
"quick-verify-no-business-turn",
"quick-verify-target-turn-missing",
"quick-verify-target-turn-trace-id-missing",
"quick-verify-trace-projection-missing",
"quick-verify-final-response-empty-incomplete",
"quick-verify-final-response-empty-terminal",
"quick-verify-final-response-empty-status-unknown",
"quick-verify-diagnostics-inconclusive",
"quick-verify-analysis-summary-unreadable",
"quick-verify-command-sequence-failed",
"quick-verify-observer-start-failed",
@@ -1836,14 +1884,105 @@ function observerCommandFailureBlocks(item: Record<string, unknown>): boolean {
});
}
function quickVerifyPrimaryControlFailure(controlFindings: Record<string, unknown>[]): string | null {
const first = controlFindings[0];
return first === undefined ? null : stringAtNullable(first, "id") ?? stringAtNullable(first, "kind") ?? stringAtNullable(first, "code");
}
function quickVerifyControlEvidenceSummary(turnDiagnostics: Record<string, unknown>, traceDiagnostics: Record<string, unknown>, targetStatus: Record<string, unknown> | null): string {
const targetStatusText = targetStatus === null ? "-" : stringAtNullable(targetStatus, "status") ?? "-";
const targetMarks = targetStatus === null ? "-" : stringAtNullable(targetStatus, "marks") ?? "-";
const targetTrace = targetStatus === null ? "-" : targetStatus.traceIdPresent === true ? "present" : "missing";
const targetFinalBytes = targetStatus === null ? "-" : String(targetStatus.finalResponseBytes ?? "-");
const traceBlocker = stringAtNullable(traceDiagnostics, "blocker") ?? "-";
const traceRowCount = traceDiagnostics.traceRowCount === null || traceDiagnostics.traceRowCount === undefined ? "-" : String(traceDiagnostics.traceRowCount);
return [
`turn-summary rows=${String(turnDiagnostics.rowCount ?? 0)}`,
`scoped=${String(turnDiagnostics.scopedRowCount ?? 0)}`,
`targetStatus=${targetStatusText}`,
`targetMarks=${targetMarks}`,
`targetTrace=${targetTrace}`,
`targetFinalBytes=${targetFinalBytes}`,
`traceFrameTrace=${traceDiagnostics.traceIdPresent === true ? "present" : "missing"}`,
`traceFrameBlocker=${traceBlocker}`,
`traceRows=${traceRowCount}`,
`traceFinalBytes=${String(traceDiagnostics.finalResponseBytes ?? "-")}`,
].join(" ");
}
function quickVerifyScopedStatuses(turnDiagnostics: Record<string, unknown>): Record<string, unknown>[] {
return Array.isArray(turnDiagnostics.scopedStatuses) ? turnDiagnostics.scopedStatuses.map(record) : [];
}
function quickVerifyLatestScopedStatus(turnDiagnostics: Record<string, unknown>): Record<string, unknown> | null {
const rows = quickVerifyScopedStatuses(turnDiagnostics);
return rows.length === 0 ? null : rows[rows.length - 1] ?? null;
}
function quickVerifyFinalResponseEmpty(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
return targetStatus?.finalResponseEmpty === true || traceDiagnostics.finalResponseEmpty === true;
}
function quickVerifyTargetTracePresent(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
return targetStatus?.traceIdPresent === true || traceDiagnostics.traceIdPresent === true;
}
function quickVerifyTraceProjectionMissing(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
if (!quickVerifyTargetTracePresent(targetStatus, traceDiagnostics)) return false;
const blocker = stringAtNullable(traceDiagnostics, "blocker");
if (blocker === "trace-rows-missing" || blocker === "sample-not-found") return true;
if (traceDiagnostics.collectOk !== true) return true;
const traceRowCount = numberAtNullable(traceDiagnostics, "traceRowCount");
return traceRowCount !== null && traceRowCount === 0;
}
function quickVerifyTargetStatusKind(targetStatus: Record<string, unknown> | null): "incomplete" | "terminal" | "unknown" {
const status = (targetStatus === null ? "" : stringAtNullable(targetStatus, "status") ?? "").toLowerCase();
const marks = (targetStatus === null ? "" : stringAtNullable(targetStatus, "marks") ?? "").toLowerCase();
if (/running|pending|queued|starting|submitting|processing|in[-_ ]?progress|waiting/u.test(status)) return "incomplete";
if (/cancel/u.test(status) || /cancel/u.test(marks)) return "incomplete";
if (/completed|succeeded|success|done|failed|failure|error|terminal|command-failed|timeout|stopped|aborted/u.test(status)) return "terminal";
return "unknown";
}
function quickVerifyControlFinding(
id: string,
titleZh: string,
summary: string,
rootCause: string,
nextAction: string,
promptIndex: number,
failure: string | null,
turnDiagnostics: Record<string, unknown>,
traceDiagnostics: Record<string, unknown>,
targetStatus: Record<string, unknown> | null,
display: Record<string, unknown> | null,
): Record<string, unknown> {
return {
id,
severity: "red",
count: 1,
summary,
displayTitleZh: titleZh,
errorTitleZh: titleZh,
timeoutDisplay: display,
rootCause,
rootCauseStatus: "confirmed",
rootCauseConfidence: "high",
evidenceSummary: quickVerifyControlEvidenceSummary(turnDiagnostics, traceDiagnostics, targetStatus),
nextAction,
turnSummaryDiagnostics: turnDiagnostics,
traceFrameDiagnostics: traceDiagnostics,
failure: failure ?? null,
promptIndex,
valuesRedacted: true,
};
}
function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null, display: Record<string, unknown> | null = null): Record<string, unknown>[] {
if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return [];
const turnDiagnostics = quickVerifyTurnSummaryDiagnostics(promptIndex, turnSummary);
const traceDiagnostics = quickVerifyTraceFrameDiagnostics(traceFrame);
const rendered = [
typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "",
typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "",
].join("\n");
const noPromptScenario = promptIndex <= 0;
if (noPromptScenario && failure === null) return [];
if (noPromptScenario && failure !== null) {
@@ -1864,29 +2003,135 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number,
valuesRedacted: true,
}];
}
const noTrace = /\s*sendPrompt|no\s+sendPrompt|\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
const emptyFinal = /Final Response[\s\S]*\(\)/iu.test(rendered);
if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
const displayTitleZh = stringAtNullable(display ?? {}, "titleZh");
return [{
id: "quick-verify-no-business-turn",
severity: "red",
count: 1,
summary: displayTitleZh ?? "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
displayTitleZh,
errorTitleZh: displayTitleZh,
timeoutDisplay: display,
rootCause: `quick verify could not confirm a durable completed turn: turn-summary scopedRows=${String(turnDiagnostics.scopedRowCount ?? 0)} rowCount=${String(turnDiagnostics.rowCount ?? 0)}, traceFrame traceIdPresent=${traceDiagnostics.traceIdPresent === true} finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
rootCauseStatus: "confirmed",
rootCauseConfidence: "high",
evidenceSummary: `turn-summary rows=${String(turnDiagnostics.rowCount ?? 0)} scoped=${String(turnDiagnostics.scopedRowCount ?? 0)} traceFrameTrace=${traceDiagnostics.traceIdPresent === true ? "present" : "missing"} finalResponseBytes=${String(traceDiagnostics.finalResponseBytes ?? "-")}`,
nextAction: "Inspect the structured turnSummary/traceFrame diagnostics first; if rows exist with completed non-empty final responses, fix sentinel interpretation instead of treating HWLAB Web as blocked.",
turnSummaryDiagnostics: turnDiagnostics,
traceFrameDiagnostics: traceDiagnostics,
failure: failure ?? null,
const rowCount = numberAtNullable(turnDiagnostics, "rowCount") ?? 0;
const scopedRowCount = numberAtNullable(turnDiagnostics, "scopedRowCount") ?? 0;
const targetStatus = quickVerifyLatestScopedStatus(turnDiagnostics);
if (rowCount === 0) {
return [quickVerifyControlFinding(
"quick-verify-no-business-turn",
"快速验证没有采集到业务轮次",
"quick verify collected zero business turn rows; this is not a trace/final-response failure.",
`turn-summary rowCount=0 for promptIndex=${promptIndex}.`,
"Inspect observe command execution and turn-summary collection before checking trace-frame or final response.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
)];
}
if (scopedRowCount === 0) {
return [quickVerifyControlFinding(
"quick-verify-target-turn-missing",
"快速验证缺少目标业务轮次",
"quick verify collected turn rows, but none matched the target prompt index.",
`turn-summary rowCount=${rowCount} but scopedRowCount=0 for promptIndex=${promptIndex}.`,
"Check promptIndex/command segmentation and rerun the bounded turn-summary view for the observer.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
)];
}
const findings: Record<string, unknown>[] = [];
if (!quickVerifyTargetTracePresent(targetStatus, traceDiagnostics)) {
findings.push(quickVerifyControlFinding(
"quick-verify-target-turn-trace-id-missing",
"目标业务轮次缺少 traceId",
"quick verify found the target business turn, but the turn has no traceId.",
`target turn exists for promptIndex=${promptIndex}, but traceIdPresent=false.`,
"Fix Workbench/Code Agent traceId projection for the target turn before inspecting trace rows.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
));
return findings;
}
if (quickVerifyTraceProjectionMissing(targetStatus, traceDiagnostics)) {
findings.push(quickVerifyControlFinding(
"quick-verify-trace-projection-missing",
"目标 trace 投影缺失",
"quick verify found a target turn traceId, but trace-frame did not contain trace rows/projection for it.",
`target turn has traceId, but trace-frame collectOk=${traceDiagnostics.collectOk === true} blocker=${stringAtNullable(traceDiagnostics, "blocker") ?? "-"} traceRowCount=${String(traceDiagnostics.traceRowCount ?? "-")}.`,
"Inspect trace-frame for the target traceId, then check Workbench trace projection/read-model hydration before treating the run as a business failure.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
));
}
if (quickVerifyFinalResponseEmpty(targetStatus, traceDiagnostics)) {
const statusKind = quickVerifyTargetStatusKind(targetStatus);
if (statusKind === "incomplete") {
findings.push(quickVerifyControlFinding(
"quick-verify-final-response-empty-incomplete",
"目标轮次未完成且最终回复为空",
"quick verify found the target business turn, but it is still incomplete/cancel-marked and Final Response is empty.",
`target turn status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} marks=${stringAtNullable(targetStatus ?? {}, "marks") ?? "-"} finalResponseEmpty=true.`,
"Inspect the same traceId in Workbench and Code Agent runtime; this is an unfinished/cancelled turn, not a zero-dialog case.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
));
} else if (statusKind === "terminal") {
findings.push(quickVerifyControlFinding(
"quick-verify-final-response-empty-terminal",
"目标轮次已终止但最终回复为空",
"quick verify found a terminal target business turn, but Final Response is empty.",
`target turn terminal status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} finalResponseEmpty=true.`,
"Inspect terminal event persistence and final-response projection for the target traceId; do not classify this as no dialog.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
));
} else {
findings.push(quickVerifyControlFinding(
"quick-verify-final-response-empty-status-unknown",
"目标轮次状态未知且最终回复为空",
"quick verify found the target business turn and empty Final Response, but the turn status is not classifiable.",
`target turn status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} finalResponseEmpty=true.`,
"Fix the turn-summary status projection so quick verify can classify incomplete versus terminal final-response failures.",
promptIndex,
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
));
}
}
if (findings.length > 0) return findings;
return [quickVerifyControlFinding(
"quick-verify-diagnostics-inconclusive",
"快速验证诊断信息不足",
"quick verify did not prove a durable completed turn, but structured diagnostics did not match any specific failure code.",
`quick verify has rowCount=${rowCount} scopedRowCount=${scopedRowCount}, traceIdPresent=${traceDiagnostics.traceIdPresent === true}, finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
"Improve turn-summary/trace-frame diagnostics before making a business recovery decision.",
promptIndex,
valuesRedacted: true,
}];
failure,
turnDiagnostics,
traceDiagnostics,
targetStatus,
display,
)];
}
function enrichObserveStartFailureFinding(finding: Record<string, unknown>, result: Record<string, unknown>): Record<string, unknown> {
@@ -2009,6 +2254,7 @@ function quickVerifyTurnSummaryDiagnostics(promptIndex: number, turnSummary: Rec
return {
round: numberAtNullable(row, "round"),
status: stringAtNullable(row, "status"),
marks: stringAtNullable(row, "marks"),
traceIdPresent: stringAtNullable(row, "traceId") !== null,
finalResponseEmpty: finalResponse.empty === true,
finalResponseBytes: numberAtNullable(finalResponse, "textBytes"),
+67 -3
View File
@@ -8,6 +8,7 @@ import { repoRoot } from "./config";
import { startJob } from "./jobs";
import type { RenderedCliResult } from "./output";
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
import {
clipTail,
@@ -32,7 +33,7 @@ import {
text,
withWarnings,
} from "./hwlab-node-web-sentinel-cicd";
import { metricNames, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55;
@@ -264,6 +265,7 @@ function compactSentinelReportRawPayload(
const findings = Array.isArray(body.findings) ? body.findings.map(record) : [];
const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : [];
const visibleFindings = mergeSentinelReportFindings(findings, artifactFindings);
const offlineReclassify = offlineQuickVerifyReclassify(state, run, visibleFindings);
const storedFindingCount = numberAtNullable(run, "finding_count") ?? numberAtNullable(run, "findingCount") ?? findings.length;
const artifactFindingCount = numberAtNullable(artifact, "findingCount") ?? artifactFindings.length;
const visibleFindingCount = Math.max(storedFindingCount, artifactFindingCount, visibleFindings.length);
@@ -306,6 +308,7 @@ function compactSentinelReportRawPayload(
},
summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]),
findings: visibleFindings.slice(0, 12).map(compactSentinelReportFinding),
offlineReclassify,
artifactSummary: Object.keys(artifact).length === 0 ? null : {
ok: artifact.ok === true,
reason: artifact.reason ?? null,
@@ -333,6 +336,53 @@ function compactSentinelReportRawPayload(
};
}
function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record<string, unknown>, findings: readonly Record<string, unknown>[]): Record<string, unknown> | null {
const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("quick-verify-no-business-turn"));
if (!hasLegacyQuickVerifyControl) return null;
const catalog = sentinelReportCheckCatalogById(state);
const result = reclassifyQuickVerifyControlFindings(state, {
runId: stringAtNullable(run, "id"),
scenarioId: stringAtNullable(run, "scenario_id") ?? stringAtNullable(run, "scenarioId"),
observerId: stringAtNullable(run, "observer_id") ?? stringAtNullable(run, "observerId"),
failure: stringAtNullable(record(run.summary), "failure"),
timeoutSeconds: SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS,
});
const resultFindings = Array.isArray(result.findings) ? result.findings.map(record) : [];
return {
...pickFields(result, ["ok", "reason", "runId", "scenarioId", "observerId", "promptIndex", "findingCount", "turnSummary", "traceFrame", "valuesRedacted"]),
source: "offline-existing-observe-artifact",
note: "This is a local CLI reclassification of existing turn-summary/trace-frame artifacts; it does not mutate the runner index.",
findings: resultFindings.slice(0, 8).map((item) => compactSentinelReportFinding(enrichSentinelReportFindingWithCatalog(item, catalog))),
valuesRedacted: true,
};
}
function sentinelReportCheckCatalogById(state: SentinelCicdState): Map<string, Record<string, unknown>> {
try {
const reportViews = record(readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.reportViews));
const catalogRef = stringAtNullable(reportViews, "checkCatalogRef");
if (catalogRef === null) return new Map();
const catalog = record(readWebProbeSentinelConfigRefTarget(state.spec, catalogRef));
const items = Array.isArray(catalog.items) ? catalog.items.map(record) : Array.isArray(catalog.checks) ? catalog.checks.map(record) : [];
return new Map(items.map((item) => [stringAtNullable(item, "id") ?? "", item]).filter((item): item is [string, Record<string, unknown>] => item[0].length > 0));
} catch {
return new Map();
}
}
function enrichSentinelReportFindingWithCatalog(item: Record<string, unknown>, catalog: ReadonlyMap<string, Record<string, unknown>>): Record<string, unknown> {
const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code");
const check = id === null ? null : catalog.get(id) ?? null;
if (check === null) return item;
return {
...item,
check,
checkCode: stringAtNullable(check, "code"),
checkTitleZh: stringAtNullable(check, "titleZh"),
valuesRedacted: true,
};
}
function mergeSentinelReportFindings(primary: readonly Record<string, unknown>[], artifact: readonly Record<string, unknown>[]): Record<string, unknown>[] {
const merged: Record<string, unknown>[] = [];
const seen = new Set<string>();
@@ -423,8 +473,12 @@ function compactSentinelReportFinding(value: Record<string, unknown>): Record<st
const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals);
if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals;
const check = record(value.check);
const checkCodeFromValue = stringAtNullable(value, "checkCode");
const checkTitleFromValue = stringAtNullable(value, "checkTitleZh");
const checkCode = stringAtNullable(check, "code");
const checkTitle = stringAtNullable(check, "titleZh");
if (checkCodeFromValue !== null) result.checkCode = checkCodeFromValue;
if (checkTitleFromValue !== null) result.checkTitleZh = checkTitleFromValue;
if (checkCode !== null || checkTitle !== null) {
result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]);
}
@@ -471,7 +525,9 @@ function renderSentinelReportSummary(payload: Record<string, unknown>, state: Se
const run = record(payload.run);
const summary = record(payload.summary);
const artifactSummary = record(payload.artifactSummary);
const offlineReclassify = record(payload.offlineReclassify);
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
const findingCount = run.findingCount ?? findings.length;
const analyzerArtifactCount = numberAtNullable(artifactSummary, "artifactCount") ?? numberAtNullable(record(artifactSummary.counts), "artifacts");
@@ -498,26 +554,34 @@ function renderSentinelReportSummary(payload: Record<string, unknown>, state: Se
"",
"Findings",
findingRows,
offlineFindings.length === 0 ? "" : "",
offlineFindings.length === 0 ? "" : "Offline Reclassify",
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
].join("\n");
}
function renderSentinelReportFindings(payload: Record<string, unknown>): string {
const run = record(payload.run);
const artifactSummary = record(payload.artifactSummary);
const offlineReclassify = record(payload.offlineReclassify);
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
return [
"Web Probe Sentinel Findings",
"=======================================================",
`run=${run.id ?? "-"} report=${reportSha ?? "-"} findings=${run.findingCount ?? findings.length}`,
findings.length === 0 ? "-" : findings.map(formatSentinelReportFindingLine).join("\n"),
offlineFindings.length === 0 ? "" : "",
offlineFindings.length === 0 ? "" : "Offline Reclassify",
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
].join("\n");
}
function formatSentinelReportFindingLine(item: Record<string, unknown>): string {
const check = record(item.check);
const code = stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-";
const title = stringAtNullable(check, "titleZh") ?? "";
const code = stringAtNullable(item, "checkCode") ?? stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-";
const title = stringAtNullable(item, "checkTitleZh") ?? stringAtNullable(check, "titleZh") ?? "";
const summary = reportText(item.summary, 180) ?? "";
const rootCause = reportText(item.rootCause, 180);
const evidence = reportText(item.evidenceSummary, 180);