fix: split quick verify control findings
This commit is contained in:
@@ -19,6 +19,7 @@ description: UniDesk monitoring and Web sentinel operations. Use when working on
|
||||
- 诊断可用 `curl` 或一次性 `web-probe script` 采证,但重复 dashboard 验证必须沉淀为受控 `web-probe sentinel dashboard verify|screenshot` 或等价入口。
|
||||
- `web-probe sentinel dashboard screenshot` 必须作为远程浏览器截图入口使用,PNG 默认下载到调用者 `/tmp`;issue/PR 证据引用 `localPath`、`sha256`、HTTP status、DOM 摘要和 overflow 结果。`VERIFIED=true` 只证明 PNG 回传和哈希校验通过,收口前仍必须打开截图或用 DOM 摘要确认不是 Chrome 网络错误页、登录页或空壳页。
|
||||
- monitor-web 的“监测项”默认必须跟随选中 run;曲线点、运行详情和监测项摘要必须区分类型数与样本数,历史聚合只能作为明确标注的历史口径展示。
|
||||
- Web 哨兵 check code 必须语义单一且确定:一个 code/id 只能表达一种处置路径;如果同一 finding 可能表示多种根因或状态,必须拆成多个固定 code/id,而不是用动态标题或摘要在同一 code 下区分。
|
||||
|
||||
## Quick Commands
|
||||
|
||||
@@ -66,6 +67,7 @@ bun scripts/cli.ts web-probe observe analyze <observerId>
|
||||
12. Any new analyzer finding id emitted by quick verify must be registered in the selected check catalog before rollout. A missing catalog entry can make `/api/health` return 503 and leave the new runner pod unhealthy even when the image is otherwise correct.
|
||||
13. If a dashboard screenshot artifact is small or visually shows `ERR_NETWORK_CHANGED`/browser error chrome while CLI status is otherwise pass, discard it as evidence and rerun after checking the public URL/API status. Treat this as a web-probe evidence-quality issue if repeated; do not close visibility issues from such a screenshot alone.
|
||||
14. Request-rate curve acceptance uses `/api/runs/{id}.requestRate` plus dashboard screenshot/DOM evidence that the request chart is above the memory chart with aligned time axis. Until `dashboard verify` exposes request-rate-specific fields, do not treat legacy `API_PAGES` / `API_SAMPLES` columns as request curve counts; see `docs/reference/observability.md`.
|
||||
15. Check code 设计必须先拆语义再实现展示:例如“没有业务轮次”“目标轮次缺 traceId”“trace rows/projection 缺失”“Final Response 为空且仍在运行/取消”“Final Response 为空且已失败/终止”应是不同固定 code,而不是一个 WBC-003 下的动态解释。
|
||||
|
||||
## Architecture Preference
|
||||
|
||||
|
||||
@@ -38,8 +38,8 @@ sentinel:
|
||||
- code: WBC-003
|
||||
id: quick-verify-no-business-turn
|
||||
level: error
|
||||
titleZh: 快速验证没有有效对话
|
||||
summaryZh: 快速验证没有看到稳定会话、过程记录或最终回复。
|
||||
titleZh: 快速验证没有采集到业务轮次
|
||||
summaryZh: 快速验证的 turn-summary 没有任何业务轮次记录。
|
||||
actionZh: 查看详情后处理。
|
||||
blocking: true
|
||||
order: 30
|
||||
@@ -867,3 +867,59 @@ sentinel:
|
||||
actionZh: 先修复性能探针采集可见性,再继续性能结论。
|
||||
blocking: false
|
||||
order: 1060
|
||||
- code: WBC-107
|
||||
id: quick-verify-target-turn-missing
|
||||
level: error
|
||||
titleZh: 快速验证缺少目标业务轮次
|
||||
summaryZh: 快速验证采集到了业务轮次,但没有采集到目标 prompt 对应的轮次。
|
||||
actionZh: 检查 promptIndex、command 分段和 turn-summary 采集。
|
||||
blocking: true
|
||||
order: 1070
|
||||
- code: WBC-108
|
||||
id: quick-verify-target-turn-trace-id-missing
|
||||
level: error
|
||||
titleZh: 目标业务轮次缺少 traceId
|
||||
summaryZh: 目标业务轮次存在,但缺少可定位过程记录的 traceId。
|
||||
actionZh: 检查 Workbench/Code Agent 的 traceId 投影和消息绑定。
|
||||
blocking: true
|
||||
order: 1080
|
||||
- code: WBC-109
|
||||
id: quick-verify-trace-projection-missing
|
||||
level: error
|
||||
titleZh: 目标 trace 投影缺失
|
||||
summaryZh: 目标业务轮次已有 traceId,但 trace-frame 没有对应 trace rows 或投影。
|
||||
actionZh: 检查 trace-frame、Workbench trace read-model 和投影水位。
|
||||
blocking: true
|
||||
order: 1090
|
||||
- code: WBC-110
|
||||
id: quick-verify-final-response-empty-incomplete
|
||||
level: error
|
||||
titleZh: 目标轮次未完成且最终回复为空
|
||||
summaryZh: 目标业务轮次仍在运行或已取消标记,Final Response 为空。
|
||||
actionZh: 检查同一 traceId 的运行状态、取消事件和 Code Agent 终态。
|
||||
blocking: true
|
||||
order: 1100
|
||||
- code: WBC-111
|
||||
id: quick-verify-final-response-empty-terminal
|
||||
level: error
|
||||
titleZh: 目标轮次已终止但最终回复为空
|
||||
summaryZh: 目标业务轮次已进入终态,但 Final Response 为空。
|
||||
actionZh: 检查终态事件持久化和最终回复投影。
|
||||
blocking: true
|
||||
order: 1110
|
||||
- code: WBC-112
|
||||
id: quick-verify-final-response-empty-status-unknown
|
||||
level: error
|
||||
titleZh: 目标轮次状态未知且最终回复为空
|
||||
summaryZh: 目标业务轮次存在且 Final Response 为空,但状态字段无法判定为运行中或终态。
|
||||
actionZh: 补齐 turn-summary 状态投影后重新分类。
|
||||
blocking: true
|
||||
order: 1120
|
||||
- code: WBC-113
|
||||
id: quick-verify-diagnostics-inconclusive
|
||||
level: error
|
||||
titleZh: 快速验证诊断信息不足
|
||||
summaryZh: 快速验证不能证明业务轮次完成,但结构化诊断不足以归入具体失败语义。
|
||||
actionZh: 先改进 turn-summary/trace-frame 诊断字段,再判断业务恢复。
|
||||
blocking: true
|
||||
order: 1130
|
||||
|
||||
@@ -35,6 +35,8 @@ Web 哨兵 dashboard/API 展示问题的第一事实源是 sentinel runner 的 `
|
||||
|
||||
Web 哨兵 findings 可见性要同时核对 runner API 和已有 observe artifact。若某个 run 的公开 `/api/report?view=findings&run=<id>` 只显示 WBC-003,但 `web-probe sentinel report --run <id> --view findings --raw` 能从 `analysis/report.json` 读出 red/amber analyzer findings,根因是索引或 artifact 可见性遮盖,不是业务没有产生 warning/error。此时应回填或重建这条既有 run 的 report index,并保留原有 report views;不要通过启动新的哨兵 run 来解释旧记录。
|
||||
|
||||
Web 哨兵 check code 是排障和验收合同,必须保持语义单一且确定。一个 code/id 只能对应一种可执行处置路径;如果一个 finding 可能同时表示“没有采集到业务轮次”“目标轮次缺 traceId”“trace rows/projection 缺失”“Final Response 为空但仍在运行/取消”“Final Response 为空且已失败/终止”等多种状态,就必须拆成多个固定 code/id。不得用同一个 code 的动态标题、动态 summary 或 rootCause 文案来承载多种互斥语义;动态字段只能补充证据,不能改变 code 的语义。
|
||||
|
||||
Web 哨兵请求频率曲线的验收事实源是 runner `/api/runs/{id}.requestRate`、已有 observe artifact 中的 request-rate summary,以及 `web-probe sentinel dashboard screenshot` 的远程浏览器证据。阈值、采样间隔、bucket 大小和红黄线只从 YAML/source-of-truth 读取,长期文档只记录字段族与验证入口。验收时应核对 `bucketSeconds`、总请求曲线、页面曲线、API path 曲线、峰值每分钟计数、数据来源和 chart/DOM 是否显示在内存曲线上方并共享时间轴。若 quick-verify 的业务链路失败,但同一 run 的 `requestRate` API 和截图已经有曲线数据,应把请求频率能力验收与业务阻塞分开记录;反之,`requestRate.source=unavailable` 或曲线为空时要继续检查 analyzer compact 输出、artifact summary、索引回填和 report fallback。除非 `dashboard verify` 已显式输出 request-rate 专用字段,不得把旧的 `API_PAGES` / `API_SAMPLES` 列当作请求频率曲线验收结果。
|
||||
|
||||
## Workbench Request Storm And Freeze
|
||||
|
||||
@@ -342,6 +342,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
|
||||
);
|
||||
const ok = analysisReadable && controlFindings.length === 0 && blockingFindings.length === 0;
|
||||
const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds);
|
||||
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
|
||||
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), analysisReadable, findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
|
||||
return recordQuickVerify(state, {
|
||||
ok,
|
||||
@@ -356,7 +357,7 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string,
|
||||
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
||||
findingCount: findings.length,
|
||||
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
|
||||
failure: !analysisReadable ? "quick-verify-analysis-missing" : controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null,
|
||||
failure: !analysisReadable ? "quick-verify-analysis-missing" : primaryControlFailure ?? (blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null),
|
||||
promptSource: prompts.summary,
|
||||
accountEnv: accountEnv.summary,
|
||||
steps: [...steps, cleanupStep],
|
||||
@@ -481,6 +482,44 @@ function inlinePromptText(item: Record<string, unknown>): string | null {
|
||||
return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value");
|
||||
}
|
||||
|
||||
export function reclassifyQuickVerifyControlFindings(state: SentinelCicdState, input: {
|
||||
readonly runId: string | null;
|
||||
readonly scenarioId: string | null;
|
||||
readonly observerId: string | null;
|
||||
readonly failure?: string | null;
|
||||
readonly timeoutSeconds?: number | null;
|
||||
}): Record<string, unknown> {
|
||||
if (input.observerId === null || input.observerId.length === 0) {
|
||||
return { ok: false, reason: "observer-id-missing", runId: input.runId, valuesRedacted: true };
|
||||
}
|
||||
const scenarioId = input.scenarioId ?? stringAt(state.cicd, "targetValidation.scenarioId");
|
||||
const scenario = findScenario(state, scenarioId);
|
||||
if (scenario === null) {
|
||||
return { ok: false, reason: "scenario-not-found", runId: input.runId, scenarioId, observerId: input.observerId, valuesRedacted: true };
|
||||
}
|
||||
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
|
||||
const promptIndex = commandSequence.reduce((count, item) => {
|
||||
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
|
||||
return stringAtNullable(item, "type") === "sendPrompt" ? count + repeat : count;
|
||||
}, 0);
|
||||
const timeoutSeconds = Math.max(5, Math.min(Math.trunc(input.timeoutSeconds ?? 55), 55));
|
||||
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, timeoutSeconds);
|
||||
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, timeoutSeconds);
|
||||
const findings = quickVerifyControlFindings(input.failure ?? null, promptIndex, turnSummary, traceFrame);
|
||||
return {
|
||||
ok: true,
|
||||
runId: input.runId,
|
||||
scenarioId,
|
||||
observerId: input.observerId,
|
||||
promptIndex,
|
||||
findingCount: findings.length,
|
||||
findings,
|
||||
turnSummary: { ok: turnSummary.ok === true, collectShape: turnSummary.collectShape, valuesRedacted: true },
|
||||
traceFrame: { ok: traceFrame.ok === true, collectShape: traceFrame.collectShape, valuesRedacted: true },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
||||
readonly runId: string;
|
||||
readonly scenarioId: string;
|
||||
@@ -542,6 +581,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
||||
&& controlFindings.length === 0
|
||||
&& blockingFindings.length === 0;
|
||||
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, targetValidationSeconds);
|
||||
const primaryControlFailure = quickVerifyPrimaryControlFailure(controlFindings);
|
||||
return {
|
||||
ok: recoveredWaitFailure,
|
||||
runId: input.runId,
|
||||
@@ -558,7 +598,8 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
|
||||
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
|
||||
findingCount: findings.length,
|
||||
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
|
||||
failure: recoveredWaitFailure ? null : input.failure,
|
||||
failure: recoveredWaitFailure ? null : primaryControlFailure ?? input.failure,
|
||||
runnerFailure: recoveredWaitFailure ? null : input.failure,
|
||||
promptSource: input.promptSource,
|
||||
steps: [...input.steps, ...cleanupSteps],
|
||||
analysis: artifactSummary,
|
||||
@@ -1811,6 +1852,13 @@ function isQuickVerifyBlockingFinding(item: Record<string, unknown>): boolean {
|
||||
if (id === "observer-command-failed") return observerCommandFailureBlocks(item);
|
||||
return [
|
||||
"quick-verify-no-business-turn",
|
||||
"quick-verify-target-turn-missing",
|
||||
"quick-verify-target-turn-trace-id-missing",
|
||||
"quick-verify-trace-projection-missing",
|
||||
"quick-verify-final-response-empty-incomplete",
|
||||
"quick-verify-final-response-empty-terminal",
|
||||
"quick-verify-final-response-empty-status-unknown",
|
||||
"quick-verify-diagnostics-inconclusive",
|
||||
"quick-verify-analysis-summary-unreadable",
|
||||
"quick-verify-command-sequence-failed",
|
||||
"quick-verify-observer-start-failed",
|
||||
@@ -1836,14 +1884,105 @@ function observerCommandFailureBlocks(item: Record<string, unknown>): boolean {
|
||||
});
|
||||
}
|
||||
|
||||
function quickVerifyPrimaryControlFailure(controlFindings: Record<string, unknown>[]): string | null {
|
||||
const first = controlFindings[0];
|
||||
return first === undefined ? null : stringAtNullable(first, "id") ?? stringAtNullable(first, "kind") ?? stringAtNullable(first, "code");
|
||||
}
|
||||
|
||||
function quickVerifyControlEvidenceSummary(turnDiagnostics: Record<string, unknown>, traceDiagnostics: Record<string, unknown>, targetStatus: Record<string, unknown> | null): string {
|
||||
const targetStatusText = targetStatus === null ? "-" : stringAtNullable(targetStatus, "status") ?? "-";
|
||||
const targetMarks = targetStatus === null ? "-" : stringAtNullable(targetStatus, "marks") ?? "-";
|
||||
const targetTrace = targetStatus === null ? "-" : targetStatus.traceIdPresent === true ? "present" : "missing";
|
||||
const targetFinalBytes = targetStatus === null ? "-" : String(targetStatus.finalResponseBytes ?? "-");
|
||||
const traceBlocker = stringAtNullable(traceDiagnostics, "blocker") ?? "-";
|
||||
const traceRowCount = traceDiagnostics.traceRowCount === null || traceDiagnostics.traceRowCount === undefined ? "-" : String(traceDiagnostics.traceRowCount);
|
||||
return [
|
||||
`turn-summary rows=${String(turnDiagnostics.rowCount ?? 0)}`,
|
||||
`scoped=${String(turnDiagnostics.scopedRowCount ?? 0)}`,
|
||||
`targetStatus=${targetStatusText}`,
|
||||
`targetMarks=${targetMarks}`,
|
||||
`targetTrace=${targetTrace}`,
|
||||
`targetFinalBytes=${targetFinalBytes}`,
|
||||
`traceFrameTrace=${traceDiagnostics.traceIdPresent === true ? "present" : "missing"}`,
|
||||
`traceFrameBlocker=${traceBlocker}`,
|
||||
`traceRows=${traceRowCount}`,
|
||||
`traceFinalBytes=${String(traceDiagnostics.finalResponseBytes ?? "-")}`,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function quickVerifyScopedStatuses(turnDiagnostics: Record<string, unknown>): Record<string, unknown>[] {
|
||||
return Array.isArray(turnDiagnostics.scopedStatuses) ? turnDiagnostics.scopedStatuses.map(record) : [];
|
||||
}
|
||||
|
||||
function quickVerifyLatestScopedStatus(turnDiagnostics: Record<string, unknown>): Record<string, unknown> | null {
|
||||
const rows = quickVerifyScopedStatuses(turnDiagnostics);
|
||||
return rows.length === 0 ? null : rows[rows.length - 1] ?? null;
|
||||
}
|
||||
|
||||
function quickVerifyFinalResponseEmpty(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
|
||||
return targetStatus?.finalResponseEmpty === true || traceDiagnostics.finalResponseEmpty === true;
|
||||
}
|
||||
|
||||
function quickVerifyTargetTracePresent(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
|
||||
return targetStatus?.traceIdPresent === true || traceDiagnostics.traceIdPresent === true;
|
||||
}
|
||||
|
||||
function quickVerifyTraceProjectionMissing(targetStatus: Record<string, unknown> | null, traceDiagnostics: Record<string, unknown>): boolean {
|
||||
if (!quickVerifyTargetTracePresent(targetStatus, traceDiagnostics)) return false;
|
||||
const blocker = stringAtNullable(traceDiagnostics, "blocker");
|
||||
if (blocker === "trace-rows-missing" || blocker === "sample-not-found") return true;
|
||||
if (traceDiagnostics.collectOk !== true) return true;
|
||||
const traceRowCount = numberAtNullable(traceDiagnostics, "traceRowCount");
|
||||
return traceRowCount !== null && traceRowCount === 0;
|
||||
}
|
||||
|
||||
function quickVerifyTargetStatusKind(targetStatus: Record<string, unknown> | null): "incomplete" | "terminal" | "unknown" {
|
||||
const status = (targetStatus === null ? "" : stringAtNullable(targetStatus, "status") ?? "").toLowerCase();
|
||||
const marks = (targetStatus === null ? "" : stringAtNullable(targetStatus, "marks") ?? "").toLowerCase();
|
||||
if (/running|pending|queued|starting|submitting|processing|in[-_ ]?progress|waiting/u.test(status)) return "incomplete";
|
||||
if (/cancel/u.test(status) || /cancel/u.test(marks)) return "incomplete";
|
||||
if (/completed|succeeded|success|done|failed|failure|error|terminal|command-failed|timeout|stopped|aborted/u.test(status)) return "terminal";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function quickVerifyControlFinding(
|
||||
id: string,
|
||||
titleZh: string,
|
||||
summary: string,
|
||||
rootCause: string,
|
||||
nextAction: string,
|
||||
promptIndex: number,
|
||||
failure: string | null,
|
||||
turnDiagnostics: Record<string, unknown>,
|
||||
traceDiagnostics: Record<string, unknown>,
|
||||
targetStatus: Record<string, unknown> | null,
|
||||
display: Record<string, unknown> | null,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
id,
|
||||
severity: "red",
|
||||
count: 1,
|
||||
summary,
|
||||
displayTitleZh: titleZh,
|
||||
errorTitleZh: titleZh,
|
||||
timeoutDisplay: display,
|
||||
rootCause,
|
||||
rootCauseStatus: "confirmed",
|
||||
rootCauseConfidence: "high",
|
||||
evidenceSummary: quickVerifyControlEvidenceSummary(turnDiagnostics, traceDiagnostics, targetStatus),
|
||||
nextAction,
|
||||
turnSummaryDiagnostics: turnDiagnostics,
|
||||
traceFrameDiagnostics: traceDiagnostics,
|
||||
failure: failure ?? null,
|
||||
promptIndex,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null, display: Record<string, unknown> | null = null): Record<string, unknown>[] {
|
||||
if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return [];
|
||||
const turnDiagnostics = quickVerifyTurnSummaryDiagnostics(promptIndex, turnSummary);
|
||||
const traceDiagnostics = quickVerifyTraceFrameDiagnostics(traceFrame);
|
||||
const rendered = [
|
||||
typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "",
|
||||
typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "",
|
||||
].join("\n");
|
||||
const noPromptScenario = promptIndex <= 0;
|
||||
if (noPromptScenario && failure === null) return [];
|
||||
if (noPromptScenario && failure !== null) {
|
||||
@@ -1864,29 +2003,135 @@ function quickVerifyControlFindings(failure: string | null, promptIndex: number,
|
||||
valuesRedacted: true,
|
||||
}];
|
||||
}
|
||||
const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
|
||||
const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered);
|
||||
if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
|
||||
const displayTitleZh = stringAtNullable(display ?? {}, "titleZh");
|
||||
return [{
|
||||
id: "quick-verify-no-business-turn",
|
||||
severity: "red",
|
||||
count: 1,
|
||||
summary: displayTitleZh ?? "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
|
||||
displayTitleZh,
|
||||
errorTitleZh: displayTitleZh,
|
||||
timeoutDisplay: display,
|
||||
rootCause: `quick verify could not confirm a durable completed turn: turn-summary scopedRows=${String(turnDiagnostics.scopedRowCount ?? 0)} rowCount=${String(turnDiagnostics.rowCount ?? 0)}, traceFrame traceIdPresent=${traceDiagnostics.traceIdPresent === true} finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
|
||||
rootCauseStatus: "confirmed",
|
||||
rootCauseConfidence: "high",
|
||||
evidenceSummary: `turn-summary rows=${String(turnDiagnostics.rowCount ?? 0)} scoped=${String(turnDiagnostics.scopedRowCount ?? 0)} traceFrameTrace=${traceDiagnostics.traceIdPresent === true ? "present" : "missing"} finalResponseBytes=${String(traceDiagnostics.finalResponseBytes ?? "-")}`,
|
||||
nextAction: "Inspect the structured turnSummary/traceFrame diagnostics first; if rows exist with completed non-empty final responses, fix sentinel interpretation instead of treating HWLAB Web as blocked.",
|
||||
turnSummaryDiagnostics: turnDiagnostics,
|
||||
traceFrameDiagnostics: traceDiagnostics,
|
||||
failure: failure ?? null,
|
||||
const rowCount = numberAtNullable(turnDiagnostics, "rowCount") ?? 0;
|
||||
const scopedRowCount = numberAtNullable(turnDiagnostics, "scopedRowCount") ?? 0;
|
||||
const targetStatus = quickVerifyLatestScopedStatus(turnDiagnostics);
|
||||
if (rowCount === 0) {
|
||||
return [quickVerifyControlFinding(
|
||||
"quick-verify-no-business-turn",
|
||||
"快速验证没有采集到业务轮次",
|
||||
"quick verify collected zero business turn rows; this is not a trace/final-response failure.",
|
||||
`turn-summary rowCount=0 for promptIndex=${promptIndex}.`,
|
||||
"Inspect observe command execution and turn-summary collection before checking trace-frame or final response.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
)];
|
||||
}
|
||||
if (scopedRowCount === 0) {
|
||||
return [quickVerifyControlFinding(
|
||||
"quick-verify-target-turn-missing",
|
||||
"快速验证缺少目标业务轮次",
|
||||
"quick verify collected turn rows, but none matched the target prompt index.",
|
||||
`turn-summary rowCount=${rowCount} but scopedRowCount=0 for promptIndex=${promptIndex}.`,
|
||||
"Check promptIndex/command segmentation and rerun the bounded turn-summary view for the observer.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
)];
|
||||
}
|
||||
|
||||
const findings: Record<string, unknown>[] = [];
|
||||
if (!quickVerifyTargetTracePresent(targetStatus, traceDiagnostics)) {
|
||||
findings.push(quickVerifyControlFinding(
|
||||
"quick-verify-target-turn-trace-id-missing",
|
||||
"目标业务轮次缺少 traceId",
|
||||
"quick verify found the target business turn, but the turn has no traceId.",
|
||||
`target turn exists for promptIndex=${promptIndex}, but traceIdPresent=false.`,
|
||||
"Fix Workbench/Code Agent traceId projection for the target turn before inspecting trace rows.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
));
|
||||
return findings;
|
||||
}
|
||||
|
||||
if (quickVerifyTraceProjectionMissing(targetStatus, traceDiagnostics)) {
|
||||
findings.push(quickVerifyControlFinding(
|
||||
"quick-verify-trace-projection-missing",
|
||||
"目标 trace 投影缺失",
|
||||
"quick verify found a target turn traceId, but trace-frame did not contain trace rows/projection for it.",
|
||||
`target turn has traceId, but trace-frame collectOk=${traceDiagnostics.collectOk === true} blocker=${stringAtNullable(traceDiagnostics, "blocker") ?? "-"} traceRowCount=${String(traceDiagnostics.traceRowCount ?? "-")}.`,
|
||||
"Inspect trace-frame for the target traceId, then check Workbench trace projection/read-model hydration before treating the run as a business failure.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
));
|
||||
}
|
||||
|
||||
if (quickVerifyFinalResponseEmpty(targetStatus, traceDiagnostics)) {
|
||||
const statusKind = quickVerifyTargetStatusKind(targetStatus);
|
||||
if (statusKind === "incomplete") {
|
||||
findings.push(quickVerifyControlFinding(
|
||||
"quick-verify-final-response-empty-incomplete",
|
||||
"目标轮次未完成且最终回复为空",
|
||||
"quick verify found the target business turn, but it is still incomplete/cancel-marked and Final Response is empty.",
|
||||
`target turn status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} marks=${stringAtNullable(targetStatus ?? {}, "marks") ?? "-"} finalResponseEmpty=true.`,
|
||||
"Inspect the same traceId in Workbench and Code Agent runtime; this is an unfinished/cancelled turn, not a zero-dialog case.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
));
|
||||
} else if (statusKind === "terminal") {
|
||||
findings.push(quickVerifyControlFinding(
|
||||
"quick-verify-final-response-empty-terminal",
|
||||
"目标轮次已终止但最终回复为空",
|
||||
"quick verify found a terminal target business turn, but Final Response is empty.",
|
||||
`target turn terminal status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} finalResponseEmpty=true.`,
|
||||
"Inspect terminal event persistence and final-response projection for the target traceId; do not classify this as no dialog.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
));
|
||||
} else {
|
||||
findings.push(quickVerifyControlFinding(
|
||||
"quick-verify-final-response-empty-status-unknown",
|
||||
"目标轮次状态未知且最终回复为空",
|
||||
"quick verify found the target business turn and empty Final Response, but the turn status is not classifiable.",
|
||||
`target turn status=${stringAtNullable(targetStatus ?? {}, "status") ?? "-"} finalResponseEmpty=true.`,
|
||||
"Fix the turn-summary status projection so quick verify can classify incomplete versus terminal final-response failures.",
|
||||
promptIndex,
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if (findings.length > 0) return findings;
|
||||
return [quickVerifyControlFinding(
|
||||
"quick-verify-diagnostics-inconclusive",
|
||||
"快速验证诊断信息不足",
|
||||
"quick verify did not prove a durable completed turn, but structured diagnostics did not match any specific failure code.",
|
||||
`quick verify has rowCount=${rowCount} scopedRowCount=${scopedRowCount}, traceIdPresent=${traceDiagnostics.traceIdPresent === true}, finalResponseEmpty=${traceDiagnostics.finalResponseEmpty === true}.`,
|
||||
"Improve turn-summary/trace-frame diagnostics before making a business recovery decision.",
|
||||
promptIndex,
|
||||
valuesRedacted: true,
|
||||
}];
|
||||
failure,
|
||||
turnDiagnostics,
|
||||
traceDiagnostics,
|
||||
targetStatus,
|
||||
display,
|
||||
)];
|
||||
}
|
||||
|
||||
function enrichObserveStartFailureFinding(finding: Record<string, unknown>, result: Record<string, unknown>): Record<string, unknown> {
|
||||
@@ -2009,6 +2254,7 @@ function quickVerifyTurnSummaryDiagnostics(promptIndex: number, turnSummary: Rec
|
||||
return {
|
||||
round: numberAtNullable(row, "round"),
|
||||
status: stringAtNullable(row, "status"),
|
||||
marks: stringAtNullable(row, "marks"),
|
||||
traceIdPresent: stringAtNullable(row, "traceId") !== null,
|
||||
finalResponseEmpty: finalResponse.empty === true,
|
||||
finalResponseBytes: numberAtNullable(finalResponse, "textBytes"),
|
||||
|
||||
@@ -8,6 +8,7 @@ import { repoRoot } from "./config";
|
||||
import { startJob } from "./jobs";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
|
||||
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
||||
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
|
||||
import {
|
||||
clipTail,
|
||||
@@ -32,7 +33,7 @@ import {
|
||||
text,
|
||||
withWarnings,
|
||||
} from "./hwlab-node-web-sentinel-cicd";
|
||||
import { metricNames, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
|
||||
const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55;
|
||||
|
||||
@@ -264,6 +265,7 @@ function compactSentinelReportRawPayload(
|
||||
const findings = Array.isArray(body.findings) ? body.findings.map(record) : [];
|
||||
const artifactFindings = Array.isArray(artifact.findings) ? artifact.findings.map(record) : [];
|
||||
const visibleFindings = mergeSentinelReportFindings(findings, artifactFindings);
|
||||
const offlineReclassify = offlineQuickVerifyReclassify(state, run, visibleFindings);
|
||||
const storedFindingCount = numberAtNullable(run, "finding_count") ?? numberAtNullable(run, "findingCount") ?? findings.length;
|
||||
const artifactFindingCount = numberAtNullable(artifact, "findingCount") ?? artifactFindings.length;
|
||||
const visibleFindingCount = Math.max(storedFindingCount, artifactFindingCount, visibleFindings.length);
|
||||
@@ -306,6 +308,7 @@ function compactSentinelReportRawPayload(
|
||||
},
|
||||
summary: pickFields(record(body.summary), ["reason", "status", "businessStatus", "failure", "valuesRedacted"]),
|
||||
findings: visibleFindings.slice(0, 12).map(compactSentinelReportFinding),
|
||||
offlineReclassify,
|
||||
artifactSummary: Object.keys(artifact).length === 0 ? null : {
|
||||
ok: artifact.ok === true,
|
||||
reason: artifact.reason ?? null,
|
||||
@@ -333,6 +336,53 @@ function compactSentinelReportRawPayload(
|
||||
};
|
||||
}
|
||||
|
||||
function offlineQuickVerifyReclassify(state: SentinelCicdState, run: Record<string, unknown>, findings: readonly Record<string, unknown>[]): Record<string, unknown> | null {
|
||||
const hasLegacyQuickVerifyControl = findings.some((item) => sentinelReportFindingIdentityCandidates(item).includes("quick-verify-no-business-turn"));
|
||||
if (!hasLegacyQuickVerifyControl) return null;
|
||||
const catalog = sentinelReportCheckCatalogById(state);
|
||||
const result = reclassifyQuickVerifyControlFindings(state, {
|
||||
runId: stringAtNullable(run, "id"),
|
||||
scenarioId: stringAtNullable(run, "scenario_id") ?? stringAtNullable(run, "scenarioId"),
|
||||
observerId: stringAtNullable(run, "observer_id") ?? stringAtNullable(run, "observerId"),
|
||||
failure: stringAtNullable(record(run.summary), "failure"),
|
||||
timeoutSeconds: SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS,
|
||||
});
|
||||
const resultFindings = Array.isArray(result.findings) ? result.findings.map(record) : [];
|
||||
return {
|
||||
...pickFields(result, ["ok", "reason", "runId", "scenarioId", "observerId", "promptIndex", "findingCount", "turnSummary", "traceFrame", "valuesRedacted"]),
|
||||
source: "offline-existing-observe-artifact",
|
||||
note: "This is a local CLI reclassification of existing turn-summary/trace-frame artifacts; it does not mutate the runner index.",
|
||||
findings: resultFindings.slice(0, 8).map((item) => compactSentinelReportFinding(enrichSentinelReportFindingWithCatalog(item, catalog))),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelReportCheckCatalogById(state: SentinelCicdState): Map<string, Record<string, unknown>> {
|
||||
try {
|
||||
const reportViews = record(readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.reportViews));
|
||||
const catalogRef = stringAtNullable(reportViews, "checkCatalogRef");
|
||||
if (catalogRef === null) return new Map();
|
||||
const catalog = record(readWebProbeSentinelConfigRefTarget(state.spec, catalogRef));
|
||||
const items = Array.isArray(catalog.items) ? catalog.items.map(record) : Array.isArray(catalog.checks) ? catalog.checks.map(record) : [];
|
||||
return new Map(items.map((item) => [stringAtNullable(item, "id") ?? "", item]).filter((item): item is [string, Record<string, unknown>] => item[0].length > 0));
|
||||
} catch {
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
function enrichSentinelReportFindingWithCatalog(item: Record<string, unknown>, catalog: ReadonlyMap<string, Record<string, unknown>>): Record<string, unknown> {
|
||||
const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code");
|
||||
const check = id === null ? null : catalog.get(id) ?? null;
|
||||
if (check === null) return item;
|
||||
return {
|
||||
...item,
|
||||
check,
|
||||
checkCode: stringAtNullable(check, "code"),
|
||||
checkTitleZh: stringAtNullable(check, "titleZh"),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function mergeSentinelReportFindings(primary: readonly Record<string, unknown>[], artifact: readonly Record<string, unknown>[]): Record<string, unknown>[] {
|
||||
const merged: Record<string, unknown>[] = [];
|
||||
const seen = new Set<string>();
|
||||
@@ -423,8 +473,12 @@ function compactSentinelReportFinding(value: Record<string, unknown>): Record<st
|
||||
const rootCauseSignals = compactRootCauseSignals(value.rootCauseSignals);
|
||||
if (rootCauseSignals !== null) result.rootCauseSignals = rootCauseSignals;
|
||||
const check = record(value.check);
|
||||
const checkCodeFromValue = stringAtNullable(value, "checkCode");
|
||||
const checkTitleFromValue = stringAtNullable(value, "checkTitleZh");
|
||||
const checkCode = stringAtNullable(check, "code");
|
||||
const checkTitle = stringAtNullable(check, "titleZh");
|
||||
if (checkCodeFromValue !== null) result.checkCode = checkCodeFromValue;
|
||||
if (checkTitleFromValue !== null) result.checkTitleZh = checkTitleFromValue;
|
||||
if (checkCode !== null || checkTitle !== null) {
|
||||
result.check = pickFields(check, ["id", "code", "level", "titleZh", "blocking", "registered"]);
|
||||
}
|
||||
@@ -471,7 +525,9 @@ function renderSentinelReportSummary(payload: Record<string, unknown>, state: Se
|
||||
const run = record(payload.run);
|
||||
const summary = record(payload.summary);
|
||||
const artifactSummary = record(payload.artifactSummary);
|
||||
const offlineReclassify = record(payload.offlineReclassify);
|
||||
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
|
||||
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
|
||||
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
|
||||
const findingCount = run.findingCount ?? findings.length;
|
||||
const analyzerArtifactCount = numberAtNullable(artifactSummary, "artifactCount") ?? numberAtNullable(record(artifactSummary.counts), "artifacts");
|
||||
@@ -498,26 +554,34 @@ function renderSentinelReportSummary(payload: Record<string, unknown>, state: Se
|
||||
"",
|
||||
"Findings",
|
||||
findingRows,
|
||||
offlineFindings.length === 0 ? "" : "",
|
||||
offlineFindings.length === 0 ? "" : "Offline Reclassify",
|
||||
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function renderSentinelReportFindings(payload: Record<string, unknown>): string {
|
||||
const run = record(payload.run);
|
||||
const artifactSummary = record(payload.artifactSummary);
|
||||
const offlineReclassify = record(payload.offlineReclassify);
|
||||
const findings = Array.isArray(payload.findings) ? payload.findings.map(record) : [];
|
||||
const offlineFindings = Array.isArray(offlineReclassify.findings) ? offlineReclassify.findings.map(record) : [];
|
||||
const reportSha = stringAtNullable(run, "reportJsonSha256") ?? stringAtNullable(artifactSummary, "reportJsonSha256");
|
||||
return [
|
||||
"Web Probe Sentinel Findings",
|
||||
"=======================================================",
|
||||
`run=${run.id ?? "-"} report=${reportSha ?? "-"} findings=${run.findingCount ?? findings.length}`,
|
||||
findings.length === 0 ? "-" : findings.map(formatSentinelReportFindingLine).join("\n"),
|
||||
offlineFindings.length === 0 ? "" : "",
|
||||
offlineFindings.length === 0 ? "" : "Offline Reclassify",
|
||||
offlineFindings.length === 0 ? "" : offlineFindings.map(formatSentinelReportFindingLine).join("\n"),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function formatSentinelReportFindingLine(item: Record<string, unknown>): string {
|
||||
const check = record(item.check);
|
||||
const code = stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-";
|
||||
const title = stringAtNullable(check, "titleZh") ?? "";
|
||||
const code = stringAtNullable(item, "checkCode") ?? stringAtNullable(check, "code") ?? stringAtNullable(item, "id") ?? "-";
|
||||
const title = stringAtNullable(item, "checkTitleZh") ?? stringAtNullable(check, "titleZh") ?? "";
|
||||
const summary = reportText(item.summary, 180) ?? "";
|
||||
const rootCause = reportText(item.rootCause, 180);
|
||||
const evidence = reportText(item.evidenceSummary, 180);
|
||||
|
||||
Reference in New Issue
Block a user