diff --git a/scripts/src/hwlab-node-web-observe-collect.ts b/scripts/src/hwlab-node-web-observe-collect.ts index f3c455dc..1b9337d8 100644 --- a/scripts/src/hwlab-node-web-observe-collect.ts +++ b/scripts/src/hwlab-node-web-observe-collect.ts @@ -331,14 +331,16 @@ function traceFrameDiagnosticLines(sample,traceId,matchedTraceRows){ function selectSample(rows){ if(requestedSampleSeq!==null){const exact=samples.find((sample)=>Number(sample.seq)===requestedSampleSeq); if(exact)return exact;} if(requestedTimestamp){const target=tsMs(requestedTimestamp); if(target!==null){const before=samples.filter((sample)=>{const ms=tsMs(sample.ts); return ms!==null&&ms<=target}).slice(-1)[0]; if(before)return before;}} - if(requestedTurn!==null&&rows[requestedTurn-1]?.traceId){const row=rows[requestedTurn-1]; const byTrace=latestSampleForTrace(samples,row.traceId,numOrNull(row.lastSeq)); if(byTrace)return byTrace;} - if(requestedTurn!==null&&rows[requestedTurn-1]?.lastSeq!==null){const byTurn=samples.find((sample)=>Number(sample.seq)===Number(rows[requestedTurn-1].lastSeq)); if(byTurn)return byTurn;} + const requestedRow=requestedTurn!==null?rows[requestedTurn-1]:null; + if(requestedRow?.traceId){const byTrace=latestSampleForTrace(samples,requestedRow.traceId,numOrNull(requestedRow.lastSeq)); if(byTrace)return byTrace;} + if(requestedRow&&requestedRow.lastSeq!==null&&requestedRow.lastSeq!==undefined){const byTurn=samples.find((sample)=>Number(sample.seq)===Number(requestedRow.lastSeq)); if(byTurn)return byTurn;} if(requestedTraceId){const byTrace=samples.filter((sample)=>traceIdsFromSamples([sample]).includes(requestedTraceId)).slice(-1)[0]; if(byTrace)return byTrace;} return samples[samples.length-1]||null; } function renderTraceFrame(sample,rows){ if(!sample)return {ok:false,renderedText:'TRACE_FRAME_BLOCKER\\nno sample matched --sample-seq/--timestamp/--trace-id/--turn',blocker:'sample-not-found'}; - const sampleTraceIds=traceIdsFromSamples([sample]); const traceId=requestedTraceId||(requestedTurn!==null?rows[requestedTurn-1]?.traceId:null)||rows.find((row)=>row.lastSeq===sample.seq)?.traceId||sampleTraceIds[0]||null; + const requestedRow=requestedTurn!==null?rows[requestedTurn-1]:null; + const sampleTraceIds=traceIdsFromSamples([sample]); const traceId=requestedTraceId||requestedRow?.traceId||rows.find((row)=>row.lastSeq===sample.seq)?.traceId||sampleTraceIds[0]||null; const traceRows=(Array.isArray(sample.traceRows)?sample.traceRows:[]).filter((row)=>!traceId||row.traceId===traceId||!row.traceId||textOf(row).includes(traceId)); const turns=(Array.isArray(sample.turns)?sample.turns:[]).filter((turn)=>!traceId||turn.traceId===traceId||textOf(turn).includes(traceId)); const texts=[...turns.map(textOf),...traceRows.map(textOf)]; diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 34413f9b..e045c5cb 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -542,6 +542,58 @@ function accountSecretEnvName(sourcePurpose: string, targetKey: string): string return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`; } +function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record } { + const sourcesByPurpose = new Map>(); + for (const source of arrayAt(state.secrets, "sources").map(record)) { + const purpose = stringAtNullable(source, "purpose"); + if (purpose !== null) sourcesByPurpose.set(purpose, source); + } + const env: NodeJS.ProcessEnv = {}; + const items: Record[] = []; + const missing: Record[] = []; + for (const runtimeSecret of arrayAt(state.secrets, "runtimeSecrets").map(record)) { + const secretName = stringAtNullable(runtimeSecret, "name"); + for (const item of arrayAt(runtimeSecret, "data").map(record)) { + const targetKey = stringAtNullable(item, "targetKey"); + const sourcePurpose = stringAtNullable(item, "sourcePurpose"); + const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); + if (envName === null || sourcePurpose === null || targetKey === null) continue; + const source = sourcesByPurpose.get(sourcePurpose); + if (source === undefined) { + missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true }); + continue; + } + const sourceRef = stringAt(source, "sourceRef"); + const sourceKey = stringAt(source, "sourceKey"); + const material = readSentinelSecretSourceValue(source); + if (!material.ok) { + missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true }); + continue; + } + const value = stringAt(material, "value"); + env[envName] = value; + items.push({ + envName, + secretName, + targetKey, + sourcePurpose, + sourceRef, + sourceKey, + fingerprint: `sha256:${createHash("sha256").update(value).digest("hex").slice(0, 16)}`, + valuesRedacted: true, + }); + } + } + const summary = { + ok: missing.length === 0, + envCount: items.length, + items, + missing, + valuesRedacted: true, + }; + return { ok: missing.length === 0, env, summary }; +} + function normalizeRoutePrefix(value: string | null): string { if (value === null || value.trim() === "" || value.trim() === "/") return "/"; const prefixed = value.trim().startsWith("/") ? value.trim() : `/${value.trim()}`; @@ -1699,6 +1751,38 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou ? readPromptSetForScenario(scenario) : { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } }; if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true }; + const accountEnv = quickVerifyAccountEnv(state); + if (!accountEnv.ok) { + const findings = [{ + id: "quick-verify-account-secret-missing", + severity: "red", + count: arrayAt(accountEnv.summary, "missing").length || 1, + summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.", + missing: arrayAt(accountEnv.summary, "missing"), + valuesRedacted: true, + }]; + return recordQuickVerify(state, { + ok: false, + runId: `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`, + scenarioId, + reason, + status: "blocked", + observerId: null, + elapsedMs: 0, + steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }], + failure: "quick-verify-account-secret-missing", + findingCount: findings.length, + findings, + promptSource: prompts.summary, + accountEnv: accountEnv.summary, + views: { + summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + }, + warnings: [], + valuesRedacted: true, + }); + } const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs"); const budgetSeconds = Math.min(timeoutSeconds, maxSeconds); const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", budgetSeconds); @@ -1714,7 +1798,7 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--command-timeout-seconds", "55", ]; - const started = runChildCli(startArgs, remainingSeconds(deadline, 55)); + const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); if (!started.ok || observerId === null) { @@ -1865,10 +1949,12 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null, promptSource: prompts.summary, + accountEnv: accountEnv.summary, steps, analysis: artifactSummary, views: { summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, }, @@ -2006,6 +2092,7 @@ function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { analysis: artifactSummary, views: { summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, }, @@ -2753,8 +2840,12 @@ function collectObserveView(state: SentinelCicdState, observerId: string, view: }; } -function runChildCli(args: string[], timeoutSeconds: number, input?: string): ChildCliResult { - const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { input, timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000 }); +function runChildCli(args: string[], timeoutSeconds: number, input?: string, env?: NodeJS.ProcessEnv): ChildCliResult { + const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { + input, + env: env === undefined ? undefined : { ...process.env, ...env }, + timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000, + }); return { ok: result.exitCode === 0 && !result.timedOut, parsed: parseJsonObject(result.stdout), @@ -3133,9 +3224,12 @@ function isQuickVerifyBlockingFinding(item: Record): boolean { const severity = (stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "").toLowerCase(); if (!["critical", "red", "fatal", "error", "failed", "blocked"].includes(severity)) return false; const id = (stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? "").toLowerCase(); + if (id === "observer-command-failed") return observerCommandFailureBlocks(item); return [ "quick-verify-no-business-turn", - "observer-command-failed", + "quick-verify-command-sequence-failed", + "quick-verify-observer-start-failed", + "quick-verify-account-secret-missing", "prompt-chat-submit-failed", "route-active-session-mismatch", "final-response-flicker", @@ -3146,15 +3240,40 @@ function isQuickVerifyBlockingFinding(item: Record): boolean { ].includes(id); } +function observerCommandFailureBlocks(item: Record): boolean { + const commands = Array.isArray(item.commands) ? item.commands.map(record) : []; + if (commands.length === 0) return true; + return commands.some((command) => { + const type = (stringAtNullable(command, "type") ?? "").toLowerCase(); + if (["stop", "cancel", "mark", "screenshot"].includes(type)) return false; + return true; + }); +} + function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): Record[] { const rendered = [ typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "", typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "", ].join("\n"); - const noPrompt = promptIndex <= 0 || /无\s*sendPrompt|no\s+sendPrompt/iu.test(rendered); - const noTrace = /无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered); + const noPromptScenario = promptIndex <= 0; + if (noPromptScenario && failure === null) return []; + if (noPromptScenario && failure !== null) { + const observerStartFailure = failure === "observe-start-failed"; + return [{ + id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed", + severity: "red", + count: 1, + summary: observerStartFailure + ? "quick verify observer failed to start before the no-prompt scenario could run." + : "quick verify no-prompt command sequence failed before the account/session workflow completed.", + failure, + promptIndex, + valuesRedacted: true, + }]; + } + const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered); const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered); - if (!noPrompt && !noTrace && !emptyFinal && failure !== "observe-start-failed") return []; + if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return []; return [{ id: "quick-verify-no-business-turn", severity: "red", @@ -3191,6 +3310,42 @@ function renderQuickVerifySummary(input: Record): string { ].join("\n"); } +function renderAuthSessionSwitchQuickVerifySummary(input: Record): string { + const artifact = record(input.artifactSummary); + const accountEnv = record(input.accountEnv); + const findingRows = Array.isArray(input.findings) + ? input.findings.map(record).slice(0, 8) + : Array.isArray(artifact.findings) + ? artifact.findings.map(record).slice(0, 8) + : []; + const steps = Array.isArray(input.steps) ? input.steps.map(record) : []; + const commandSteps = steps + .filter((step) => { + const phase = stringAtNullable(step, "phase") ?? ""; + return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env"; + }) + .map((step) => { + const phase = stringAtNullable(step, "phase") ?? "-"; + const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-"; + const result = record(step.result); + const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-"; + return `${phase} ${ok} ${status}`; + }); + return [ + "Auth Session Switch Quick Verify", + "=======================================================", + `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, + `status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`, + `accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`, + "", + "Command Sequence", + commandSteps.length === 0 ? "-" : commandSteps.join("\n"), + "", + "Findings", + findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"} ${item.summary ?? item.message ?? ""}`).join("\n"), + ].join("\n"); +} + function renderMaintenanceResult(result: Record): string { const serviceHealth = record(result.serviceHealth); const maintenance = record(result.maintenance);