mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c8f165392ad73102be00a6dfb887bd635d4ec83e
pikasTech-unidesk/scripts/src/hwlab-node-web-sentinel-p5-observe.ts
T
Codex 88fa7db59f fix: bound sentinel report evidence output
2026-06-30 08:28:14 +00:00

1588 lines
93 KiB
TypeScript
Raw Blame History

// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
// Responsibility: Quick-verify observe orchestration and artifact interpretation for web-probe sentinel P5 validation.
import { createHash, randomUUID } from "node:crypto";
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import type { CommandResult } from "./command";
import { runCommand } from "./command";
import { repoRoot, rootPath } from "./config";
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
import type { ChildCliResult, CompactCommandResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd";
import {
arrayAt,
compactCommand,
displayPath,
isRecord,
mergeWarnings,
numberAt,
numberAtNullable,
parseEnvFile,
parseJsonObject,
quickVerifyAccountEnv,
record,
recordTarget,
nonEmptyString,
secretSourcePaths,
sentinelCliSuffix,
shellQuote,
short,
stringAt,
stringAtNullable,
targetValidationElapsedWarnings,
text,
withWarnings,
} from "./hwlab-node-web-sentinel-cicd";
function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record<string, unknown> = {}): void {
const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => {
if (typeof value === "string") return [key, short(value)];
if (Array.isArray(value)) return [key, value.slice(0, 8)];
return [key, value];
}));
process.stdout.write(`${JSON.stringify({
event: "sentinel.quick-verify.progress",
at: new Date().toISOString(),
node: state.spec.nodeId,
lane: state.spec.lane,
sentinelId: state.sentinelId,
runId,
...compactExtra,
phase,
status,
valuesRedacted: true,
})}\n`);
}
export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record<string, unknown> {
const startedAt = Date.now();
const elapsedMs = () => Date.now() - startedAt;
const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId");
const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds");
const scenario = findScenario(state, scenarioId);
if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true };
const commandSequence = arrayAt(scenario, "commandSequence").map(record);
const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt" && inlinePromptText(item) === null);
const prompts = needsPromptSet
? readPromptSetForScenario(state, scenario)
: { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } };
if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true };
const accountEnv = quickVerifyAccountEnv(state);
if (!accountEnv.ok) {
const findings = [{
id: "quick-verify-account-secret-missing",
severity: "red",
count: arrayAt(accountEnv.summary, "missing").length || 1,
summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.",
missing: arrayAt(accountEnv.summary, "missing"),
valuesRedacted: true,
}];
return recordQuickVerify(state, {
ok: false,
runId: `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`,
scenarioId,
reason,
status: "blocked",
observerId: null,
elapsedMs: 0,
businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds),
steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }],
failure: "quick-verify-account-secret-missing",
findingCount: findings.length,
findings,
promptSource: prompts.summary,
accountEnv: accountEnv.summary,
views: {
summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
},
warnings: [],
valuesRedacted: true,
});
}
const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs");
const warningBudgetSeconds = maxSeconds;
const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds")));
const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds);
const deadline = Date.now() + hardBudgetSeconds * 1000;
const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds });
const steps: Record<string, unknown>[] = [];
const startArgs = [
"web-probe", "observe", "start",
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--target-path", stringAt(scenario, "observeTargetPath"),
"--sample-interval-ms", String(sampleIntervalMs),
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
"--command-timeout-seconds", "55",
];
const viewport = stringAtNullable(scenario, "viewport");
if (viewport !== null) startArgs.push("--viewport", viewport);
printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) });
const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env);
steps.push({ phase: "observe-start", ok: started.ok, result: started.result });
const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? ""));
printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() });
if (!started.ok || observerId === null) {
const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null);
return recordQuickVerify(state, {
ok: false,
runId,
scenarioId,
reason,
status: "blocked",
observerId,
elapsedMs: elapsedMs(),
businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds),
steps,
failure: "observe-start-failed",
findingCount: findings.length,
findings,
warnings: elapsedWarnings(),
valuesRedacted: true,
});
}
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) });
const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds);
steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady });
printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() });
if (startupReady.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex: 0,
steps,
failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"),
elapsedMs: elapsedMs(),
warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()),
promptSource: prompts.summary,
}));
}
let promptIndex = 0;
const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario);
const nonBlockingCanaryWarnings: string[] = [];
for (const item of commandSequence) {
const type = stringAt(item, "type");
const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1);
for (let index = 0; index < repeat; index += 1) {
if (Date.now() >= deadline) {
printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds });
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex,
steps,
failure: "quick-verify-timeout-over-budget",
elapsedMs: elapsedMs(),
warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()),
promptSource: prompts.summary,
}));
}
const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000));
const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55));
const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55));
const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))];
if (type === "selectProvider") args.push("--provider", stringAt(item, "provider"));
if (type === "loginAccount" || type === "listSessions" || type === "logout") {
const accountId = stringAtNullable(item, "accountId");
if (accountId !== null) args.push("--account-id", accountId);
}
if (type === "switchSessions") {
const fromAccountId = stringAtNullable(item, "fromAccountId");
const toAccountId = stringAtNullable(item, "toAccountId");
if (fromAccountId !== null) args.push("--from-account-id", fromAccountId);
if (toAccountId !== null) args.push("--to-account-id", toAccountId);
}
if (type === "sendPrompt") {
args.push("--text", inlinePromptText(item) ?? prompts.prompts[promptIndex % prompts.prompts.length] ?? "");
args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000));
promptIndex += 1;
}
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat });
const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10)));
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() });
if (!commandResult.ok) {
if (type === "sendPrompt") {
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult });
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() });
if (delayedWaitResult.ok === true) {
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
if (invariantResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex,
steps,
failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()),
}));
}
continue;
}
}
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex,
steps,
failure: `observe-command-${type}-failed`,
elapsedMs: elapsedMs(),
warnings: elapsedWarnings(),
promptSource: prompts.summary,
}));
}
if (type === "sendPrompt") {
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() });
if (waitResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex,
steps,
failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()),
}));
}
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
if (invariantResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex,
steps,
failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()),
}));
}
}
}
}
printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) });
const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120));
steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result });
printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() });
const indexEntry = readLocalObserveIndex(observerId);
const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30));
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30));
const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30));
const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame);
const artifactSummaryRecord = record(artifactSummary);
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
const findings = mergeFindingRecords(artifactFindings, controlFindings);
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."];
const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0;
const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds);
printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() });
return recordQuickVerify(state, {
ok,
runId,
scenarioId,
reason,
status: ok ? "analyzed" : "blocked",
observerId,
elapsedMs: elapsedMs(),
businessStatus,
stateDir: indexEntry?.stateDir ?? null,
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
findingCount: findings.length,
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null,
promptSource: prompts.summary,
accountEnv: accountEnv.summary,
steps,
analysis: artifactSummary,
views: {
summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
},
findings,
screenshot: record(artifactSummary).screenshot,
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, elapsedWarnings()),
valuesRedacted: true,
});
}
function sessionInvarianceChecksByRound(scenario: Record<string, unknown>): Map<number, Record<string, unknown>[]> {
const checks = new Map<number, Record<string, unknown>[]>();
const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : [];
for (const item of items) {
const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null;
if (afterRound === null || afterRound < 0) continue;
const list = checks.get(afterRound) ?? [];
list.push(item);
checks.set(afterRound, list);
}
return checks;
}
function runQuickVerifySessionInvarianceChecks(
state: SentinelCicdState,
observerId: string,
checks: readonly Record<string, unknown>[],
deadline: number,
promptIndex: number,
steps: Record<string, unknown>[],
): Record<string, unknown> {
const warnings: string[] = [];
for (const check of checks) {
const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`;
const blocking = check.blocking === true;
const commands: { readonly type: string; readonly enabled: boolean }[] = [
{ type: "refreshCurrentSession", enabled: check.refreshCurrent === true },
{ type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true },
{ type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false },
];
for (const command of commands) {
if (!command.enabled) continue;
const args = [
"web-probe", "observe", "command", observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--type", command.type,
"--after-round", String(promptIndex),
"--wait-ms", "55000",
"--command-timeout-seconds", String(remainingSeconds(deadline, 55)),
];
const severity = nonEmptyString(check.severity);
const findingId = nonEmptyString(check.findingId);
const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange);
const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy);
if (severity !== null) args.push("--severity", severity);
if (findingId !== null) args.push("--finding-id", findingId);
if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange);
if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy);
if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready");
args.push(check.blocking === true ? "--blocking" : "--non-blocking");
const result = runChildCli(args, remainingSeconds(deadline, 60));
steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result });
if (!result.ok) {
if (!blocking) {
warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`);
continue;
}
return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true };
}
}
}
return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true };
}
function appendScenarioObserveCommandArgs(args: string[], item: Record<string, unknown>, options: { readonly skipText?: boolean } = {}): void {
const mappings: readonly (readonly [string, string])[] = [
["path", "--path"],
["label", "--label"],
["sessionId", "--session-id"],
["provider", "--provider"],
["accountId", "--account-id"],
["fromAccountId", "--from-account-id"],
["toAccountId", "--to-account-id"],
["sourceId", "--source-id"],
["fileRef", "--file-ref"],
["filename", "--filename"],
["taskRef", "--task-ref"],
["taskId", "--task-id"],
["task", "--task"],
["field", "--field"],
["link", "--link"],
["title", "--title"],
["body", "--body"],
["status", "--status"],
["hwpodId", "--hwpod-id"],
["nodeId", "--node-id"],
["workspaceRoot", "--workspace-root"],
["root", "--root"],
];
for (const [key, flag] of mappings) {
if (args.includes(flag)) continue;
const value = stringAtNullable(item, key);
if (value !== null) args.push(flag, value);
}
if (options.skipText !== true && !args.includes("--text")) {
const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value");
if (text !== null) args.push("--text", text);
}
if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready");
}
function inlinePromptText(item: Record<string, unknown>): string | null {
return stringAtNullable(item, "text") ?? stringAtNullable(item, "prompt") ?? stringAtNullable(item, "value");
}
function finalizeQuickVerifyFailure(state: SentinelCicdState, input: {
readonly runId: string;
readonly scenarioId: string;
readonly reason: string;
readonly observerId: string;
readonly promptIndex: number;
readonly steps: readonly Record<string, unknown>[];
readonly failure: string;
readonly promptSource?: Record<string, unknown>;
readonly elapsedMs?: number;
readonly warnings?: readonly unknown[];
}): Record<string, unknown> {
const cleanupSteps: Record<string, unknown>[] = [];
if (input.promptIndex > 0) {
const cancel = runChildCli([
"web-probe", "observe", "command", input.observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--type", "cancel",
"--wait-ms", "55000",
"--command-timeout-seconds", "55",
], 60);
cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result });
}
const stop = runChildCli([
"web-probe", "observe", "stop", input.observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--force",
"--command-timeout-seconds", "55",
], 30);
cleanupSteps.push({ phase: "observe-stop-after-failure", ok: stop.ok, result: stop.result });
const analysis = runChildCli([
"web-probe", "observe", "analyze", input.observerId,
"--node", state.spec.nodeId,
"--lane", state.spec.lane,
"--command-timeout-seconds", "55",
], 60);
cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result });
const indexEntry = readLocalObserveIndex(input.observerId);
const artifactSummary = indexEntry === null
? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true }
: readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30);
const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30);
const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30);
const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame);
const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame);
const artifactSummaryRecord = record(artifactSummary);
const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : [];
const findings = mergeFindingRecords(artifactFindings, controlFindings);
const blockingFindings = findings.filter(isQuickVerifyBlockingFinding);
const recoveredWaitFailure = durableBusinessTurn
&& isRecoverableQuickVerifyWaitFailure(input.failure)
&& record(artifactSummary).ok === true
&& controlFindings.length === 0
&& blockingFindings.length === 0;
const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, numberAt(state.cicd, "targetValidation.maxSeconds"));
return {
ok: recoveredWaitFailure,
runId: input.runId,
scenarioId: input.scenarioId,
reason: input.reason,
status: recoveredWaitFailure ? "analyzed" : "blocked",
observerId: input.observerId,
elapsedMs: input.elapsedMs ?? null,
businessStatus,
stateDir: indexEntry?.stateDir ?? null,
reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"),
findingCount: findings.length,
artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0,
failure: recoveredWaitFailure ? null : input.failure,
promptSource: input.promptSource,
steps: [...input.steps, ...cleanupSteps],
analysis: artifactSummary,
views: {
summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) },
"turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok },
"trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok },
},
findings,
screenshot: record(artifactSummary).screenshot,
publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"),
warnings: mergeWarnings(
Array.isArray(input.warnings) ? input.warnings : [],
recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [],
targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", numberAt(state.cicd, "targetValidation.maxSeconds")),
),
valuesRedacted: true,
};
}
function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number): Record<string, unknown> {
const namespace = stringAt(state.runtime, "namespace");
const serviceName = stringAt(state.runtime, "serviceName");
const servicePort = numberAt(state.runtime, "servicePort");
const deploymentName = stringAt(state.runtime, "deploymentName");
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
const postScript = [
"const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
`const url = 'http://127.0.0.1:${servicePort}' + path;`,
"fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
"}).catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(23);",
"});",
].join(" ");
const script = method === "GET"
? `kubectl get --raw ${shellQuote(proxyPath)}`
: [
"set -eu",
`kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`,
].join("\n");
const maxAttempts = method === "GET" ? 3 : 1;
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
const attempts: Record<string, unknown>[] = [];
let result: CommandResult | null = null;
let parsed: Record<string, unknown> | null = null;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 });
parsed = parseJsonObject(result.stdout);
attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true });
if (result.exitCode === 0) break;
}
return {
ok: result?.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`,
httpStatus: result?.exitCode === 0 ? 200 : null,
bodyJson: record(compactSentinelServiceBodyJson(parsed)),
bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "",
bodyBytes: Buffer.byteLength(result?.stdout ?? ""),
error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000),
proxyPath,
result: result === null ? null : compactCommand(result),
attempts,
valuesRedacted: true,
};
}
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
if (value === null || typeof value.renderedText !== "string") return value;
return {
ok: value.ok,
view: value.view,
error: value.error,
availableViews: value.availableViews,
valuesRedacted: true,
};
}
function clipTail(value: string, maxLength: number): string {
return value.length <= maxLength ? value : value.slice(value.length - maxLength);
}
function recordQuickVerify(state: SentinelCicdState, payload: Record<string, unknown>): Record<string, unknown> {
const views = compactQuickVerifyRecordViews(record(payload.views));
const summary = {
reason: payload.reason,
status: payload.status,
businessStatus: payload.businessStatus ?? null,
elapsedMs: payload.elapsedMs,
failure: payload.failure,
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
analysis: compactQuickVerifyRecordAnalysis(payload.analysis),
promptSource: payload.promptSource,
steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [],
valuesRedacted: true,
};
const recordResult = callSentinelService(state, "POST", "/api/runs/record", {
runId: payload.runId,
scenarioId: payload.scenarioId,
status: payload.status,
observerId: payload.observerId,
stateDir: payload.stateDir,
reportJsonSha256: payload.reportJsonSha256,
findingCount: payload.findingCount,
artifactCount: payload.artifactCount,
summary,
businessStatus: payload.businessStatus ?? null,
findings: payload.findings,
views,
screenshot: payload.screenshot,
publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"),
maintenance: payload.reason === "maintenance-stop",
valuesRedacted: true,
}, 60);
return withWarnings({ ...payload, views, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : ["quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]);
}
function compactQuickVerifyRecordViews(views: Record<string, unknown>): Record<string, unknown> {
const compacted: Record<string, unknown> = {};
for (const [key, value] of Object.entries(views)) {
const item = record(value);
const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000;
compacted[key] = {
...item,
renderedText: boundQuickVerifyRecordText(item.renderedText, limit),
valuesRedacted: true,
};
}
return compacted;
}
function compactQuickVerifyRecordAnalysis(value: unknown): Record<string, unknown> | null {
const item = record(value);
if (Object.keys(item).length === 0) return null;
return {
ok: item.ok === true ? true : item.ok === false ? false : null,
reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null,
reason: stringAtNullable(item, "reason"),
stateDir: stringAtNullable(item, "stateDir"),
reportJsonSha256: stringAtNullable(item, "reportJsonSha256"),
reportMdSha256: stringAtNullable(item, "reportMdSha256"),
findingCount: numberAtNullable(item, "findingCount"),
artifactCount: numberAtNullable(item, "artifactCount"),
counts: compactQuickVerifyRecordCounts(record(item.counts)),
screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)),
findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [],
pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [],
valuesRedacted: true,
};
}
function compactQuickVerifyRecordCounts(value: Record<string, unknown>): Record<string, unknown> {
return {
samples: numberAtNullable(value, "samples"),
control: numberAtNullable(value, "control"),
network: numberAtNullable(value, "network"),
console: numberAtNullable(value, "console"),
errors: numberAtNullable(value, "errors"),
artifacts: numberAtNullable(value, "artifacts"),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordScreenshot(value: Record<string, unknown>): Record<string, unknown> | null {
if (Object.keys(value).length === 0) return null;
return {
path: stringAtNullable(value, "path"),
sha256: stringAtNullable(value, "sha256"),
bytes: numberAtNullable(value, "bytes"),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordFinding(value: unknown): Record<string, unknown> {
const item = record(value);
return {
id: stringAtNullable(item, "id"),
kind: stringAtNullable(item, "kind"),
code: stringAtNullable(item, "code"),
severity: stringAtNullable(item, "severity"),
level: stringAtNullable(item, "level"),
count: numberAtNullable(item, "count"),
summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220),
rootCause: boundQuickVerifyRecordText(item.rootCause, 140),
rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90),
rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40),
nextAction: boundQuickVerifyRecordText(item.nextAction, 240),
evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence),
rootCauseSignals: compactQuickVerifyRootCauseSignals(item.rootCauseSignals),
timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100),
timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60),
timingAlert: item.timingAlert === true,
blocking: item.blocking === true,
valuesRedacted: true,
};
}
function compactQuickVerifyRootCauseSignals(value: unknown): Record<string, unknown> | null {
const item = record(value);
const keys = [
"sessionListReadCount",
"traceEventsReadCount",
"webPerformanceBeaconFailureCount",
"eventSourceFailureCount",
"requestFailedCount",
"httpErrorCount",
"consoleAlertCount",
"requestfailedTop",
"httpStatusTop",
];
const compact: Record<string, unknown> = {};
for (const key of keys) {
const raw = item[key];
if (raw === null || raw === undefined) continue;
compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
}
return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true };
}
function compactQuickVerifyFindingEvidence(value: unknown): string | null {
const item = record(value);
if (Object.keys(item).length === 0) return null;
const keys = [
"http404Count",
"responseErrorCount",
"requestFailedCount",
"statuses",
"afterProjectedSeqs",
"sinceSeqs",
"traceIds",
"maxFallbackRatio",
"maxFallbackTitleCount",
"overThresholdSampleCount",
"majorityFallbackSampleCount",
];
const compact: Record<string, unknown> = {};
for (const key of keys) {
const raw = item[key];
if (raw === null || raw === undefined) continue;
compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw;
}
return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240);
}
function compactQuickVerifyRecordStep(value: unknown): Record<string, unknown> {
const item = record(value);
return {
phase: stringAtNullable(item, "phase"),
ok: item.ok === true ? true : item.ok === false ? false : null,
promptIndex: numberAtNullable(item, "promptIndex"),
checkId: stringAtNullable(item, "checkId"),
failure: stringAtNullable(item, "failure"),
result: compactQuickVerifyRecordStepResult(record(item.result)),
valuesRedacted: true,
};
}
function compactQuickVerifyRecordStepResult(value: Record<string, unknown>): Record<string, unknown> {
return {
ok: value.ok === true ? true : value.ok === false ? false : null,
status: stringAtNullable(value, "status"),
view: stringAtNullable(value, "view"),
exitCode: numberAtNullable(value, "exitCode"),
timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null,
stdoutBytes: numberAtNullable(value, "stdoutBytes"),
stderrBytes: numberAtNullable(value, "stderrBytes"),
stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240),
stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240),
valuesRedacted: true,
};
}
function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null {
if (typeof value !== "string") return null;
if (value.length <= maxChars) return value;
return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`;
}
function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record<string, unknown> {
if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true };
const script = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
"node - \"$state_dir\" <<'NODE'",
"const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');",
"const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');",
"const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);",
"const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;",
"const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);",
"const compactRootCauseSignals=(value)=>{const v=rec(value); const keys=['sessionListReadCount','traceEventsReadCount','webPerformanceBeaconFailureCount','eventSourceFailureCount','requestFailedCount','httpErrorCount','consoleAlertCount','requestfailedTop','httpStatusTop']; const out={}; for(const key of keys){if(v[key]!=null)out[key]=Array.isArray(v[key])?v[key].slice(0,8):v[key];} if(Object.keys(out).length===0)return null; out.valuesRedacted=true; return out;};",
"let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}",
"let artifactCount=0; let screenshot=null;",
"function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}",
"walk(stateDir);",
"const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,rootCauseSignals:compactRootCauseSignals(v.rootCauseSignals),blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});",
"const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});",
"console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));",
"NODE",
].join("\n");
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
const parsed = parseJsonObject(result.stdout);
return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true };
}
function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: string, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record<string, unknown> {
const observations: Record<string, unknown>[] = [];
const indexEntry = readLocalObserveIndex(observerId);
if (indexEntry === null) {
return {
ok: false,
failure: "observe-index-entry-missing",
observerId,
valuesRedacted: true,
};
}
const pollSleepMs = Math.max(250, Math.min(500, Math.trunc(pollIntervalMs / 2) || 250));
while (Date.now() < deadline) {
const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now()));
const script = quickVerifyObserverStartupWaitScript(indexEntry.stateDir, waitMs, pollSleepMs);
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 });
const payload = parseJsonObject(result.stdout);
if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record));
const terminalPayload = {
observerId,
stateDir: indexEntry.stateDir,
status: typeof payload?.status === "string" ? payload.status : null,
heartbeatStatus: typeof payload?.heartbeatStatus === "string" ? payload.heartbeatStatus : null,
startup: record(payload?.startup),
observations: observations.slice(-6),
waitResult: compactCommand(result),
valuesRedacted: true,
};
if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-startup-wait-chunk-timeout") {
return {
ok: false,
failure: text(payload?.failure ?? "quick-verify-startup-artifact-wait-failed"),
...terminalPayload,
};
}
if (payload.ok === true) return { ok: true, ...terminalPayload };
}
return {
ok: false,
failure: "quick-verify-timeout-over-budget",
observerId,
stateDir: indexEntry.stateDir,
observations: observations.slice(-6),
warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for the observe runner startup to finish before sending the first command.`],
valuesRedacted: true,
};
}
function quickVerifyObserverStartupWaitScript(stateDir: string, timeoutMs: number, pollSleepMs: number): string {
return [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
`timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`,
`poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`,
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }",
"node - \"$state_dir\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'",
"const fs = require('node:fs');",
"const path = require('node:path');",
"const dir = process.argv[2];",
"const timeoutMs = Number(process.argv[3]);",
"const pollMs = Number(process.argv[4]);",
"const startedAt = Date.now();",
"const startupIds = ['startup-login', 'startup-goto', 'startup-observer-goto'];",
"const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };",
"const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };",
"const clip = (value, limit = 160) => value == null ? null : String(value).replace(/\\s+/gu, ' ').trim().slice(0, limit);",
"const norm = (value) => String(value || '').trim().toLowerCase().replace(/_/gu, '-');",
"const terminal = new Set(['failed', 'force-stopped', 'stopped', 'abandoned', 'completed']);",
"function commandEvents(control, id) { return control.filter((item) => item && item.commandId === id); }",
"function lastPhase(control, id) { return commandEvents(control, id).filter((item) => typeof item.phase === 'string').slice(-1)[0]?.phase || null; }",
"function firstFailedStartup(control) { return control.filter((item) => item && startupIds.includes(item.commandId) && item.phase === 'failed').slice(-1)[0] || null; }",
"function rowFor() {",
" const heartbeat = readJson('heartbeat.json') || {};",
" const manifest = readJson('manifest.json') || {};",
" const control = readJsonl('control.jsonl');",
" const phases = Object.fromEntries(startupIds.map((id) => [id, lastPhase(control, id)]));",
" const failed = firstFailedStartup(control);",
" const heartbeatStatus = norm(heartbeat.status || manifest.status);",
" const ready = startupIds.every((id) => phases[id] === 'completed') && heartbeatStatus === 'running';",
" const terminalBeforeReady = !ready && terminal.has(heartbeatStatus);",
" const degraded = control.filter((item) => item && item.type === 'observer-startup-degraded').slice(-1)[0] || null;",
" return {",
" ok: ready,",
" status: ready ? 'startup-ready' : terminalBeforeReady ? 'startup-terminal' : 'startup-waiting',",
" heartbeatStatus,",
" startup: { phases, failedCommandId: failed?.commandId || null, failedType: failed?.type || null, failedMessage: clip(failed?.detail?.error?.message || failed?.detail?.error || failed?.error?.message), observerStartupDegraded: !!degraded, degradedReason: clip(degraded?.reason || degraded?.result?.failureKind || degraded?.result?.reason), sampleSeq: heartbeat.sampleSeq ?? null, commandSeq: heartbeat.commandSeq ?? null, currentUrl: clip(heartbeat.currentUrl, 180), observerUrl: clip(heartbeat.observerUrl, 180), valuesRedacted: true },",
" valuesRedacted: true",
" };",
"}",
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
"(async () => {",
" const observations = [];",
" while (Date.now() - startedAt <= timeoutMs) {",
" const row = rowFor();",
" observations.push(row);",
" if (row.ok === true) { console.log(JSON.stringify({ ok: true, ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
" if (row.startup.failedCommandId) { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-command-failed', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
" if (row.status === 'startup-terminal') { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-terminal', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
" await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));",
" }",
" const row = rowFor();",
" observations.push(row);",
" console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-chunk-timeout', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));",
"})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });",
"NODE",
].join("\n");
}
function collectObserveView(state: SentinelCicdState, observerId: string, view: "turn-summary" | "trace-frame", turn: number | null, timeoutSeconds: number): Record<string, unknown> {
const args = ["web-probe", "observe", "collect", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--view", view, "--command-timeout-seconds", String(Math.max(5, Math.min(timeoutSeconds, 55))), "--raw", "--compact-raw"];
if (turn !== null) args.push("--turn", String(turn));
const result = runChildCli(args, timeoutSeconds);
const payload = cliDataPayload(result.parsed);
const collect = record(payload.collect);
return {
ok: result.ok && result.parsed !== null && payload.ok !== false && collect.ok !== false,
view,
renderedText: typeof collect.renderedText === "string" ? collect.renderedText : typeof payload.renderedText === "string" ? payload.renderedText : String(record(result.result).stdoutTail ?? record(result.result).stdoutPreview ?? ""),
collect,
payload,
result: result.result,
valuesRedacted: true,
};
}
export function runChildCli(args: string[], timeoutSeconds: number, input?: string, env?: NodeJS.ProcessEnv): ChildCliResult {
const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, {
input,
env: env === undefined ? undefined : { ...process.env, ...env },
timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000,
});
return {
ok: result.exitCode === 0 && !result.timedOut,
parsed: parseJsonObject(result.stdout),
result: compactCommandWithTail(result),
};
}
function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number, chunkSeconds = 45): Record<string, unknown> {
const observations: Record<string, unknown>[] = [];
const indexEntry = readLocalObserveIndex(observerId);
if (indexEntry === null) {
return {
ok: false,
failure: "observe-index-entry-missing",
round: promptIndex,
observerId,
valuesRedacted: true,
};
}
const pollSleepMs = Math.max(1000, Math.min(3000, Math.trunc(pollIntervalMs * 2) || 1000));
while (Date.now() < deadline) {
const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now()));
const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs);
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 });
const payload = parseJsonObject(result.stdout);
if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record));
const status = typeof payload?.status === "string" ? payload.status : null;
const terminalPayload = {
round: promptIndex,
status,
traceId: payload?.traceId ?? null,
finalResponseEmpty: payload?.finalResponseEmpty === true,
composerReadyForTurn: payload?.composerReadyForTurn === true,
composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null,
observations: observations.slice(-6),
waitResult: compactCommand(result),
valuesRedacted: true,
};
if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") {
const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex);
if (fallback.ok === true) {
return {
ok: true,
...terminalPayload,
status: stringAtNullable(fallback, "status") ?? status,
traceId: stringAtNullable(fallback, "traceId") ?? payload?.traceId ?? null,
finalResponseEmpty: false,
fallback,
warnings: ["quick verify artifact wait command failed, but bounded turn-summary artifacts show this round completed; continuing validation."],
};
}
return {
ok: false,
failure: text(payload?.failure ?? "quick-verify-artifact-wait-failed"),
...terminalPayload,
fallback,
};
}
if (payload.ok === false && payload.failure === "quick-verify-wait-chunk-timeout") {
const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex);
if (fallback.ok === true) {
return {
ok: true,
...terminalPayload,
status: stringAtNullable(fallback, "status") ?? status,
traceId: stringAtNullable(fallback, "traceId") ?? payload.traceId ?? null,
finalResponseEmpty: false,
fallback,
warnings: ["quick verify wait chunk timed out, but bounded turn-summary artifacts show this round completed; continuing validation."],
};
}
}
if (payload.ok === true) return { ok: true, ...terminalPayload };
if (isQuickVerifyTurnSuccessful(status)) {
if (payload?.finalResponseEmpty !== true) return { ok: true, ...terminalPayload };
continue;
}
if (isQuickVerifyTurnTerminal(status)) {
return {
ok: false,
failure: "observe-turn-terminal-non-success",
...terminalPayload,
};
}
}
return {
ok: false,
failure: "quick-verify-timeout-over-budget",
round: promptIndex,
observations: observations.slice(-6),
warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for a submitted turn to become terminal; investigate Code Agent multi-round continuity before retrying.`],
valuesRedacted: true,
};
}
function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, timeoutMs: number, pollSleepMs: number): string {
return [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
`prompt_index=${shellQuote(String(promptIndex))}`,
`timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`,
`poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`,
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }",
"node - \"$state_dir\" \"$prompt_index\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'",
"const fs = require('node:fs');",
"const path = require('node:path');",
"const dir = process.argv[2];",
"const promptIndex = Number(process.argv[3]);",
"const timeoutMs = Number(process.argv[4]);",
"const pollMs = Number(process.argv[5]);",
"const startedAt = Date.now();",
"const short = (value, limit = 160) => String(value || '').replace(/\\s+/gu, ' ').trim().slice(0, limit);",
"const textOf = (value) => String(value?.text || value?.textPreview || value?.preview || '');",
"const arr = (value) => Array.isArray(value) ? value : [];",
"const unique = (values) => Array.from(new Set(values.filter(Boolean)));",
"const numOrNull = (value) => { const n = Number(value); return Number.isFinite(n) ? n : null; };",
"const tsMs = (value) => { const ms = Date.parse(String(value || '')); return Number.isFinite(ms) ? ms : null; };",
"const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };",
"const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };",
"const readJsonlTail = (rel, maxBytes = 2000000) => {",
" try {",
" const file = path.join(dir, rel);",
" const stat = fs.statSync(file);",
" const start = Math.max(0, stat.size - maxBytes);",
" const length = stat.size - start;",
" const fd = fs.openSync(file, 'r');",
" try {",
" const buffer = Buffer.alloc(length);",
" fs.readSync(fd, buffer, 0, length, start);",
" const lines = buffer.toString('utf8').split(/\\r?\\n/u);",
" if (start > 0) lines.shift();",
" return lines.filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);",
" } finally {",
" fs.closeSync(fd);",
" }",
" } catch {",
" return [];",
" }",
"};",
"const readDone = (id) => id ? readJson(path.join('commands', 'done', `${id}.json`)) : null;",
"const readFailed = (id) => id ? readJson(path.join('commands', 'failed', `${id}.json`)) : null;",
"function sessionIdFromUrl(value) { const match = String(value || '').match(/\\/workbench\\/sessions\\/(ses_[A-Za-z0-9_-]+)/u); return match ? match[1] : null; }",
"function commandSessionId(item) { const done = readDone(item?.commandId); return item?.sessionId || item?.detail?.sessionId || item?.input?.sessionId || item?.result?.sessionId || done?.result?.sessionId || done?.result?.observer?.sessionId || sessionIdFromUrl(item?.afterUrl) || sessionIdFromUrl(item?.detail?.afterUrl) || sessionIdFromUrl(done?.result?.afterUrl) || null; }",
"function commandTextHash(item) { const done = readDone(item?.commandId); return item?.detail?.textHash || item?.input?.textHash || item?.result?.textHash || done?.result?.textHash || null; }",
"function firstTraceId(value) { const match = String(value || '').match(/\\btrc_[A-Za-z0-9_-]+\\b/u); return match ? match[0] : null; }",
"function commandTraceId(item) { const done = readDone(item?.commandId); return item?.traceId || item?.detail?.chatSubmit?.traceId || item?.detail?.traceId || item?.input?.traceId || item?.result?.chatSubmit?.traceId || done?.result?.chatSubmit?.traceId || done?.result?.traceId || null; }",
"function itemTraceId(item) { return item?.traceId || firstTraceId(textOf(item)) || null; }",
"function completedNewSessionIdsBefore(control, ts) { const limit = tsMs(ts); return control.filter((item) => item.type === 'newSession' && item.phase === 'completed').filter((item) => limit === null || tsMs(item.ts) === null || tsMs(item.ts) <= limit).map(commandSessionId).filter(Boolean); }",
"function authoritativeSessionIdForPrompts(control, prompts) { const ids = completedNewSessionIdsBefore(control, prompts[0]?.firstTs || null); return ids.slice(-1)[0] || unique(prompts.map((item) => item.sessionId))[0] || null; }",
"function promptCommands(control) {",
" const map = new Map();",
" for (const item of control) {",
" if (item.type !== 'sendPrompt' || !['started', 'completed', 'failed'].includes(item.phase)) continue;",
" const id = item.commandId || item.seq || String(map.size + 1);",
" const existing = map.get(id) || {};",
" map.set(id, { ...existing, ...item, input: { ...(existing.input || {}), ...(item.input || {}) }, sessionId: existing.sessionId || commandSessionId(item), traceId: existing.traceId || commandTraceId(item), textHash: existing.textHash || commandTextHash(item), firstTs: existing.firstTs || item.ts, lastTs: item.ts });",
" }",
" const prompts = Array.from(map.values()).filter((item) => tsMs(item.firstTs) !== null).sort((a, b) => tsMs(a.firstTs) - tsMs(b.firstTs));",
" const sessionId = authoritativeSessionIdForPrompts(control, prompts);",
" if (!sessionId) return prompts;",
" const scoped = prompts.filter((item) => item.sessionId === sessionId);",
" return scoped.length > 0 ? scoped : prompts;",
"}",
"function segmentFor(samples, prompts, index) { const start = tsMs(prompts[index]?.firstTs); const end = index + 1 < prompts.length ? tsMs(prompts[index + 1].firstTs) : Infinity; return samples.filter((sample) => { const ms = tsMs(sample.ts); return ms !== null && ms >= start && ms < end; }); }",
"function entryGroups(sample) { return [...arr(sample.turns).map((item) => ({ group: 'turn', item })), ...arr(sample.traceRows).map((item) => ({ group: 'traceRow', item })), ...arr(sample.messages).map((item) => ({ group: 'message', item }))]; }",
"function traceIdsFromSamples(items) { const ids = []; for (const sample of items) for (const entry of entryGroups(sample)) { const id = itemTraceId(entry.item); if (id) ids.push(id); } return unique(ids); }",
"function chooseTraceId(segment, prompt) { const promptTraceId = commandTraceId(prompt) || prompt?.traceId || null; const ids = traceIdsFromSamples(segment); if (promptTraceId && (ids.length === 0 || ids.includes(promptTraceId))) return promptTraceId; return ids.slice(-1)[0] || promptTraceId || null; }",
"function traceIdForPromptUserMessage(items, prompt) {",
" const hash = prompt?.textHash || commandTextHash(prompt);",
" if (!hash) return null;",
" for (const sample of items) {",
" for (const message of arr(sample.messages)) {",
" const role = String(message?.role || message?.dataRole || message?.messageRole || '').toLowerCase();",
" if (role && !/user/u.test(role)) continue;",
" if (message?.textHash === hash) {",
" const id = itemTraceId(message);",
" if (id) return id;",
" }",
" }",
" }",
" return null;",
"}",
"function traceEntries(items, traceId) { const entries = []; for (const sample of items) for (const entry of entryGroups(sample)) { const text = textOf(entry.item); const id = itemTraceId(entry.item); if (!traceId || id === traceId || text.includes(traceId)) entries.push({ ...entry, sample, text }); } return entries; }",
"function normalizeLifecycleStatus(value) { const raw = String(value || '').trim().toLowerCase(); if (/^(canceled|cancelled)$/u.test(raw)) return 'canceled'; if (/^(failed|failure|error)$/u.test(raw)) return 'failed'; if (/^(completed|complete|succeeded|success|terminal)$/u.test(raw)) return 'completed'; if (/^(running|admitting|queued|pending|in_progress|in-progress)$/u.test(raw)) return 'running'; return null; }",
"function statusFor(items, traceId) { const entries = traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) }))); const lastTurn = entries.filter((entry) => entry.group === 'turn').slice(-1)[0]?.item || null; const turnStatus = normalizeLifecycleStatus(lastTurn?.status); if (turnStatus) return turnStatus; const lastMessage = entries.filter((entry) => entry.group === 'message').slice(-1)[0]?.item || null; return normalizeLifecycleStatus(lastMessage?.status) || 'unknown'; }",
"function cleanFinalResponseText(value) { const raw = String(value || '').trim(); if (!raw) return ''; if (/^(completed|failed|canceled|cancelled|轮次完成|轮次失败|轮次取消|已记录)$/iu.test(raw.replace(/\\s+/gu, ' '))) return ''; if (/^(admitted|run|ok|error)\\s+/iu.test(raw)) return ''; return raw; }",
"function finalResponseTextFromEntry(entry) {",
" const explicit = cleanFinalResponseText(entry.item?.finalResponse?.text || entry.item?.finalResponse?.preview || '');",
" if (explicit && !/^Code Agent\\s*耗时/iu.test(explicit)) return explicit;",
" if (entry.group !== 'message') return '';",
" const role = String(entry.item?.role || entry.item?.dataRole || entry.item?.messageRole || '').toLowerCase();",
" if (role && !/assistant|agent|system/u.test(role)) return '';",
" const text = cleanFinalResponseText(entry.text);",
" return text && !/^Code Agent\\s*耗时/iu.test(text) ? text : '';",
"}",
"function finalResponseEmpty(items, traceId) { if (!/^(completed|failed|canceled)$/u.test(statusFor(items, traceId))) return true; const entries = (traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) })))).slice().reverse(); for (const entry of entries) { if (finalResponseTextFromEntry(entry)) return false; } return true; }",
"function rowFor() {",
" const control = readJsonl('control.jsonl');",
" const samples = readJsonlTail('samples.jsonl');",
" const prompts = promptCommands(control);",
" const prompt = prompts[promptIndex - 1] || null;",
" if (!prompt) return { ok: true, round: promptIndex, status: 'command-pending', traceId: null, finalResponseEmpty: true, lastSeq: null, lastTs: null, promptMissing: true, valuesRedacted: true };",
" const done = readDone(prompt.commandId);",
" const failed = readFailed(prompt.commandId);",
" if (failed) return { ok: false, failure: 'observe-command-sendPrompt-failed', round: promptIndex, status: 'command-failed', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandFailure: short(failed.error?.message || failed.failure || failed.status || 'command failed'), valuesRedacted: true };",
" const promptTraceId = commandTraceId(prompt);",
" if (!done) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: promptTraceId || null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: !promptTraceId, valuesRedacted: true };",
" const segment = segmentFor(samples, prompts, promptIndex - 1);",
" const controlSegment = segment.filter((sample) => sample.pageRole === 'control');",
" const traceId = promptTraceId || traceIdForPromptUserMessage(segment, prompt) || chooseTraceId(controlSegment, prompt) || chooseTraceId(segment, prompt);",
" if (!traceId) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: true, segmentSampleCount: segment.length, valuesRedacted: true };",
" const controlTraceSegment = traceId ? controlSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)) : [];",
" const statusSegment = controlTraceSegment.length > 0 ? controlSegment : segment;",
" const status = statusFor(statusSegment, traceId);",
" const sampleForTrace = traceId ? statusSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;",
" const lastSample = sampleForTrace || segment.slice(-1)[0] || null;",
" const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;",
" const composer = composerSample?.composer || {};",
" const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';",
" return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(statusSegment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, sampleScope: controlSegment.length > 0 ? 'control' : 'all', segmentSampleCount: segment.length, statusSampleCount: statusSegment.length, source: 'observe-artifact-wait-script', valuesRedacted: true };",
"}",
"function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }",
"function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }",
"function terminal(value) { return ['completed', 'succeeded', 'success', 'failed', 'error', 'blocked', 'timeout', 'canceled', 'cancelled', 'terminal'].includes(norm(value)); }",
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
"(async () => {",
" const observations = [];",
" while (Date.now() - startedAt <= timeoutMs) {",
" const row = rowFor();",
" observations.push(row);",
" if (row.ok === false) { console.log(JSON.stringify({ ...row, ok: false, failure: row.failure || 'quick-verify-artifact-row-failed', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
" if (successful(row.status) && row.finalResponseEmpty !== true) { console.log(JSON.stringify({ ...row, ok: true, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
" if (!successful(row.status) && terminal(row.status)) { console.log(JSON.stringify({ ...row, ok: false, failure: 'observe-turn-terminal-non-success', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }",
" await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));",
" }",
" const row = rowFor();",
" observations.push(row);",
" console.log(JSON.stringify({ ...row, ok: false, failure: 'quick-verify-wait-chunk-timeout', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));",
"})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-artifact-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });",
"NODE",
].join("\n");
}
function isQuickVerifyTurnSuccessful(value: string | null): boolean {
const status = normalizeQuickVerifyStatus(value);
return status === "completed" || status === "succeeded" || status === "success";
}
function isQuickVerifyTurnTerminal(value: string | null): boolean {
const status = normalizeQuickVerifyStatus(value);
return status === "completed"
|| status === "succeeded"
|| status === "success"
|| status === "failed"
|| status === "error"
|| status === "blocked"
|| status === "timeout"
|| status === "canceled"
|| status === "cancelled"
|| status === "terminal";
}
function normalizeQuickVerifyStatus(value: string | null): string {
return String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
}
function cliDataPayload(parsed: Record<string, unknown> | null): Record<string, unknown> {
const root = record(parsed);
const payload = isRecord(root.data) ? root.data : root;
return cliDumpPayload(payload) ?? payload;
}
function cliDumpPayload(payload: Record<string, unknown>): Record<string, unknown> | null {
if (payload.outputTruncated !== true) return null;
const dumpPath = stringAtNullable(record(payload.dump), "path");
if (dumpPath === null || !existsSync(dumpPath)) return null;
const dumped = parseJsonObject(readFileSync(dumpPath, "utf8"));
if (dumped === null) return null;
const dumpedRoot = record(dumped);
return isRecord(dumpedRoot.data) ? dumpedRoot.data : dumpedRoot;
}
function findScenario(state: SentinelCicdState, scenarioId: string): Record<string, unknown> | null {
const scenarios = readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.scenarios);
const items = Array.isArray(scenarios) ? scenarios : isRecord(scenarios) ? [scenarios] : [];
return items.map(record).find((item) => item.id === scenarioId) ?? null;
}
function readPromptSetForScenario(state: SentinelCicdState, scenario: Record<string, unknown>): { ok: true; prompts: string[]; summary: Record<string, unknown> } | { ok: false; error: string; summary: Record<string, unknown> } {
const promptSetRef = stringAt(scenario, "promptSetRef");
const promptSet = recordTarget(readWebProbeSentinelConfigRefTarget(state.spec, promptSetRef), promptSetRef);
const sourceRef = stringAt(promptSet, "promptSourceRef");
const key = stringAt(promptSet, "promptSourceKey");
const paths = secretSourcePaths(sourceRef);
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true };
const runtimeRaw = process.env[key];
const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
const raw = values[key] ?? runtimeRaw;
const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null;
if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary };
if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary };
const parsed = parsePromptJson(raw);
if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary };
return {
ok: true,
prompts: parsed,
summary: {
...summary,
sourceMode,
promptCount: parsed.length,
promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))),
promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`),
promptTextBytes: parsed.map((item) => Buffer.byteLength(item)),
valuesRedacted: true,
},
};
}
function parsePromptJson(raw: string): string[] {
try {
const parsed = JSON.parse(raw) as unknown;
if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string" && item.length > 0);
const recordValue = record(parsed);
if (Array.isArray(recordValue.prompts)) return recordValue.prompts.filter((item): item is string => typeof item === "string" && item.length > 0);
if (typeof recordValue.prompt === "string" && recordValue.prompt.length > 0) return [recordValue.prompt];
} catch {
if (raw.trim().length > 0) return [raw];
}
return [];
}
function readLocalObserveIndex(observerId: string): { stateDir: string } | null {
const path = rootPath(".state/web-observe/index.json");
if (!existsSync(path)) return null;
const parsed = parseJsonObject(readFileSync(path, "utf8"));
const entry = record(parsed?.[observerId]);
const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : null;
return stateDir === null ? null : { stateDir };
}
function observerIdFromText(textValue: string): string | null {
return /\bwebobs-[a-z0-9-]+\b/iu.exec(textValue)?.[0] ?? null;
}
export function remainingSeconds(deadline: number, cap: number): number {
return Math.max(5, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000)));
}
export function metricNames(textValue: unknown): string[] {
if (typeof textValue !== "string") return [];
return textValue.split(/\r?\n/u).map((line) => /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line)?.[1]).filter((item): item is string => typeof item === "string");
}
export function validationBlocker(health: Record<string, unknown>, metrics: Record<string, unknown>, report: Record<string, unknown>, publicExposure: Record<string, unknown>, publicDashboard: Record<string, unknown>, quickVerify: Record<string, unknown> | null): Record<string, unknown> {
const blockers = [];
if (!health.ok || record(health.bodyJson).ok !== true) blockers.push("health");
if (!metrics.ok || !metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health")) blockers.push("metrics");
if (!report.ok) blockers.push("recent-report");
if (publicExposure.ok !== true) blockers.push("public-exposure");
if (publicDashboard.ok !== true) blockers.push("public-dashboard");
if (quickVerify !== null && quickVerify.ok !== true) blockers.push("quick-verify");
return { code: "sentinel-validation-failed", blockers, valuesRedacted: true };
}
export function serviceUnavailableBlocker(state: SentinelCicdState): Record<string, unknown> {
return {
code: "sentinel-service-unavailable",
policy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"),
reason: "sentinel service must be reachable through k3s internal Service DNS before quick verify can run; no public/fallback path is used.",
retry: `bun scripts/cli.ts web-probe sentinel validate --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
valuesRedacted: true,
};
}
export function sentinelP5Next(state: SentinelCicdState): Record<string, string> {
const node = state.spec.nodeId;
const lane = state.spec.lane;
const suffix = sentinelCliSuffix(state);
return {
validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`,
quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`,
maintenanceStart: `bun scripts/cli.ts web-probe sentinel maintenance start --node ${node} --lane ${lane}${suffix} --confirm --wait`,
maintenanceStop: `bun scripts/cli.ts web-probe sentinel maintenance stop --node ${node} --lane ${lane}${suffix} --confirm --wait`,
report: `bun scripts/cli.ts web-probe sentinel report --node ${node} --lane ${lane}${suffix} --view summary`,
};
}
function isSafeRelativeStateDir(value: string): boolean {
return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes("..");
}
function mergeFindingRecords(primary: readonly Record<string, unknown>[], extra: readonly Record<string, unknown>[]): Record<string, unknown>[] {
const merged: Record<string, unknown>[] = [];
const seen = new Set<string>();
for (const item of [...primary, ...extra]) {
const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? stringAtNullable(item, "finding_id") ?? "finding";
const severity = stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "unknown";
const key = `${id}\0${severity}`;
if (seen.has(key)) continue;
seen.add(key);
merged.push(item);
}
return merged;
}
function isQuickVerifyBlockingFinding(item: Record<string, unknown>): boolean {
const severity = (stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "").toLowerCase();
if (!["critical", "red", "fatal", "error", "failed", "blocked"].includes(severity)) return false;
const id = (stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? "").toLowerCase();
if (id === "observer-command-failed") return observerCommandFailureBlocks(item);
return [
"quick-verify-no-business-turn",
"quick-verify-command-sequence-failed",
"quick-verify-observer-start-failed",
"quick-verify-account-secret-missing",
"prompt-chat-submit-failed",
"route-active-session-mismatch",
"final-response-flicker",
"round-completion-final-response-missing",
"turn-trace-id-missing",
"no-samples",
"jsonl-read-issues",
].includes(id);
}
function observerCommandFailureBlocks(item: Record<string, unknown>): boolean {
const commands = Array.isArray(item.commands) ? item.commands.map(record) : [];
if (commands.length === 0) return true;
return commands.some((command) => {
const type = (stringAtNullable(command, "type") ?? "").toLowerCase();
if (["stop", "cancel", "mark", "screenshot"].includes(type)) return false;
return true;
});
}
function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null): Record<string, unknown>[] {
if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return [];
const rendered = [
typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "",
typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "",
].join("\n");
const noPromptScenario = promptIndex <= 0;
if (noPromptScenario && failure === null) return [];
if (noPromptScenario && failure !== null) {
const observerStartFailure = failure === "observe-start-failed";
return [{
id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed",
severity: "red",
count: 1,
summary: observerStartFailure
? "quick verify observer failed to start before the no-prompt scenario could run."
: "quick verify no-prompt command sequence failed before the account/session workflow completed.",
failure,
promptIndex,
valuesRedacted: true,
}];
}
const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered);
const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered);
if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return [];
return [{
id: "quick-verify-no-business-turn",
severity: "red",
count: 1,
summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.",
failure: failure ?? null,
promptIndex,
valuesRedacted: true,
}];
}
function quickVerifyCompletedTurnSummaryRow(promptIndex: number, turnSummary: Record<string, unknown> | null): Record<string, unknown> | null {
const rows = Array.isArray(record(turnSummary?.collect).rows) ? record(turnSummary?.collect).rows.map(record) : [];
const scopedRows = promptIndex > 0 ? rows.filter((row) => numberAtNullable(row, "round") === promptIndex) : rows;
return scopedRows.find((row) => {
const finalResponse = record(row.finalResponse);
return isQuickVerifyTurnSuccessful(stringAtNullable(row, "status"))
&& stringAtNullable(row, "traceId") !== null
&& finalResponse.empty !== true;
}) ?? null;
}
function quickVerifyTurnSummaryFallback(state: SentinelCicdState, observerId: string, promptIndex: number): Record<string, unknown> {
const turnSummary = collectObserveView(state, observerId, "turn-summary", null, 25);
const row = quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary);
const rows = Array.isArray(record(turnSummary.collect).rows) ? record(turnSummary.collect).rows.map(record) : [];
if (row === null) {
return {
ok: false,
source: "turn-summary-fallback",
collectOk: turnSummary.ok === true,
rowCount: rows.length,
promptIndex,
result: turnSummary.result ?? null,
valuesRedacted: true,
};
}
const finalResponse = record(row.finalResponse);
return {
ok: true,
source: "turn-summary-fallback",
collectOk: turnSummary.ok === true,
rowCount: rows.length,
promptIndex,
status: stringAtNullable(row, "status"),
traceId: stringAtNullable(row, "traceId"),
finalResponseEmpty: finalResponse.empty === true,
finalResponseBytes: numberAtNullable(finalResponse, "textBytes"),
result: turnSummary.result ?? null,
valuesRedacted: true,
};
}
function quickVerifyHasDurableBusinessTurn(promptIndex: number, turnSummary: Record<string, unknown> | null, traceFrame: Record<string, unknown> | null): boolean {
if (quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary) !== null) return true;
const renderedTrace = typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "";
if (!renderedTrace) return false;
if (/Final Response\s*\n\s*\(空内容\)/iu.test(renderedTrace)) return false;
return /Code Agent[^\n]*completed|轮次完成（总耗时/iu.test(renderedTrace)
&& /Final Response\s*\n\s*\S/iu.test(renderedTrace)
&& !/无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(renderedTrace);
}
function quickVerifyBusinessStatus(
failure: string | null,
promptIndex: number,
turnSummary: Record<string, unknown> | null,
traceFrame: Record<string, unknown> | null,
elapsedMs: unknown,
budgetSeconds: number,
): Record<string, unknown> {
const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame);
const elapsed = typeof elapsedMs === "number" && Number.isFinite(elapsedMs) ? elapsedMs : null;
const budgetMs = Math.max(0, budgetSeconds) * 1000;
const budgetExceeded = elapsed !== null && budgetMs > 0 && elapsed > budgetMs;
const observerTimeout = budgetExceeded || (failure !== null && isRecoverableQuickVerifyWaitFailure(failure));
const status = durableBusinessTurn
? "business-turn-completed"
: observerTimeout
? "observer-timeout"
: "scenario-incomplete";
return {
status,
durableBusinessTurn,
observerTimeout,
scenarioComplete: durableBusinessTurn,
failure: failure ?? null,
promptIndex,
elapsedMs: elapsed,
budgetSeconds,
sourceOfTruth: durableBusinessTurn ? "turn-summary-or-trace-frame" : observerTimeout ? "runner-wait-budget" : "control-findings",
valuesRedacted: true,
};
}
function isRecoverableQuickVerifyWaitFailure(failure: string): boolean {
return failure === "quick-verify-wait-chunk-timeout"
|| failure === "quick-verify-timeout-over-budget"
|| failure === "observe-turn-terminal-wait-failed";
}
function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } {
return {
...compactCommand(result),
stdoutPreview: result.stdout.trim().slice(0, 1200),
stderrPreview: result.stderr.trim().slice(0, 1200),
stdoutTail: result.stdout.trim().slice(-4000),
stderrTail: result.stderr.trim().slice(-4000),
};
}
function renderQuickVerifySummary(input: Record<string, unknown>): string {
const artifact = record(input.artifactSummary);
const findings = Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : [];
return [
"Web Probe Sentinel Quick Verify",
"=======================================================",
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
`report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${artifact.findingCount ?? findings.length}`,
`publicOrigin=${input.publicOrigin ?? "-"}`,
"",
"Findings",
findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"),
].join("\n");
}
function renderAuthSessionSwitchQuickVerifySummary(input: Record<string, unknown>): string {
const artifact = record(input.artifactSummary);
const accountEnv = record(input.accountEnv);
const findingRows = Array.isArray(input.findings)
? input.findings.map(record).slice(0, 8)
: Array.isArray(artifact.findings)
? artifact.findings.map(record).slice(0, 8)
: [];
const steps = Array.isArray(input.steps) ? input.steps.map(record) : [];
const commandSteps = steps
.filter((step) => {
const phase = stringAtNullable(step, "phase") ?? "";
return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env";
})
.map((step) => {
const phase = stringAtNullable(step, "phase") ?? "-";
const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-";
const result = record(step.result);
const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-";
return `${phase} ${ok} ${status}`;
});
return [
"Auth Session Switch Quick Verify",
"=======================================================",
`run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`,
`status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`,
`accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`,
"",
"Command Sequence",
commandSteps.length === 0 ? "-" : commandSteps.join("\n"),
"",
"Findings",
findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"),
].join("\n");
}
function formatQuickVerifyTimingSuffix(item: Record<string, unknown>): string {
const status = stringAtNullable(item, "timingStatus");
const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind");
if (status === null && source === null) return "";
return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`;
}
Reference in New Issue View Git Blame Copy Permalink