From 5d7b5d7cae6bbd58f91959e588adbc13561c050f Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 28 Jun 2026 04:23:17 +0000 Subject: [PATCH] fix: keep sentinel quick verify waiting on durable turns --- .../scenarios.fake-echo.workbench.yaml | 3 ++ scripts/src/hwlab-node-web-sentinel-cicd.ts | 41 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/config/hwlab-web-probe-sentinel/scenarios.fake-echo.workbench.yaml b/config/hwlab-web-probe-sentinel/scenarios.fake-echo.workbench.yaml index 8bed53e1..3e42e213 100644 --- a/config/hwlab-web-probe-sentinel/scenarios.fake-echo.workbench.yaml +++ b/config/hwlab-web-probe-sentinel/scenarios.fake-echo.workbench.yaml @@ -14,6 +14,9 @@ sentinel: sampleIntervalMs: 1000 screenshotIntervalMs: 60000 maxRunSeconds: 1200 + commandWaitMs: 180000 + commandTimeoutSeconds: 180 + turnWaitChunkSeconds: 120 providerProfile: fake-echo providerProfileMode: exact promptSetRef: config/hwlab-web-probe-sentinel/prompt-set.fake-echo.yaml#sentinel.promptSet diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index cd626a55..1350263b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -2625,7 +2625,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou promptSource: prompts.summary, })); } - const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", "55000", "--command-timeout-seconds", String(remainingSeconds(deadline, 55))]; + const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000)); + const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55)); + const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55)); + const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))]; if (type === "selectProvider") args.push("--provider", stringAt(item, "provider")); if (type === "loginAccount" || type === "listSessions" || type === "logout") { const accountId = stringAtNullable(item, "accountId"); @@ -2644,10 +2647,36 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou } appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" }); printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat }); - const commandResult = runChildCli(args, remainingSeconds(deadline, 60)); + const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10))); steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() }); if (!commandResult.ok) { + if (type === "sendPrompt") { + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); + const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); + steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult }); + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() }); + if (delayedWaitResult.ok === true) { + const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); + nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); + printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); + if (invariantResult.ok !== true) { + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex, + steps, + failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"), + promptSource: prompts.summary, + elapsedMs: elapsedMs(), + warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()), + })); + } + continue; + } + } return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { runId, scenarioId, @@ -2662,8 +2691,8 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou })); } if (type === "sendPrompt") { - printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, 55) }); - const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds); + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); + const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult }); printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() }); if (waitResult.ok !== true) { @@ -3935,7 +3964,7 @@ function runChildCli(args: string[], timeoutSeconds: number, input?: string, env }; } -function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record { +function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number, chunkSeconds = 45): Record { const observations: Record[] = []; const indexEntry = readLocalObserveIndex(observerId); if (indexEntry === null) { @@ -3949,7 +3978,7 @@ function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: stri } const pollSleepMs = Math.max(1000, Math.min(3000, Math.trunc(pollIntervalMs * 2) || 1000)); while (Date.now() < deadline) { - const waitMs = Math.max(1000, Math.min(45_000, deadline - Date.now())); + const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now())); const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs); const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 }); const payload = parseJsonObject(result.stdout);