mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: keep sentinel quick verify waiting on durable turns

Browse Source
This commit is contained in:
Codex
2026-06-28 04:23:17 +00:00
parent fb833247ec
commit 5d7b5d7cae
2 changed files with 38 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/hwlab-web-probe-sentinel/scenarios.fake-echo.workbench.yaml
+3
View File
@@ -14,6 +14,9 @@ sentinel:
sampleIntervalMs: 1000
screenshotIntervalMs: 60000
maxRunSeconds: 1200
commandWaitMs: 180000
commandTimeoutSeconds: 180
turnWaitChunkSeconds: 120
providerProfile: fake-echo
providerProfileMode: exact
promptSetRef: config/hwlab-web-probe-sentinel/prompt-set.fake-echo.yaml#sentinel.promptSet
scripts/src/hwlab-node-web-sentinel-cicd.ts
+35 -6
View File
@@ -2625,7 +2625,10 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
promptSource: prompts.summary,
}));
}
const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", "55000", "--command-timeout-seconds", String(remainingSeconds(deadline, 55))];
const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000));
const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55));
const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55));
const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))];
if (type === "selectProvider") args.push("--provider", stringAt(item, "provider"));
if (type === "loginAccount" || type === "listSessions" || type === "logout") {
const accountId = stringAtNullable(item, "accountId");
@@ -2644,10 +2647,36 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
}
appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" });
printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat });
const commandResult = runChildCli(args, remainingSeconds(deadline, 60));
const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10)));
steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result });
printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() });
if (!commandResult.ok) {
if (type === "sendPrompt") {
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult });
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() });
if (delayedWaitResult.ok === true) {
const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps);
nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings));
printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() });
if (invariantResult.ok !== true) {
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
reason,
observerId,
promptIndex,
steps,
failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"),
promptSource: prompts.summary,
elapsedMs: elapsedMs(),
warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()),
}));
}
continue;
}
}
return recordQuickVerify(state, finalizeQuickVerifyFailure(state, {
runId,
scenarioId,
@@ -2662,8 +2691,8 @@ function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeou
}));
}
if (type === "sendPrompt") {
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, 55) });
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds);
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) });
const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds);
steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult });
printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() });
if (waitResult.ok !== true) {
@@ -3935,7 +3964,7 @@ function runChildCli(args: string[], timeoutSeconds: number, input?: string, env
};
}
function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record<string, unknown> {
function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number, chunkSeconds = 45): Record<string, unknown> {
const observations: Record<string, unknown>[] = [];
const indexEntry = readLocalObserveIndex(observerId);
if (indexEntry === null) {
@@ -3949,7 +3978,7 @@ function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: stri
}
const pollSleepMs = Math.max(1000, Math.min(3000, Math.trunc(pollIntervalMs * 2) || 1000));
while (Date.now() < deadline) {
const waitMs = Math.max(1000, Math.min(45_000, deadline - Date.now()));
const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now()));
const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs);
const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 });
const payload = parseJsonObject(result.stdout);