fix: gate changed sub2api accounts behind sentinel probes

This commit is contained in:
Codex
2026-06-11 10:20:23 +00:00
parent f012b4ab81
commit 9f85274da0
6 changed files with 408 additions and 35 deletions
@@ -3,7 +3,7 @@ import { rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rootPath } from "./src/config";
import { codexPoolHelp, defaultCodexTempUnschedulablePolicy } from "./src/platform-infra-sub2api-codex";
import { codexPoolHelp, codexPoolSentinelProbeConfigFingerprint, defaultCodexTempUnschedulablePolicy } from "./src/platform-infra-sub2api-codex";
import {
codexPoolSentinelRuntimeImage,
defaultCodexPoolSentinelConfig,
@@ -18,6 +18,7 @@ function assertCondition(condition: unknown, message: string, detail: unknown =
}
const configPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml");
const codexPoolSourcePath = rootPath("scripts", "src", "platform-infra-sub2api-codex.ts");
const sentinelDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile");
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as {
sentinel?: unknown;
@@ -25,6 +26,7 @@ const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as {
};
const sentinel = readCodexPoolSentinelConfig(parsed.sentinel, defaultCodexPoolSentinelConfig(), configPath);
const sentinelRuntimeImage = codexPoolSentinelRuntimeImage(sentinel);
const codexPoolSource = readFileSync(codexPoolSourcePath, "utf8");
const sentinelDockerfile = readFileSync(sentinelDockerfilePath, "utf8");
const yamlTempUnschedulableRules = parsed.pool?.defaultTempUnschedulable?.rules ?? [];
@@ -44,10 +46,52 @@ assertCondition(!("concurrency" in sentinel.probe), "sentinel must not cap probe
assertCondition(!("maxAccountsPerRun" in sentinel.probe), "sentinel must not cap accounts per run; all due accounts are eligible", sentinel.probe);
assertCondition(sentinel.cadence.successInitialIntervalMinutes === 1, "success trust backoff must start at 1 minute", sentinel.cadence);
assertCondition(sentinel.cadence.successMaxIntervalMinutes === 20, "success trust backoff must cap at 20 minutes", sentinel.cadence);
assertCondition(sentinel.freeze.initialTtlMinutes === 2, "freeze backoff must start at 2 minutes", sentinel.freeze);
assertCondition(sentinel.freeze.maxTtlMinutes === 120, "freeze backoff must cap at 2 hours", sentinel.freeze);
assertCondition(sentinel.freeze.initialTtlMinutes === 1, "failure freeze backoff must start at 1 minute", sentinel.freeze);
assertCondition(sentinel.freeze.maxTtlMinutes === 10, "failure freeze backoff must cap at 10 minutes", sentinel.freeze);
assertCondition(!("budget" in sentinel), "sentinel must not use token budgets as a probe gate; usage is recorded only", sentinel);
const probeFingerprint = codexPoolSentinelProbeConfigFingerprint({
accountName: "unidesk-codex-example",
profile: "example",
baseUrl: "https://example.invalid/v1/",
apiKeyFingerprint: "key-a",
upstreamUserAgent: null,
openaiResponsesWebSocketsV2Mode: null,
});
assertCondition(
probeFingerprint === codexPoolSentinelProbeConfigFingerprint({
accountName: "unidesk-codex-example",
profile: "example",
baseUrl: "https://example.invalid/v1",
apiKeyFingerprint: "key-a",
upstreamUserAgent: null,
openaiResponsesWebSocketsV2Mode: null,
}),
"sentinel probe fingerprint must normalize base URL trailing slash",
);
assertCondition(
probeFingerprint !== codexPoolSentinelProbeConfigFingerprint({
accountName: "unidesk-codex-example",
profile: "example",
baseUrl: "https://example.invalid/v1",
apiKeyFingerprint: "key-b",
upstreamUserAgent: null,
openaiResponsesWebSocketsV2Mode: null,
}),
"sentinel probe fingerprint must change when API key fingerprint changes",
);
assertCondition(
probeFingerprint !== codexPoolSentinelProbeConfigFingerprint({
accountName: "unidesk-codex-example",
profile: "example",
baseUrl: "https://example.invalid/v1",
apiKeyFingerprint: "key-a",
upstreamUserAgent: "CodexCLI/0.1",
openaiResponsesWebSocketsV2Mode: "ctx_pool",
}),
"sentinel probe fingerprint must change when direct probe request-shape inputs change",
);
const manifest = renderCodexPoolSentinelManifest(sentinel, [
{
accountName: "unidesk-codex-example",
@@ -102,6 +146,33 @@ assertCondition(runner.includes('"responseBodyPreview": item.get("responseBodyPr
assertCondition(runner.includes("SENTINEL_ACCOUNT_NAMES"), "sentinel runner must support forced account probes for CLI manual measurement", runner);
assertCondition(runner.includes('parsed.get("code") not in (None, 0)'), "sentinel admin client must treat Sub2API {code:0,message:success,data} envelopes as successful", runner);
assertCondition(runner.includes("page_size=20&platform=openai&type=apikey&search="), "sentinel admin client must query one target account instead of fetching all accounts into the 64KiB admin response cap", runner);
assertCondition(runner.includes('account_state["qualityGate"] = {**quality_gate, "pending": False'), "sentinel runner must clear pending YAML quality gates after marker-matched recovery", runner);
assertCondition(codexPoolSource.includes("sentinelProbeConfigFingerprint: codexPoolSentinelProbeConfigFingerprint({"), "sync payload must include per-account sentinel probe fingerprints", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0"), "sync must require quality gate only for new or direct-probe-changed accounts", codexPoolSourcePath);
assertCondition(codexPoolSource.includes('ensure_account_schedulable(token, data["id"], profile["accountName"], not quality_gate_required and not keep_frozen)'), "sync must default-freeze changed/new accounts before they enter scheduling", codexPoolSourcePath);
assertCondition(codexPoolSource.includes('"sentinelProbeRequired": quality_gate_required'), "sync result must report whether each account needs sentinel quality gate", codexPoolSourcePath);
assertCondition(codexPoolSource.includes('"reason": "yaml-account-change-pending-sentinel-probe"'), "sync state must record YAML-change quality-gate quarantine reason", codexPoolSourcePath);
assertCondition(codexPoolSource.includes('account_state["nextProbeAfter"] = now'), "sync state must schedule changed/new account probes immediately", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("planned_account_results = planned_sentinel_account_results(profiles, existing_accounts)"), "sync must compute changed/new accounts from pre-mutation runtime state", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)"), "sync must prepare changed/new account quality gate state before mutating accounts", codexPoolSourcePath);
assertCondition(codexPoolSource.indexOf("sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)") < codexPoolSource.indexOf("account_results, pruned_account_results = ensure_accounts("), "sync must prepare quality gate before account reconcile", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("sentinel_quality = ensure_sentinel_state_for_sync(account_results)"), "sync response must surface sentinel quality-gate reconciliation", codexPoolSourcePath);
const reassertFunction = codexPoolSource.slice(
codexPoolSource.indexOf("def reassert_sentinel_freezes_after_sync("),
codexPoolSource.indexOf("def list_user_keys("),
);
assertCondition(!reassertFunction.includes("until_epoch"), "sync freeze reassert must not restore active quarantines just because TTL has expired", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("protected_frozen_names = active_sentinel_quarantine_names()"), "sync must read active sentinel quarantines before reconciling accounts", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("sentinelFreezeProtected"), "sync output must reveal accounts kept frozen by pre-existing sentinel quarantine", codexPoolSourcePath);
assertCondition(codexPoolSource.includes("def clamp_sentinel_freezes_for_config("), "sync must migrate active freeze state when YAML freeze max is lowered", codexPoolSourcePath);
assertCondition(codexPoolSource.includes('reason = "freeze-backoff-clamped-to-current-config"'), "sync quality gate output must report freeze backoff clamping", codexPoolSourcePath);
const stateUpdateFunction = codexPoolSource.slice(
codexPoolSource.indexOf("def update_sentinel_state_configmap("),
codexPoolSource.indexOf("def ensure_sentinel_state_for_sync("),
);
assertCondition(!stateUpdateFunction.includes('"patch"'), "sentinel state updates must not pass large state.json through kubectl patch argv", codexPoolSourcePath);
assertCondition(stateUpdateFunction.includes('"-f", "-"'), "sentinel state updates must stream the ConfigMap manifest through stdin", codexPoolSourcePath);
const disabledMonitor = {
...sentinel,
@@ -149,7 +220,8 @@ console.log(JSON.stringify({
"marker match is the only health standard",
"budget is record-only and does not gate probes",
"success trust backoff is 1m to 20m",
"freeze backoff is 2m to 120m",
"failure freeze backoff is 1m to 10m",
"YAML changed/new accounts default-freeze until marker-matched sentinel recovery",
"CronJob is k8s-native with Forbid concurrency and minimal RBAC",
"monitor switch controls CronJob suspend state",
"rendered Secret avoids plaintext upstream credentials",