fix: gate changed sub2api accounts behind sentinel probes
This commit is contained in:
@@ -3,7 +3,7 @@ import { rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { rootPath } from "./src/config";
|
||||
import { codexPoolHelp, defaultCodexTempUnschedulablePolicy } from "./src/platform-infra-sub2api-codex";
|
||||
import { codexPoolHelp, codexPoolSentinelProbeConfigFingerprint, defaultCodexTempUnschedulablePolicy } from "./src/platform-infra-sub2api-codex";
|
||||
import {
|
||||
codexPoolSentinelRuntimeImage,
|
||||
defaultCodexPoolSentinelConfig,
|
||||
@@ -18,6 +18,7 @@ function assertCondition(condition: unknown, message: string, detail: unknown =
|
||||
}
|
||||
|
||||
const configPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml");
|
||||
const codexPoolSourcePath = rootPath("scripts", "src", "platform-infra-sub2api-codex.ts");
|
||||
const sentinelDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile");
|
||||
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as {
|
||||
sentinel?: unknown;
|
||||
@@ -25,6 +26,7 @@ const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as {
|
||||
};
|
||||
const sentinel = readCodexPoolSentinelConfig(parsed.sentinel, defaultCodexPoolSentinelConfig(), configPath);
|
||||
const sentinelRuntimeImage = codexPoolSentinelRuntimeImage(sentinel);
|
||||
const codexPoolSource = readFileSync(codexPoolSourcePath, "utf8");
|
||||
const sentinelDockerfile = readFileSync(sentinelDockerfilePath, "utf8");
|
||||
const yamlTempUnschedulableRules = parsed.pool?.defaultTempUnschedulable?.rules ?? [];
|
||||
|
||||
@@ -44,10 +46,52 @@ assertCondition(!("concurrency" in sentinel.probe), "sentinel must not cap probe
|
||||
assertCondition(!("maxAccountsPerRun" in sentinel.probe), "sentinel must not cap accounts per run; all due accounts are eligible", sentinel.probe);
|
||||
assertCondition(sentinel.cadence.successInitialIntervalMinutes === 1, "success trust backoff must start at 1 minute", sentinel.cadence);
|
||||
assertCondition(sentinel.cadence.successMaxIntervalMinutes === 20, "success trust backoff must cap at 20 minutes", sentinel.cadence);
|
||||
assertCondition(sentinel.freeze.initialTtlMinutes === 2, "freeze backoff must start at 2 minutes", sentinel.freeze);
|
||||
assertCondition(sentinel.freeze.maxTtlMinutes === 120, "freeze backoff must cap at 2 hours", sentinel.freeze);
|
||||
assertCondition(sentinel.freeze.initialTtlMinutes === 1, "failure freeze backoff must start at 1 minute", sentinel.freeze);
|
||||
assertCondition(sentinel.freeze.maxTtlMinutes === 10, "failure freeze backoff must cap at 10 minutes", sentinel.freeze);
|
||||
assertCondition(!("budget" in sentinel), "sentinel must not use token budgets as a probe gate; usage is recorded only", sentinel);
|
||||
|
||||
const probeFingerprint = codexPoolSentinelProbeConfigFingerprint({
|
||||
accountName: "unidesk-codex-example",
|
||||
profile: "example",
|
||||
baseUrl: "https://example.invalid/v1/",
|
||||
apiKeyFingerprint: "key-a",
|
||||
upstreamUserAgent: null,
|
||||
openaiResponsesWebSocketsV2Mode: null,
|
||||
});
|
||||
assertCondition(
|
||||
probeFingerprint === codexPoolSentinelProbeConfigFingerprint({
|
||||
accountName: "unidesk-codex-example",
|
||||
profile: "example",
|
||||
baseUrl: "https://example.invalid/v1",
|
||||
apiKeyFingerprint: "key-a",
|
||||
upstreamUserAgent: null,
|
||||
openaiResponsesWebSocketsV2Mode: null,
|
||||
}),
|
||||
"sentinel probe fingerprint must normalize base URL trailing slash",
|
||||
);
|
||||
assertCondition(
|
||||
probeFingerprint !== codexPoolSentinelProbeConfigFingerprint({
|
||||
accountName: "unidesk-codex-example",
|
||||
profile: "example",
|
||||
baseUrl: "https://example.invalid/v1",
|
||||
apiKeyFingerprint: "key-b",
|
||||
upstreamUserAgent: null,
|
||||
openaiResponsesWebSocketsV2Mode: null,
|
||||
}),
|
||||
"sentinel probe fingerprint must change when API key fingerprint changes",
|
||||
);
|
||||
assertCondition(
|
||||
probeFingerprint !== codexPoolSentinelProbeConfigFingerprint({
|
||||
accountName: "unidesk-codex-example",
|
||||
profile: "example",
|
||||
baseUrl: "https://example.invalid/v1",
|
||||
apiKeyFingerprint: "key-a",
|
||||
upstreamUserAgent: "CodexCLI/0.1",
|
||||
openaiResponsesWebSocketsV2Mode: "ctx_pool",
|
||||
}),
|
||||
"sentinel probe fingerprint must change when direct probe request-shape inputs change",
|
||||
);
|
||||
|
||||
const manifest = renderCodexPoolSentinelManifest(sentinel, [
|
||||
{
|
||||
accountName: "unidesk-codex-example",
|
||||
@@ -102,6 +146,33 @@ assertCondition(runner.includes('"responseBodyPreview": item.get("responseBodyPr
|
||||
assertCondition(runner.includes("SENTINEL_ACCOUNT_NAMES"), "sentinel runner must support forced account probes for CLI manual measurement", runner);
|
||||
assertCondition(runner.includes('parsed.get("code") not in (None, 0)'), "sentinel admin client must treat Sub2API {code:0,message:success,data} envelopes as successful", runner);
|
||||
assertCondition(runner.includes("page_size=20&platform=openai&type=apikey&search="), "sentinel admin client must query one target account instead of fetching all accounts into the 64KiB admin response cap", runner);
|
||||
assertCondition(runner.includes('account_state["qualityGate"] = {**quality_gate, "pending": False'), "sentinel runner must clear pending YAML quality gates after marker-matched recovery", runner);
|
||||
|
||||
assertCondition(codexPoolSource.includes("sentinelProbeConfigFingerprint: codexPoolSentinelProbeConfigFingerprint({"), "sync payload must include per-account sentinel probe fingerprints", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0"), "sync must require quality gate only for new or direct-probe-changed accounts", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes('ensure_account_schedulable(token, data["id"], profile["accountName"], not quality_gate_required and not keep_frozen)'), "sync must default-freeze changed/new accounts before they enter scheduling", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes('"sentinelProbeRequired": quality_gate_required'), "sync result must report whether each account needs sentinel quality gate", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes('"reason": "yaml-account-change-pending-sentinel-probe"'), "sync state must record YAML-change quality-gate quarantine reason", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes('account_state["nextProbeAfter"] = now'), "sync state must schedule changed/new account probes immediately", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("planned_account_results = planned_sentinel_account_results(profiles, existing_accounts)"), "sync must compute changed/new accounts from pre-mutation runtime state", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)"), "sync must prepare changed/new account quality gate state before mutating accounts", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.indexOf("sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)") < codexPoolSource.indexOf("account_results, pruned_account_results = ensure_accounts("), "sync must prepare quality gate before account reconcile", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("sentinel_quality = ensure_sentinel_state_for_sync(account_results)"), "sync response must surface sentinel quality-gate reconciliation", codexPoolSourcePath);
|
||||
const reassertFunction = codexPoolSource.slice(
|
||||
codexPoolSource.indexOf("def reassert_sentinel_freezes_after_sync("),
|
||||
codexPoolSource.indexOf("def list_user_keys("),
|
||||
);
|
||||
assertCondition(!reassertFunction.includes("until_epoch"), "sync freeze reassert must not restore active quarantines just because TTL has expired", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("protected_frozen_names = active_sentinel_quarantine_names()"), "sync must read active sentinel quarantines before reconciling accounts", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("sentinelFreezeProtected"), "sync output must reveal accounts kept frozen by pre-existing sentinel quarantine", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes("def clamp_sentinel_freezes_for_config("), "sync must migrate active freeze state when YAML freeze max is lowered", codexPoolSourcePath);
|
||||
assertCondition(codexPoolSource.includes('reason = "freeze-backoff-clamped-to-current-config"'), "sync quality gate output must report freeze backoff clamping", codexPoolSourcePath);
|
||||
const stateUpdateFunction = codexPoolSource.slice(
|
||||
codexPoolSource.indexOf("def update_sentinel_state_configmap("),
|
||||
codexPoolSource.indexOf("def ensure_sentinel_state_for_sync("),
|
||||
);
|
||||
assertCondition(!stateUpdateFunction.includes('"patch"'), "sentinel state updates must not pass large state.json through kubectl patch argv", codexPoolSourcePath);
|
||||
assertCondition(stateUpdateFunction.includes('"-f", "-"'), "sentinel state updates must stream the ConfigMap manifest through stdin", codexPoolSourcePath);
|
||||
|
||||
const disabledMonitor = {
|
||||
...sentinel,
|
||||
@@ -149,7 +220,8 @@ console.log(JSON.stringify({
|
||||
"marker match is the only health standard",
|
||||
"budget is record-only and does not gate probes",
|
||||
"success trust backoff is 1m to 20m",
|
||||
"freeze backoff is 2m to 120m",
|
||||
"failure freeze backoff is 1m to 10m",
|
||||
"YAML changed/new accounts default-freeze until marker-matched sentinel recovery",
|
||||
"CronJob is k8s-native with Forbid concurrency and minimal RBAC",
|
||||
"monitor switch controls CronJob suspend state",
|
||||
"rendered Secret avoids plaintext upstream credentials",
|
||||
|
||||
Reference in New Issue
Block a user