mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: gate changed sub2api accounts behind sentinel probes

Browse Source
This commit is contained in:
Codex
2026-06-11 10:20:23 +00:00
parent f012b4ab81
commit 9f85274da0
6 changed files with 408 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-sub2api-codex-sentinel.ts
+5 -2
View File
@@ -107,8 +107,8 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
jitterPercent: 10,
},
freeze: {
initialTtlMinutes: 2,
maxTtlMinutes: 120,
initialTtlMinutes: 1,
maxTtlMinutes: 10,
backoffMultiplier: 2,
jitterPercent: 10,
},
@@ -1149,6 +1149,9 @@ def apply_result(result, state, config, now, admin):
except Exception as exc:
action = {"taken": False, "type": "restore-failed", "error": str(exc)}
account_state["quarantine"] = {"active": False, "clearedAt": iso(now), "lastApplied": quarantine.get("applied") is True}
quality_gate = account_state.get("qualityGate") if isinstance(account_state.get("qualityGate"), dict) else None
if quality_gate and quality_gate.get("pending") is True:
account_state["qualityGate"] = {**quality_gate, "pending": False, "clearedAt": iso(now)}
account_state["successStreak"] = 0
account_state["successIntervalMinutes"] = 0
interval = next_success_interval(account_state, config)
scripts/src/platform-infra-sub2api-codex.ts
+317 -23
View File
@@ -462,6 +462,14 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi
apiKey: profile.apiKey,
apiKeySource: profile.apiKeySource,
apiKeyFingerprint: fingerprint(profile.apiKey ?? ""),
sentinelProbeConfigFingerprint: codexPoolSentinelProbeConfigFingerprint({
accountName: profile.accountName,
profile: profile.profile,
baseUrl: profile.baseUrl,
apiKeyFingerprint: fingerprint(profile.apiKey ?? ""),
upstreamUserAgent: profile.upstreamUserAgent,
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
}),
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
upstreamUserAgent: profile.upstreamUserAgent,
priority: profile.priority,
@@ -1721,6 +1729,8 @@ function compactSentinelStatus(block: unknown): unknown {
const configMap = isRecord(runtime.configMap) ? runtime.configMap : {};
const state = isRecord(runtime.state) ? runtime.state : {};
const freezeReassert = isRecord(block.freezeReassert) ? block.freezeReassert : {};
const qualityGatePrepare = isRecord(block.qualityGatePrepare) ? block.qualityGatePrepare : {};
const qualityGate = isRecord(block.qualityGate) ? block.qualityGate : {};
return {
ok: block.ok,
action: block.action,
@@ -1769,6 +1779,25 @@ function compactSentinelStatus(block: unknown): unknown {
itemCount: freezeReassert.itemCount,
attentionItems: freezeReassert.attentionItems,
} : undefined,
qualityGatePrepare: Object.keys(qualityGatePrepare).length > 0 ? {
ok: qualityGatePrepare.ok,
skipped: qualityGatePrepare.skipped,
reason: qualityGatePrepare.reason,
changedCount: qualityGatePrepare.changedCount,
fingerprintOnlyCount: qualityGatePrepare.fingerprintOnlyCount,
pendingOnly: qualityGatePrepare.pendingOnly,
items: qualityGatePrepare.items,
} : undefined,
qualityGate: Object.keys(qualityGate).length > 0 ? {
ok: qualityGate.ok,
skipped: qualityGate.skipped,
reason: qualityGate.reason,
changedCount: qualityGate.changedCount,
fingerprintOnlyCount: qualityGate.fingerprintOnlyCount,
clampedCount: qualityGate.clampedCount,
items: qualityGate.items,
clampedItems: qualityGate.clampedItems,
} : undefined,
valuesPrinted: false,
};
}
@@ -2728,6 +2757,24 @@ function fingerprint(value: string): string {
return createHash("sha256").update(value).digest("hex").slice(0, 12);
}
export function codexPoolSentinelProbeConfigFingerprint(input: {
accountName: string;
profile: string;
baseUrl: string;
apiKeyFingerprint: string | null;
upstreamUserAgent: string | null;
openaiResponsesWebSocketsV2Mode: string | null;
}): string {
return fingerprint(JSON.stringify({
accountName: input.accountName,
profile: input.profile,
baseUrl: normalizeBaseUrl(input.baseUrl) ?? input.baseUrl,
apiKeyFingerprint: input.apiKeyFingerprint,
upstreamUserAgent: input.upstreamUserAgent,
openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode,
}));
}
function syncScript(payload: unknown, pool: CodexPoolConfig): string {
const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
return remotePythonScript("sync", encoded, pool);
@@ -2750,7 +2797,7 @@ set -eu
python3 - <<'PY'
import json
import subprocess
from datetime import datetime, timezone
from datetime import datetime, timezone, timedelta
NAMESPACE = ${JSON.stringify(namespace)}
STATE_NAME = ${JSON.stringify(stateName)}
@@ -3122,12 +3169,13 @@ set -u
python3 - <<'PY'
import base64
import json
import re
import secrets
import string
import subprocess
import sys
import time
from datetime import datetime
from datetime import datetime, timezone, timedelta
from urllib.parse import quote
NAMESPACE = "${namespace}"
@@ -3237,6 +3285,60 @@ def parse_curl_output(proc):
"transportExitCode": proc.returncode,
}
def utc_iso(offset_seconds=0):
return (datetime.now(timezone.utc) + timedelta(seconds=offset_seconds)).strftime("%Y-%m-%dT%H:%M:%SZ")
def normalize_runtime_base_url(value):
if not isinstance(value, str):
return None
value = value.strip().rstrip("/")
return value or None
def empty_to_none(value):
return value if isinstance(value, str) and value else None
def sentinel_quality_gate_enabled():
return (SENTINEL_CONFIG.get("monitor") or {}).get("enabled") is True and (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is True
def account_notes_fingerprint(account):
notes = account.get("notes") if isinstance(account, dict) else None
if not isinstance(notes, str):
return None
match = re.search(r"fingerprint=([A-Za-z0-9_-]+)", notes)
return match.group(1) if match else None
def runtime_account_credentials(account):
credentials = account.get("credentials") if isinstance(account, dict) and isinstance(account.get("credentials"), dict) else {}
return credentials
def runtime_account_extra(account):
extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {}
return extra
def sentinel_probe_change_reasons(current, profile):
if not isinstance(current, dict) or current.get("id") is None:
return ["created"]
credentials = runtime_account_credentials(current)
extra = runtime_account_extra(current)
expected_base_url = normalize_runtime_base_url(profile.get("baseUrl"))
runtime_base_url = normalize_runtime_base_url(credentials.get("base_url"))
expected_user_agent = empty_to_none(profile.get("upstreamUserAgent"))
runtime_user_agent = empty_to_none(credentials.get("user_agent"))
expected_ws_mode = empty_to_none(profile.get("openaiResponsesWebSocketsV2Mode"))
runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode"))
reasons = []
if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"):
reasons.append("profile")
if runtime_base_url != expected_base_url:
reasons.append("base-url")
if account_notes_fingerprint(current) != profile.get("apiKeyFingerprint"):
reasons.append("api-key-fingerprint")
if runtime_user_agent != expected_user_agent:
reasons.append("upstream-user-agent")
if runtime_ws_mode != expected_ws_mode:
reasons.append("responses-websockets-v2-mode")
return reasons
def curl_api(method, path, bearer=None, payload=None):
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
script = r'''
@@ -3478,21 +3580,44 @@ def account_payload(profile, group_id):
"confirm_mixed_channel_risk": True,
}
def ensure_account_schedulable(token, account_id, account_name):
def ensure_account_schedulable(token, account_id, account_name, schedulable=True):
data = ensure_success(
curl_api("POST", f"/api/v1/admin/accounts/{account_id}/schedulable", bearer=token, payload={"schedulable": True}),
f"set account {account_name} schedulable",
curl_api("POST", f"/api/v1/admin/accounts/{account_id}/schedulable", bearer=token, payload={"schedulable": bool(schedulable)}),
f"set account {account_name} schedulable={bool(schedulable)}",
)
return data if isinstance(data, dict) else {}
def ensure_accounts(token, profiles, group_id, prune_removed=False):
existing_accounts = list_accounts(token)
def planned_sentinel_account_results(profiles, existing_accounts):
existing = {item.get("name"): item for item in existing_accounts if isinstance(item, dict)}
results = []
for profile in profiles:
change_reasons = sentinel_probe_change_reasons(existing.get(profile["accountName"]), profile)
quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0
results.append({
"profile": profile["profile"],
"accountName": profile["accountName"],
"sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"),
"sentinelProbeRequired": quality_gate_required,
"sentinelChangeReasons": change_reasons if quality_gate_required else [],
"sentinelDefaultFrozen": quality_gate_required,
"valuesPrinted": False,
})
return results
def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_frozen_names=None, existing_accounts=None):
if not isinstance(protected_frozen_names, set):
protected_frozen_names = set()
if not isinstance(existing_accounts, list):
existing_accounts = list_accounts(token)
existing = {item.get("name"): item for item in existing_accounts}
desired_names = {profile["accountName"] for profile in profiles}
results = []
for profile in profiles:
payload = account_payload(profile, group_id)
current = existing.get(profile["accountName"])
change_reasons = sentinel_probe_change_reasons(current, profile)
quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0
keep_frozen = profile["accountName"] in protected_frozen_names
if current and current.get("id") is not None:
account_id = current["id"]
update_payload = dict(payload)
@@ -3504,7 +3629,7 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False):
data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}")
action = "created"
if isinstance(data, dict) and data.get("id") is not None:
schedulable_data = ensure_account_schedulable(token, data["id"], profile["accountName"])
schedulable_data = ensure_account_schedulable(token, data["id"], profile["accountName"], not quality_gate_required and not keep_frozen)
if schedulable_data:
data = schedulable_data
results.append({
@@ -3515,6 +3640,11 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False):
"baseUrl": profile["baseUrl"],
"apiKeySource": profile["apiKeySource"],
"apiKeyFingerprint": profile["apiKeyFingerprint"],
"sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"),
"sentinelProbeRequired": quality_gate_required,
"sentinelChangeReasons": change_reasons if quality_gate_required else [],
"sentinelDefaultFrozen": quality_gate_required,
"sentinelFreezeProtected": keep_frozen,
"openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"),
"priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
"capacity": int(profile.get("capacity", 5) or 5),
@@ -3745,20 +3875,181 @@ def parse_epoch_z(value):
def sentinel_state_object():
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
if not state_name:
return None
return None, None
obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
if not isinstance(obj, dict):
return None
return None, None
raw_state = (obj.get("data") or {}).get("state.json")
if not isinstance(raw_state, str) or not raw_state:
return None
return obj, None
try:
return json.loads(raw_state)
return obj, json.loads(raw_state)
except Exception:
return None
return obj, None
def active_sentinel_quarantine_names():
_, state = sentinel_state_object()
if not isinstance(state, dict):
return set()
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
names = set()
for name, account_state in accounts_state.items():
if not isinstance(name, str) or not isinstance(account_state, dict):
continue
quarantine = account_state.get("quarantine")
if isinstance(quarantine, dict) and quarantine.get("active") is True and quarantine.get("applied") is True:
names.add(name)
return names
def default_sentinel_state():
return {"version": 1, "accounts": {}, "ledger": {}, "history": []}
def clamp_sentinel_freezes_for_config(state, now):
freeze_config = SENTINEL_CONFIG.get("freeze") if isinstance(SENTINEL_CONFIG.get("freeze"), dict) else {}
try:
max_interval = int(freeze_config.get("maxTtlMinutes") or 10)
except Exception:
max_interval = 10
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
now_epoch = time.time()
items = []
for name, account_state in accounts_state.items():
if not isinstance(name, str) or not isinstance(account_state, dict):
continue
quarantine = account_state.get("quarantine")
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
continue
try:
interval = int(quarantine.get("intervalMinutes") or 0)
except Exception:
interval = 0
until_epoch = parse_epoch_z(quarantine.get("until"))
old_until = quarantine.get("until")
if interval <= max_interval and (until_epoch is None or until_epoch <= now_epoch + max_interval * 60):
continue
quarantine["previousIntervalMinutes"] = interval
quarantine["intervalMinutes"] = max_interval
quarantine["until"] = now
quarantine["clampedAt"] = now
quarantine["clampedBy"] = "sync-freeze-max-ttl"
account_state["nextProbeAfter"] = now
items.append({
"accountName": name,
"previousIntervalMinutes": interval,
"maxIntervalMinutes": max_interval,
"previousUntil": old_until,
"nextProbeAfter": now,
})
return items
def update_sentinel_state_configmap(obj, state):
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
if not state_name:
return {"ok": False, "reason": "state-configmap-missing"}
state_json = json.dumps(state, ensure_ascii=False, indent=2)
manifest = {
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": state_name,
"namespace": NAMESPACE,
"labels": {
"app.kubernetes.io/name": SERVICE_NAME,
"app.kubernetes.io/component": "account-sentinel",
"app.kubernetes.io/managed-by": "unidesk-platform-infra",
},
},
"data": {"state.json": state_json},
}
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
action = "applied" if isinstance(obj, dict) else "created"
if proc.returncode != 0:
return {"ok": False, "reason": f"{action}-failed", "stderrTail": text(proc.stderr, 2000)}
return {"ok": True, "action": action}
def ensure_sentinel_state_for_sync(account_results, pending_only=False):
if not sentinel_quality_gate_enabled():
return {"ok": True, "skipped": True, "reason": "sentinel-quality-gate-disabled", "changedCount": 0, "items": [], "valuesPrinted": False}
state_obj, state = sentinel_state_object()
if not isinstance(state, dict):
state = default_sentinel_state()
state.setdefault("version", 1)
accounts_state = state.setdefault("accounts", {})
if not isinstance(accounts_state, dict):
accounts_state = {}
state["accounts"] = accounts_state
now = utc_iso()
pending_until = utc_iso(3600)
items = []
clamped_items = [] if pending_only else clamp_sentinel_freezes_for_config(state, now)
changed_count = 0
fingerprint_only_count = 0
for item in account_results:
name = item.get("accountName")
if not isinstance(name, str) or not name:
continue
account_state = accounts_state.setdefault(name, {})
if not isinstance(account_state, dict):
account_state = {}
accounts_state[name] = account_state
fingerprint_value = item.get("sentinelProbeConfigFingerprint")
if isinstance(fingerprint_value, str) and fingerprint_value:
account_state["probeConfigFingerprint"] = fingerprint_value
if item.get("sentinelProbeRequired") is not True:
fingerprint_only_count += 1
continue
changed_count += 1
reasons = item.get("sentinelChangeReasons") if isinstance(item.get("sentinelChangeReasons"), list) else []
account_state["quarantine"] = {
"active": True,
"applied": True,
"until": pending_until if pending_only else now,
"intervalMinutes": 0,
"reason": "yaml-account-change-pending-sentinel-probe",
"failureKind": "pending-quality-gate",
"changeReasons": reasons,
"startedAt": now,
"lastBadAt": now,
}
account_state["nextProbeAfter"] = pending_until if pending_only else now
account_state["successStreak"] = 0
account_state["successIntervalMinutes"] = 0
account_state["lastStatus"] = "pending-sentinel-quality-gate"
account_state["qualityGate"] = {
"pending": True,
"reason": "yaml-account-change",
"changeReasons": reasons,
"markedAt": now,
"pendingOnly": pending_only,
}
items.append({"accountName": name, "changeReasons": reasons, "nextProbeAfter": pending_until if pending_only else now, "defaultFrozen": True, "pendingOnly": pending_only})
if changed_count <= 0 and len(clamped_items) <= 0:
return {"ok": True, "skipped": False, "reason": "no-new-or-changed-accounts", "changedCount": 0, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": 0, "items": [], "valuesPrinted": False}
update = update_sentinel_state_configmap(state_obj, state)
if pending_only and changed_count > 0:
reason = "new-or-changed-accounts-pending-quality-gate-prepared"
elif changed_count > 0 and len(clamped_items) > 0:
reason = "new-or-changed-accounts-default-frozen-and-freeze-backoff-clamped"
elif changed_count > 0:
reason = "new-or-changed-accounts-default-frozen"
else:
reason = "freeze-backoff-clamped-to-current-config"
return {
"ok": update.get("ok") is True,
"skipped": False,
"reason": reason,
"changedCount": changed_count,
"fingerprintOnlyCount": fingerprint_only_count,
"clampedCount": len(clamped_items),
"pendingOnly": pending_only,
"items": items,
"clampedItems": clamped_items,
"update": update,
"valuesPrinted": False,
}
def sentinel_state_summary():
state = sentinel_state_object()
_, state = sentinel_state_object()
if not isinstance(state, dict):
return {"exists": False, "valuesPrinted": False}
accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
@@ -3813,13 +4104,12 @@ def sentinel_state_summary():
def reassert_sentinel_freezes_after_sync(token):
if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True:
return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False}
state = sentinel_state_object()
_, state = sentinel_state_object()
if not isinstance(state, dict):
return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False}
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
accounts = list_accounts(token)
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
now_epoch = time.time()
items = []
for name, account_state in accounts_state.items():
if not isinstance(account_state, dict):
@@ -3827,9 +4117,6 @@ def reassert_sentinel_freezes_after_sync(token):
quarantine = account_state.get("quarantine")
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
continue
until_epoch = parse_epoch_z(quarantine.get("until"))
if until_epoch is not None and until_epoch <= now_epoch:
continue
account = by_name.get(name)
if not account or account.get("id") is None:
items.append({"accountName": name, "ok": False, "reason": "account-not-found"})
@@ -4949,7 +5236,13 @@ def run_sync():
group_id = group.get("id") if isinstance(group, dict) else None
if group_id is None:
raise RuntimeError("pool group id missing after ensure")
account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed)
existing_accounts = list_accounts(token)
planned_account_results = planned_sentinel_account_results(profiles, existing_accounts)
sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)
if sentinel_quality_prepare.get("ok") is not True:
raise RuntimeError("prepare sentinel quality gate failed: " + json.dumps(sentinel_quality_prepare, ensure_ascii=False))
protected_frozen_names = active_sentinel_quarantine_names()
account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed, protected_frozen_names, existing_accounts)
capacity_status = account_capacity_status(token)
load_factor_status = account_load_factor_status(token)
ws_v2_status = account_ws_v2_status(token)
@@ -4964,9 +5257,10 @@ def run_sync():
responses_evidence = recent_responses_gateway_evidence()
runtime_capabilities = validate_runtime_capabilities(token)
sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest"))
sentinel_quality = ensure_sentinel_state_for_sync(account_results)
sentinel_reassert = reassert_sentinel_freezes_after_sync(token)
return {
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and sentinel.get("ok") is True and sentinel_reassert.get("ok") is True,
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and sentinel.get("ok") is True and sentinel_quality_prepare.get("ok") is True and sentinel_quality.get("ok") is True and sentinel_reassert.get("ok") is True,
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
"mode": "sync",
"namespace": NAMESPACE,
@@ -4982,7 +5276,7 @@ def run_sync():
"pruneMode": "explicit" if prune_removed else "disabled-by-default",
"items": account_results,
"prunedItems": pruned_account_results,
"processControl": {"schedulableRestore": "POST /api/v1/admin/accounts/:id/schedulable", "durableConfig": False},
"processControl": {"schedulableRestore": "POST /api/v1/admin/accounts/:id/schedulable for non-quarantined accounts only", "durableConfig": False},
"valuesPrinted": False,
},
"capacity": capacity_status,
@@ -5003,7 +5297,7 @@ def run_sync():
},
"ownerBalance": owner_balance,
"ownerConcurrency": owner_concurrency,
"sentinel": {**sentinel, "freezeReassert": sentinel_reassert},
"sentinel": {**sentinel, "qualityGatePrepare": sentinel_quality_prepare, "qualityGate": sentinel_quality, "freezeReassert": sentinel_reassert},
"runtimeCapabilities": runtime_capabilities,
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
}