From 551059a2418054733ae133020870ac7acef8594c Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 09:32:48 +0200 Subject: [PATCH 01/12] feat: reconcile Sub2API runtime batch writes --- .../platform-infra-sub2api-codex/runtime.ts | 221 ++++++++++++++---- .../src/platform-infra-sub2api-codex/types.ts | 2 + 2 files changed, 181 insertions(+), 42 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index e5f1b561..410ce649 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -1,17 +1,20 @@ import type { UniDeskConfig } from "../config"; +import type { RenderedCliResult } from "../output"; import { desiredAccountNames } from "./accounts"; import { readCodexPoolConfig } from "./config"; +import { isRecord, stringValue } from "./config-utils"; import { protectedManualAccountsForTarget } from "./public-exposure"; import { renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; +import { renderTable } from "./render"; import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; interface RuntimeOptions { action: RuntimeAction; - account: string | null; + accounts: string[]; template: string | null; kind: "temp-unschedulable"; confirm: boolean; @@ -22,7 +25,7 @@ interface RuntimeOptions { targetId: string; } -export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise> { +export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const options = parseRuntimeOptions(args); const pool = readCodexPoolConfig(); const target = codexPoolRuntimeTarget(options.targetId); @@ -32,7 +35,8 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P } const payload = { action: options.action, - account: options.account, + account: options.accounts.length === 1 ? options.accounts[0] : null, + accounts: options.accounts, kind: options.kind, confirm: options.confirm, full: options.full, @@ -57,6 +61,9 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P const result = await runRemoteCodexPoolScript(config, "runtime", runtimeScript(payload, target), target); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && boolField(parsed, "ok", false); + if (!options.full && !options.raw && (options.action === "apply" || options.action === "delete")) { + return renderRuntimeMutation(ok, options, parsed, result.exitCode); + } if (options.raw) { return { ok, @@ -78,7 +85,8 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P }, request: { operation: options.action, - account: options.account, + account: options.accounts.length === 1 ? options.accounts[0] : undefined, + accounts: options.accounts.length > 1 ? options.accounts : undefined, template: options.template, kind: options.kind, confirm: options.confirm, @@ -100,9 +108,10 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P export function parseRuntimeOptions(args: string[]): RuntimeOptions { const [actionRaw = "list", ...rest] = args; if (actionRaw !== "list" && actionRaw !== "get" && actionRaw !== "errors" && actionRaw !== "apply" && actionRaw !== "delete") { - throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--full|--raw]"); + throw new Error("runtime usage: list|get|errors|apply|delete [--account |--accounts ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--full|--raw]"); } let account: string | null = null; + let accounts: string[] | null = null; let template: string | null = null; let kind = "temp-unschedulable" as const; let confirm = false; @@ -124,6 +133,8 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { else if (arg === "--raw") raw = true; else if (arg === "--account") account = readValue("--account"); else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); + else if (arg === "--accounts") accounts = parseRuntimeAccounts(readValue("--accounts")); + else if (arg.startsWith("--accounts=")) accounts = parseRuntimeAccounts(arg.slice("--accounts=".length)); else if (arg === "--template") template = readValue("--template"); else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); @@ -136,17 +147,27 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { else if (arg.startsWith("--tail=")) tail = parseRuntimeTail(arg.slice("--tail=".length)); else throw new Error(`unsupported runtime option: ${arg}`); } - if ((actionRaw === "get" || actionRaw === "apply" || actionRaw === "delete") && !account) { + if (account !== null && accounts !== null) throw new Error("use only one of --account or --accounts"); + const selectors = accounts ?? (account === null ? [] : [account]); + if ((actionRaw === "get" || actionRaw === "apply" || actionRaw === "delete") && selectors.length === 0) { throw new Error(`runtime ${actionRaw} requires --account `); } - if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); + if (actionRaw !== "apply" && actionRaw !== "delete" && accounts !== null) throw new Error(`runtime ${actionRaw} does not accept --accounts`); + if (selectors.some((selector) => selector.length > 256 || /[\r\n]/u.test(selector))) throw new Error("account selector has an unsupported format"); if (actionRaw === "apply" && !template) throw new Error("runtime apply requires --template "); if (actionRaw !== "apply" && template !== null) throw new Error(`runtime ${actionRaw} does not accept --template`); if (actionRaw !== "errors" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${actionRaw} does not accept --since or --tail`); if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); if ((actionRaw === "list" || actionRaw === "get" || actionRaw === "errors") && confirm) throw new Error(`runtime ${actionRaw} does not accept --confirm`); if (full && raw) throw new Error("use only one of --full or --raw"); - return { action: actionRaw, account, template, kind, confirm, full, raw, since, tail, targetId }; + return { action: actionRaw, accounts: selectors, template, kind, confirm, full, raw, since, tail, targetId }; +} + +function parseRuntimeAccounts(value: string): string[] { + const accounts = value.split(",").map((item) => item.trim()); + if (accounts.length === 0 || accounts.some((item) => item.length === 0)) throw new Error("--accounts requires comma-separated selectors"); + if (new Set(accounts).size !== accounts.length) throw new Error("--accounts contains duplicate selectors"); + return accounts; } function parseRuntimeKind(value: string): "temp-unschedulable" { @@ -160,6 +181,60 @@ function parseRuntimeTail(value: string): number { return parsed; } +function renderRuntimeMutation(ok: boolean, options: RuntimeOptions, parsed: Record | null, exitCode: number): RenderedCliResult { + const summary = parsed !== null && isRecord(parsed.summary) ? parsed.summary : {}; + const items = parsed !== null && Array.isArray(parsed.items) ? parsed.items.filter(isRecord) : []; + const lines = [[ + "SUB2API RUNTIME", + `operation=${options.action}`, + `mode=${stringValue(parsed?.mode) ?? (options.confirm ? "confirmed" : "dry-run")}`, + `ok=${ok ? "true" : "false"}`, + `selected=${summary.selected ?? items.length}`, + `changed=${summary.changed ?? "?"}`, + `writeSucceeded=${summary.writeSucceeded ?? "?"}`, + `writeFailed=${summary.writeFailed ?? "?"}`, + `reconciled=${summary.reconciled ?? "?"}`, + `mismatched=${summary.mismatched ?? "?"}`, + ].join(" ")]; + if (items.length > 0) { + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "WRITE", "RECONCILED", "BEFORE", "DESIRED", "ACTUAL", "BEFORE_CODES", "DESIRED_CODES", "ACTUAL_CODES", "FAILURE"], + ...items.map((item) => { + const write = isRecord(item.write) ? item.write : {}; + const before = isRecord(item.before) ? item.before : {}; + const desired = isRecord(item.desired) ? item.desired : {}; + const actual = isRecord(item.actual) ? item.actual : {}; + const codes = (value: unknown): string => Array.isArray(value) ? value.join(",") || "-" : "-"; + return [ + stringValue(item.accountName) ?? "-", + String(item.accountId ?? "-"), + stringValue(item.change) ?? "-", + stringValue(write.status) ?? "-", + item.reconciled === true ? "true" : item.reconciled === false ? "false" : "-", + String(before.ruleCount ?? "-"), + String(desired.ruleCount ?? "-"), + String(actual.ruleCount ?? "-"), + codes(before.statusCodes), + codes(desired.statusCodes), + codes(actual.statusCodes), + stringValue(write.error) ?? stringValue(item.reconcileError) ?? "-", + ]; + }), + ])); + } else if (parsed !== null && typeof parsed.error === "string") { + lines.push(`ERROR ${parsed.error}`); + } else if (parsed === null) { + lines.push(`ERROR runtime output unavailable remote_exit=${exitCode}`); + } + return { + ok, + command: "platform-infra sub2api codex-pool runtime", + renderedText: lines.join("\n"), + contentType: "text/plain", + projection: parsed ?? undefined, + }; +} + function pyJson(value: unknown): string { return `json.loads(${JSON.stringify(JSON.stringify(value))})`; } @@ -487,6 +562,78 @@ def observed_runtime_errors(accounts): "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", } +def selected_account_details(token, accounts): + selectors = PAYLOAD.get("accounts") + if not isinstance(selectors, list) or not selectors: + selector = PAYLOAD.get("account") + selectors = [selector] if isinstance(selector, str) and selector else [] + details = [] + selected_ids = set() + for raw_selector in selectors: + selector = str(raw_selector) + detail = find_account(token, accounts, selector) + account_id = detail.get("id") + if account_id in selected_ids: + raise RuntimeError("duplicate-account-selector: " + selector) + if detail.get("type") != "apikey": + raise RuntimeError("runtime temp-unschedulable policy requires an apikey account: " + selector) + selected_ids.add(account_id) + details.append(detail) + return details + +def mutation_plan(detail, action, include_rules): + credentials = dict(detail.get("credentials") or {}) + before = normalize_policy(credentials) + if action == "apply": + template = PAYLOAD.get("selectedTemplate") + if not isinstance(template, dict): + raise RuntimeError("selected runtime template missing") + desired = dict(credentials) + desired.update(template.get("credentials") or {}) + desired_policy = normalize_policy(desired) + exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials + change = "noop" if before == desired_policy else ("update" if exists else "create") + else: + desired = dict(credentials) + desired.pop("temp_unschedulable_enabled", None) + desired.pop("temp_unschedulable_rules", None) + desired_policy = normalize_policy(desired) + change = "delete" if before != desired_policy or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" + return { + "accountName": detail.get("name"), + "accountId": detail.get("id"), + "change": change, + "before": policy_summary(before, include_rules), + "desired": policy_summary(desired_policy, include_rules), + "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, + "_desiredCredentials": desired, + "_desiredPolicy": desired_policy, + } + +def write_mutation_plan(token, plan): + if plan["change"] == "noop": + return {"attempted": False, "status": "noop", "error": None} + try: + data_of(curl_api("PUT", f"/api/v1/admin/accounts/{plan['accountId']}", bearer=token, payload={"credentials": plan["_desiredCredentials"]}), "update account runtime config") + return {"attempted": True, "status": "succeeded", "error": None} + except Exception as exc: + return {"attempted": True, "status": "failed", "error": compact_error(str(exc))} + +def reconcile_mutation_plan(token, plan, write, include_rules): + item = {key: value for key, value in plan.items() if not key.startswith("_")} + item["write"] = write + try: + actual_detail = account_detail(token, plan["accountId"]) + actual_policy = normalize_policy(actual_detail.get("credentials")) + item["actual"] = policy_summary(actual_policy, include_rules) + item["reconciled"] = actual_policy == plan["_desiredPolicy"] + item["reconcileError"] = None + except Exception as exc: + item["actual"] = None + item["reconciled"] = False + item["reconcileError"] = compact_error(str(exc)) + return item + def runtime_result(): token = login() group = next((item for item in list_groups(token) if item.get("name") == PAYLOAD["groupName"]), None) @@ -506,43 +653,33 @@ def runtime_result(): if action == "errors": errors = observed_runtime_errors(accounts) return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} - detail = find_account(token, accounts, str(PAYLOAD["account"])) + detail = find_account(token, accounts, str(PAYLOAD["account"])) if action == "get" else None if action == "get": return {**base, "ok": True, "operation": "get", "mutation": False, "account": account_summary(detail, True)} - if detail.get("type") != "apikey": - raise RuntimeError("runtime temp-unschedulable policy requires an apikey account") - credentials = dict(detail.get("credentials") or {}) - before = normalize_policy(credentials) - if action == "apply": - template = PAYLOAD.get("selectedTemplate") - if not isinstance(template, dict): - raise RuntimeError("selected runtime template missing") - desired_fields = template.get("credentials") or {} - desired = dict(credentials) - desired.update(desired_fields) - after_plan = normalize_policy(desired) - exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials - change = "noop" if before == after_plan else ("update" if exists else "create") - else: - desired = dict(credentials) - desired.pop("temp_unschedulable_enabled", None) - desired.pop("temp_unschedulable_rules", None) - after_plan = normalize_policy(desired) - change = "delete" if before != after_plan or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" include_plan_rules = PAYLOAD.get("full") is True - plan = { - "change": change, - "account": account_summary(detail, include_plan_rules), - "before": policy_summary(before, include_plan_rules), - "after": policy_summary(after_plan, include_plan_rules), - "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, - "credentialsPrinted": False, - } - if PAYLOAD.get("confirm") is not True or change == "noop": - return {**base, "ok": True, "operation": action, "mode": "dry-run" if PAYLOAD.get("confirm") is not True else "confirmed-noop", "mutation": False, "plan": plan} - updated = data_of(curl_api("PUT", f"/api/v1/admin/accounts/{detail['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") - after_detail = updated if isinstance(updated, dict) else account_detail(token, detail["id"]) - return {**base, "ok": True, "operation": action, "mode": "confirmed", "mutation": True, "plan": plan, "account": account_summary(after_detail, include_plan_rules)} + details = selected_account_details(token, accounts) + plans = [mutation_plan(item, action, include_plan_rules) for item in details] + changed = sum(1 for plan in plans if plan["change"] != "noop") + if PAYLOAD.get("confirm") is not True: + items = [] + for plan in plans: + item = {key: value for key, value in plan.items() if not key.startswith("_")} + item["write"] = {"attempted": False, "status": "dry-run", "error": None} + item["actual"] = item["before"] + item["reconciled"] = None + item["reconcileError"] = None + items.append(item) + summary = {"selected": len(items), "changed": changed, "writeSucceeded": 0, "writeFailed": 0, "reconciled": 0, "mismatched": 0} + return {**base, "ok": True, "operation": action, "mode": "dry-run", "mutation": False, "partialWrite": False, "summary": summary, "items": items, "credentialsPrinted": False} + writes = [write_mutation_plan(token, plan) for plan in plans] + items = [reconcile_mutation_plan(token, plan, write, include_plan_rules) for plan, write in zip(plans, writes)] + write_succeeded = sum(1 for item in items if item["write"]["status"] == "succeeded") + write_failed = sum(1 for item in items if item["write"]["status"] == "failed") + reconciled = sum(1 for item in items if item["reconciled"] is True) + mismatched = sum(1 for item in items if item["reconciled"] is False) + summary = {"selected": len(items), "changed": changed, "writeSucceeded": write_succeeded, "writeFailed": write_failed, "reconciled": reconciled, "mismatched": mismatched} + ok = write_failed == 0 and mismatched == 0 + return {**base, "ok": ok, "operation": action, "mode": "confirmed", "mutation": write_succeeded > 0, "partialWrite": write_succeeded > 0 and write_failed > 0, "summary": summary, "items": items, "credentialsPrinted": False} try: result = runtime_result() diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 347d4685..aee11961 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -352,7 +352,9 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account ] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account --template [--target PK01] [--confirm]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --accounts --template [--target PK01] [--confirm] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account --kind temp-unschedulable [--target PK01] [--confirm]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --accounts --kind temp-unschedulable [--target PK01] [--confirm] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm", From aed6cba625bf1fc8274ecc41961c8ed69187c864 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 11:12:03 +0200 Subject: [PATCH 02/12] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20Sub2API=20?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=8F=8D=E9=A6=88=E5=8F=AA=E8=AF=BB=E8=BF=BD?= =?UTF-8?q?=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/reference/cli.md | 2 +- docs/reference/sub2api-user-feedback.md | 61 ++ .../platform-infra-sub2api-codex/feedback.ts | 593 ++++++++++++++++++ .../src/platform-infra-sub2api-codex/index.ts | 1 + .../platform-infra-sub2api-codex/options.ts | 63 +- .../src/platform-infra-sub2api-codex/types.ts | 14 +- scripts/src/platform-infra/entry.ts | 2 + 7 files changed, 732 insertions(+), 4 deletions(-) create mode 100644 docs/reference/sub2api-user-feedback.md create mode 100644 scripts/src/platform-infra-sub2api-codex/feedback.ts diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d8da82dc..2f7a0ac5 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -164,7 +164,7 @@ PipelineRun 失败或长时间未完成时，先按定点 `control-plane status - `hwlab g14 git-mirror status|apply|sync|flush [--dry-run|--confirm]` 是 `devops-infra` git mirror/relay 的受控维护入口：`apply` 渲染并 server-side apply `devops-infra/git-mirror.yaml`，同时删除遗留 `git-mirror-hwlab-sync` CronJob；`sync` 创建一次性 manual Job，把 GitHub allowlist refs 拉入本地 mirror；`flush` 创建一次性 manual Job，把本地 `v0.2-gitops` 快进推回 GitHub。 `status` 返回 read/write URL、last sync/write/flush、本地 ref、GitHub staging ref 和 pending flush 状态，并在 `cache.summary` 给出 `localV02`、`localGitops`、`githubGitops`、`pendingFlush`、`flushNeeded`、`githubInSync` 和下一条受控 `flushCommand`。confirmed `sync` 和 `flush` 默认创建 `.state/jobs/` 异步 job 并立刻返回可查询状态，只有现场同步调试才显式加 `--wait`；mirror 不设置 CronJob。 如果 PipelineRun 的 `gitops-promote` 阶段报 git mirror 控制面漂移或 refs 不一致，先执行 `hwlab g14 git-mirror apply --confirm` 重新应用当前 `devops-infra/git-mirror.yaml` hook/ConfigMap，再执行 `hwlab g14 git-mirror sync --confirm --wait` 复核 refs；失败的同名 PipelineRun 只能通过 `hwlab g14 control-plane cleanup-runs --lane --pipeline-run --confirm` 受控清理后重试，不要用原生 `kubectl delete` 或手工改 mirror hook。修复后仍必须用 `control-plane status --pipeline-run ` 和 `git-mirror status` 分别确认 runtime closeout 与 GitHub flush。 -- `platform-infra sub2api|langbot|n8n|wechat-archive ...` 是 `platform-infra` namespace 内平台公共服务和公共工作流的受控入口；`sub2api` 支持 `plan|apply|status|validate|codex-pool`，`langbot` 和 `n8n` 支持各自 YAML-controlled `plan|apply|status|logs|validate` 等公共服务操作，`wechat-archive` 支持 `plan|apply|status|validate|pull`，用 `config/platform-infra/wechat-archive.yaml` 声明 LangBot/n8n/Baidu Netdisk 归档链路，以及 D601 Windows 隔离 PC 微信 + WeChatFerry host + D601 k3s 只读 collector 的 personal WeChat upstream；collector 必须复用 D601 已有 `platform-infra` namespace，`createNamespace=false`。`--target` 选择运行目标，默认 `G14` 为 active runtime，`D601` 为同一 YAML 控制的 standby predeploy target。镜像版本和 target 边界由 `config/platform-infra/*.yaml` 控制，Codex 上游池、统一 API key Secret、FRP 公网端口和 master `~/.codex` 消费端由 `config/platform-infra/sub2api-codex-pool.yaml` 控制；完整 Sub2API 日常部署、上游增删、FRP 暴露、local Codex 配置、验收和排障步骤统一见 `$unidesk-sub2api`（`.agents/skills/unidesk-sub2api/SKILL.md`）。`docs/reference/platform-infra.md` 保留 namespace、YAML-first、路由、Secret 脱敏、PK01 Caddy+FRP 和探针开发边界。 +- `platform-infra sub2api|langbot|n8n|wechat-archive ...` 是 `platform-infra` namespace 内平台公共服务和公共工作流的受控入口；`sub2api` 支持 `plan|apply|status|validate|codex-pool`，其中用户反馈一键只读追踪见 `docs/reference/sub2api-user-feedback.md`；`langbot` 和 `n8n` 支持各自 YAML-controlled `plan|apply|status|logs|validate` 等公共服务操作，`wechat-archive` 支持 `plan|apply|status|validate|pull`，用 `config/platform-infra/wechat-archive.yaml` 声明 LangBot/n8n/Baidu Netdisk 归档链路，以及 D601 Windows 隔离 PC 微信 + WeChatFerry host + D601 k3s 只读 collector 的 personal WeChat upstream；collector 必须复用 D601 已有 `platform-infra` namespace，`createNamespace=false`。`--target` 选择运行目标，默认 `G14` 为 active runtime，`D601` 为同一 YAML 控制的 standby predeploy target。镜像版本和 target 边界由 `config/platform-infra/*.yaml` 控制，Codex 上游池、统一 API key Secret、FRP 公网端口和 master `~/.codex` 消费端由 `config/platform-infra/sub2api-codex-pool.yaml` 控制；完整 Sub2API 日常部署、上游增删、FRP 暴露、local Codex 配置、验收和排障步骤统一见 `$unidesk-sub2api`（`.agents/skills/unidesk-sub2api/SKILL.md`）。`docs/reference/platform-infra.md` 保留 namespace、YAML-first、路由、Secret 脱敏、PK01 Caddy+FRP 和探针开发边界。 - `platform-infra observability plan|apply|status|validate|trace --target ` 是 `platform-infra` 内 OTel Collector 和 Tempo trace backend 的受控入口，`--target` 必须选择 trace 所属运行面节点；具体 trace 查询、噪声压制、Code Agent/AgentRun 排障和“先改进 OTel”的操作面统一见 `$unidesk-otel`（`.agents/skills/unidesk-otel/SKILL.md`）。 - `secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra` 是平台服务本地 Secret sourceRef 到 Kubernetes Secret key 的受控下发入口。`plan` 只读展示 sourceRef、必需 key、目标 Secret/key、missingKeys 和 fingerprint；`sync --confirm` 只按 YAML 声明创建允许生成的本地 key 并下发到声明的目标 Secret；`status` 只验证 live Secret key presence。该入口禁止从 live pod 或 Kubernetes Secret 反推密码、API key、JWT secret、n8n encryption key 或 `DATABASE_URL`，输出也不得打印 base64、解码值或远端 raw transcript；即使显式 `--raw` 也只返回脱敏 summary 和 raw omission 标记。LangBot/n8n Secret 轮换和缺 key 修复规则见 `docs/reference/platform-infra.md#secret-distribution-boundary`。 - `hwlab g14 observability status|apply|query|targets|boundary|closeout [--lane v02] [--promql ] [--expect-count N] [--expect-value V] [--dry-run|--confirm]` 是 G14 `devops-infra` 共享监控基础设施和 HWLAB v0.2 监控 closeout 的受控入口。`apply` 固定安装 Prometheus Operator `v0.91.0`、Prometheus `v3.12.0`、Prometheus 发现 RBAC、`devops-infra` 内 Prometheus 实例和 ClusterIP query Service，并给被允许发现的 workload namespace 打低风险 label；它不把 Prometheus、Grafana 或 Alertmanager 部署到 `hwlab-v02`，也不接管 HWLAB runtime Deployment/Service。`status` 只读汇总 CRD、operator Deployment、Prometheus CR/pod/service、`hwlab-v02` ServiceMonitor/PrometheusRule 和 bounded `up` 查询；`query` 只通过 Kubernetes service proxy 查询 Prometheus，支持 `--expect-count` / `--expect-value` 输出 `assertion`、bad values 和 missing/extra series；`targets` 汇总 ServiceMonitor/PrometheusRule、metrics sidecar readiness/restart、三层指标值和 `metrics.k8s.io` 当前 CPU/内存资源快照；`boundary` 验证 workload namespace 没有 Prometheus/Alertmanager，并对 `19666/19667` 公网 `/metrics` 做负向验证；`closeout` 聚合平台 ready、scrape reachable、sidecar serving、business health probe、resource snapshot、namespace boundary 和 public metrics exposure 语义结论。长期边界见 `docs/reference/g14-observability-infra.md`。 diff --git a/docs/reference/sub2api-user-feedback.md b/docs/reference/sub2api-user-feedback.md new file mode 100644 index 00000000..b56d700c --- /dev/null +++ b/docs/reference/sub2api-user-feedback.md @@ -0,0 +1,61 @@ +# Sub2API 用户反馈只读追踪 + +## 入口 + +- 按用户邮箱或正整数用户 ID 查询： + + ```bash + bun scripts/cli.ts platform-infra sub2api codex-pool feedback \ + --target PK01 \ + --user + ``` + +- 默认窗口为 `30m`；可显式选择 `5m|30m|1h|6h|24h|7d|30d`。 +- 默认输出 Kubernetes 列表风格紧凑文本；只有显式 `--json` 才输出完整脱敏 JSON。 +- 请求表固定返回 10 条，并输出 `returned`、`total`、`moreAvailable`、`nextPageToken`；下一页使用 CLI 给出的完整 `Next` 命令，不提供手工 `limit`。 +- `REQUEST ID INDEX` 输出当前页完整、可复制的 request ID；主表使用页内序号避免重复宽字段。 +- 按默认表格中的 client request ID 或内部 request ID 下钻： + + ```bash + bun scripts/cli.ts platform-infra sub2api codex-pool feedback \ + --target PK01 \ + --user \ + --request-id + ``` + +- 稳定游标下一页： + + ```bash + bun scripts/cli.ts platform-infra sub2api codex-pool feedback \ + --target PK01 \ + --user \ + --window 24h \ + --page-token + ``` + +## 证据组合 + +- 用户与 API Key：使用 Sub2API 原生 `admin users` 和用户 API Key 列表，输出邮箱、名称、分组与当前并发，不输出 Key 值。 +- 请求时间线：自动分页读取窗口内全部 `ops requests`，逐项显示相邻请求空窗、模型、状态、耗时、账号和 proxy。 +- ID 关联：一次读取 `ops system-logs`，在本地关联 client request ID 与内部 request ID；指定 `--request-id` 时披露关联日志。 +- 实时状态：组合用户、账号和分组并发，区分无入站证据、排队、在途与已完成。 +- 基础设施：组合账号可用性、账号当前状态、proxy 状态和 Sub2API 组件健康摘要。 + +## 归因边界 + +- `client-before-submit-or-no-inbound-evidence` 表示窗口内没有 Sub2API 入站记录且实时并发为零，不等价于证明客户端没有尝试调用。 +- `sub2api-queue` 需要实时等待队列证据；`in-flight` 需要实时并发证据。 +- `upstream-first-byte-stream`、`proxy-egress` 和 `client-connection` 只根据原生 Ops 日志、账号、proxy 与基础设施证据分类，不执行探针或写操作。 +- 无足够分层证据时返回 `sub2api-or-upstream-unknown`，禁止扩大结论。 + +## 脱敏与只读 + +- 入口只调用 GET 类 admin/ops 接口、管理员登录和运行组件健康读取。 +- 不调用账号测试、探针、调度、取消、清理、更新或其他运行面写接口。 +- JSON 和文本均不披露 API Key、管理员 token、账号 credentials、proxy 用户名、密码或其他 Secret。 +- 所有分页由 CLI 自动完成，不提供手工 `limit` 作为输出规避手段。 + +## 调用量 + +- issue `#2000` 的人工调查包含用户、请求、系统日志、两种 request ID trace、账号、proxy、进程/端口和 journal 共 9 次工具入口。 +- 本入口将用户侧工具调用收敛为 1 次，减少 8 次，即 `88.9%`；内部只读 API 来源在 JSON `sources` 字段披露。 diff --git a/scripts/src/platform-infra-sub2api-codex/feedback.ts b/scripts/src/platform-infra-sub2api-codex/feedback.ts new file mode 100644 index 00000000..d2e9f6e3 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/feedback.ts @@ -0,0 +1,593 @@ +import type { UniDeskConfig } from "../config"; +import type { RenderedCliResult } from "../output"; + +import { readCodexPoolConfig } from "./config"; +import { capture, compactCapture, parseJsonOutput } from "./remote"; +import { renderedCliResult, renderTable, shorten, shortIso, textValue } from "./render"; +import { codexPoolRuntimeTarget } from "./runtime-target"; +import type { CodexPoolConfig, CodexPoolRuntimeTarget, FeedbackOptions } from "./types"; + +function pyJson(value: unknown): string { + return `json.loads(${JSON.stringify(JSON.stringify(value))})`; +} + +export async function codexPoolFeedback(config: UniDeskConfig, options: FeedbackOptions): Promise | RenderedCliResult> { + const pool = readCodexPoolConfig(); + const target = codexPoolRuntimeTarget(options.targetId); + const result = await capture(config, target.route, ["sh"], feedbackScript(pool, options, target)); + const report = parseJsonOutput(result.stdout); + const ok = result.exitCode === 0 && report?.ok === true; + const remote = compactCapture(result, { full: result.exitCode !== 0 || report === null }); + if (options.json) { + return { + ok, + action: "platform-infra-sub2api-codex-pool-feedback", + target: { id: target.id, route: target.route, runtimeMode: target.runtimeMode }, + report, + remote, + valuesPrinted: false, + }; + } + return renderedCliResult(ok, "platform-infra sub2api codex-pool feedback", renderFeedbackReport(report, options, remote)); +} + +function feedbackScript(pool: CodexPoolConfig, options: FeedbackOptions, target: CodexPoolRuntimeTarget): string { + return ` +set -u +python3 - <<'PY' +import base64 +import json +import re +import subprocess +from datetime import datetime, timezone, timedelta +from urllib.parse import quote, urlencode + +RUNTIME_MODE = ${pyJson(target.runtimeMode)} +NAMESPACE = ${pyJson(target.namespace)} +APP_SECRET_NAME = ${pyJson(target.appSecretName)} +ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)} +HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} +HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)} +USER_SELECTOR = ${pyJson(options.user)} +WINDOW = ${pyJson(options.window)} +REQUEST_SELECTOR = ${pyJson(options.requestId)} +PAGE_TOKEN = ${pyJson(options.pageToken)} +APP_CONTAINER = "sub2api-app" +PAGE_SIZE = 10 + +def run(command, input_bytes=None): + return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +def text(value, limit=1000): + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + return str(value or "")[-limit:] + +def docker(args): + proc = run(["docker", *args]) + if proc.returncode == 0: + return proc + fallback = run(["sudo", "-n", "docker", *args]) + return fallback if fallback.returncode == 0 else proc + +def read_host_env(): + if RUNTIME_MODE != "host-docker": + return {} + try: + with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except PermissionError: + proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH]) + if proc.returncode != 0: + raise RuntimeError("read host-docker env failed: " + text(proc.stderr)) + lines = proc.stdout.decode("utf-8", errors="replace").splitlines() + values = {} + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + values[key.strip()] = value + return values + +def runtime_secret(key): + if RUNTIME_MODE == "host-docker": + return read_host_env().get(key) + proc = run(["kubectl", "-n", NAMESPACE, "get", "secret", APP_SECRET_NAME, "-o", "json"]) + if proc.returncode != 0: + raise RuntimeError("read app secret failed: " + text(proc.stderr)) + raw = (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key) + return base64.b64decode(raw).decode("utf-8") if raw else None + +def runtime_config(key): + if RUNTIME_MODE == "host-docker": + return read_host_env().get(key) + proc = run(["kubectl", "-n", NAMESPACE, "get", "configmap", "sub2api-config", "-o", "json"]) + if proc.returncode != 0: + return None + return (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key) + +def parse_curl(proc): + stdout = proc.stdout.decode("utf-8", errors="replace") + marker = "\\n__HTTP_CODE__:" + position = stdout.rfind(marker) + if position < 0: + return {"ok": False, "httpStatus": 0, "json": None, "message": text(proc.stderr)} + body = stdout[:position] + try: + status = int(stdout[position + len(marker):].strip()[-3:]) + except Exception: + status = 0 + try: + parsed = json.loads(body) if body.strip() else None + except Exception: + parsed = None + message = parsed.get("message") if isinstance(parsed, dict) else text(body) + return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "message": message} + +def curl_api(method, path, bearer=None, payload=None): + body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") + script = r'''set -eu +method="$1" +url="$2" +token="\${3:-}" +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +cat > "$tmp" +args="" +if [ -n "$token" ]; then args="Authorization: Bearer $token"; fi +if [ "$method" = GET ] && [ ! -s "$tmp" ]; then + if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -H "$args" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' "$url"; fi +else + if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "$args" --data-binary @"$tmp" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"; fi +fi''' + if RUNTIME_MODE == "host-docker": + proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body) + else: + proc = run(["kubectl", "-n", NAMESPACE, "exec", "deploy/sub2api", "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body) + return parse_curl(proc) + +def api_data(method, path, token=None, payload=None, label=None): + response = curl_api(method, path, token, payload) + parsed = response.get("json") + code = parsed.get("code") if isinstance(parsed, dict) else None + if not response.get("ok") or (code is not None and code != 0): + raise RuntimeError(f"{label or path} failed: http={response.get('httpStatus')} message={response.get('message')}") + return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed + +def find_token(value): + if isinstance(value, dict): + for key in ("access_token", "token"): + if isinstance(value.get(key), str) and value.get(key): + return value[key] + for item in value.values(): + token = find_token(item) + if token: + return token + return None + +def login(): + email = runtime_config("ADMIN_EMAIL") or ADMIN_EMAIL_DEFAULT + password = runtime_secret("ADMIN_PASSWORD") + if not password: + raise RuntimeError("ADMIN_PASSWORD missing") + token = find_token(api_data("POST", "/api/v1/auth/login", payload={"email": email, "password": password}, label="admin login")) + if not token: + raise RuntimeError("admin login returned no token") + return email, token + +def items(data): + if isinstance(data, list): + return data + if isinstance(data, dict): + if isinstance(data.get("items"), list): + return data["items"] + for key in ("users", "api_keys", "keys", "groups", "accounts", "proxies", "logs"): + if isinstance(data.get(key), list): + return data[key] + return [] + +def page_all(token, path, query=None, page_size=100): + query = dict(query or {}) + collected = [] + page = 1 + while True: + params = {**query, "page": page, "page_size": page_size} + separator = "&" if "?" in path else "?" + data = api_data("GET", path + separator + urlencode(params), token, label=path) + batch = items(data) + collected.extend(batch) + total = data.get("total") if isinstance(data, dict) else None + if not batch or (isinstance(total, int) and len(collected) >= total) or len(batch) < page_size: + break + page += 1 + return collected + +def exact_user(token): + if USER_SELECTOR.isdigit() and int(USER_SELECTOR) > 0: + user = api_data("GET", f"/api/v1/admin/users/{int(USER_SELECTOR)}", token, label="get user") + return user if isinstance(user, dict) else None + candidates = page_all(token, "/api/v1/admin/users", {"search": USER_SELECTOR}, 100) + exact = [user for user in candidates if isinstance(user, dict) and str(user.get("email") or "").lower() == USER_SELECTOR.lower()] + if len(exact) > 1: + raise RuntimeError("email matched multiple users") + return exact[0] if exact else None + +def safe_call(errors, name, callback, fallback): + try: + return callback() + except Exception as exc: + errors.append({"source": name, "error": redact_text(str(exc))}) + return fallback + +def redact_text(value): + value = str(value or "") + value = re.sub(r"(?i)(bearer\\s+)[A-Za-z0-9._~+\\-/=]+", r"\\1", value) + value = re.sub(r"(?i)((?:api[_-]?key|token|password)\\s*[=:]\\s*)[^\\s,;]+", r"\\1", value) + value = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-", value) + return value[:500] + +def iso_epoch(value): + if not isinstance(value, str) or not value: + return None + try: + match = re.match(r"^(\\d{4}-\\d\\d-\\d\\d[T ]\\d\\d:\\d\\d:\\d\\d)(?:\\.(\\d+))?(Z|[+-]\\d\\d:?\\d\\d)$", value) + if not match: + return None + base = datetime.strptime(match.group(1).replace(" ", "T"), "%Y-%m-%dT%H:%M:%S") + fraction = int((match.group(2) or "0")[:6].ljust(6, "0")) + zone = match.group(3) + if zone == "Z": + offset = timezone.utc + else: + sign = 1 if zone[0] == "+" else -1 + digits = zone[1:].replace(":", "") + offset = timezone(sign * timedelta(hours=int(digits[:2]), minutes=int(digits[2:]))) + return base.replace(microsecond=fraction, tzinfo=offset).timestamp() + except Exception: + return None + +def encode_page_token(row): + payload = json.dumps({"v": 1, "at": row.get("at"), "requestId": row.get("requestId")}, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + +def decode_page_token(value): + if not isinstance(value, str) or not value: + return None + try: + padded = value + "=" * (-len(value) % 4) + parsed = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")) + except Exception: + raise RuntimeError("invalid page token") + if not isinstance(parsed, dict) or parsed.get("v") != 1 or not parsed.get("at") or not parsed.get("requestId"): + raise RuntimeError("invalid page token") + return parsed + +def dict_by_id(rows): + return {str(row.get("id")): row for row in rows if isinstance(row, dict) and row.get("id") is not None} + +def keyed_map(value, key): + source = value.get(key) if isinstance(value, dict) else None + return source if isinstance(source, dict) else {} + +def runtime_health(): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", APP_CONTAINER, "sub2api-redis"]) + if proc.returncode != 0: + return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)} + rows = [] + for item in json.loads(proc.stdout.decode("utf-8")): + state = item.get("State") or {} + health = state.get("Health") or {} + rows.append({"name": str(item.get("Name") or "").lstrip("/"), "running": state.get("Running"), "status": health.get("Status") or state.get("Status")}) + return {"ok": all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows} + proc = run(["kubectl", "-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api", "-o", "json"]) + if proc.returncode != 0: + return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)} + rows = [] + for item in json.loads(proc.stdout.decode("utf-8")).get("items") or []: + status = item.get("status") or {} + containers = status.get("containerStatuses") or [] + rows.append({"name": (item.get("metadata") or {}).get("name"), "running": status.get("phase") == "Running", "status": "ready" if containers and all(entry.get("ready") is True for entry in containers) else status.get("phase")}) + return {"ok": bool(rows) and all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows} + +def compact_log(row): + extra = row.get("extra") if isinstance(row.get("extra"), dict) else {} + allow = {} + for key in ("phase", "error_owner", "status_code", "duration_ms", "latency_ms", "time_to_first_token_ms", "upstream_status", "path", "stream"): + if key in extra: + allow[key] = extra.get(key) + return { + "id": row.get("id"), + "at": row.get("created_at"), + "level": row.get("level"), + "component": row.get("component"), + "message": redact_text(row.get("message")), + "requestId": row.get("request_id"), + "clientRequestId": row.get("client_request_id"), + "accountId": row.get("account_id"), + "extra": allow, + } + +def related_logs(request, logs): + ids = {str(request.get("request_id") or "")} + matched = [] + changed = True + while changed: + changed = False + for row in logs: + request_id = str(row.get("request_id") or "") + client_id = str(row.get("client_request_id") or "") + if row in matched or (request_id not in ids and client_id not in ids): + continue + matched.append(row) + before = len(ids) + ids.update(value for value in (request_id, client_id) if value) + changed = changed or len(ids) != before + return matched + +def classify(request, logs, account, proxy, queued): + evidence = " ".join([str(request.get("phase") or ""), str(request.get("message") or ""), *[str(log.get("component") or "") + " " + str(log.get("message") or "") for log in logs]]).lower() + if any(marker in evidence for marker in ("client disconnect", "broken pipe", "context canceled", "client canceled")): + return "client-connection" + if any(marker in evidence for marker in ("proxy", "dial tcp", "dns", "connection refused", "network")) or (proxy and proxy.get("status") not in (None, "active")): + return "proxy-egress" + if any(marker in evidence for marker in ("upstream", "provider", "first token", "stream")) or str(request.get("phase") or "") == "upstream": + return "upstream-first-byte-stream" + if queued > 0: + return "sub2api-queue" + if request.get("kind") == "error" or (isinstance(request.get("status_code"), int) and request.get("status_code") >= 400): + return "sub2api-or-upstream-unknown" + return "completed" + +def main(): + errors = [] + admin_email, token = login() + user = exact_user(token) + if not isinstance(user, dict): + return {"ok": False, "mode": "feedback", "error": "user-not-found", "selector": USER_SELECTOR, "valuesPrinted": False} + user_id = user.get("id") + groups = safe_call(errors, "groups", lambda: items(api_data("GET", "/api/v1/admin/groups/all", token, label="groups")), []) + keys = safe_call(errors, "api-keys", lambda: page_all(token, f"/api/v1/admin/users/{user_id}/api-keys", {}, 100), []) + requests = safe_call(errors, "ops-requests", lambda: page_all(token, "/api/v1/admin/ops/requests", {"user_id": user_id, "time_range": WINDOW, "kind": "all", "sort": "created_at_desc"}, 100), []) + logs = safe_call(errors, "ops-system-logs", lambda: page_all(token, "/api/v1/admin/ops/system-logs", {"user_id": user_id, "time_range": WINDOW}, 200), []) + concurrency = safe_call(errors, "ops-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/concurrency", token, label="concurrency"), {}) + user_concurrency = safe_call(errors, "ops-user-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/user-concurrency", token, label="user concurrency"), {}) + availability = safe_call(errors, "ops-account-availability", lambda: api_data("GET", "/api/v1/admin/ops/account-availability", token, label="account availability"), {}) + accounts = safe_call(errors, "accounts", lambda: page_all(token, "/api/v1/admin/accounts", {}, 100), []) + proxies = safe_call(errors, "proxies", lambda: page_all(token, "/api/v1/admin/proxies", {}, 100), []) + infrastructure = safe_call(errors, "infrastructure", runtime_health, {"ok": False, "mode": RUNTIME_MODE}) + + groups_by_id = dict_by_id(groups) + keys_by_id = dict_by_id(keys) + accounts_by_id = dict_by_id(accounts) + proxies_by_id = dict_by_id(proxies) + user_current_map = keyed_map(user_concurrency, "user") + account_concurrency_map = keyed_map(concurrency, "account") + account_availability_map = keyed_map(availability, "account") + current_user = user_current_map.get(str(user_id)) or {} + user_current = int(current_user.get("current_in_use") or user.get("current_concurrency") or 0) + user_waiting = int(current_user.get("waiting_in_queue") or 0) + + compact_keys = [] + for key in keys: + group = groups_by_id.get(str(key.get("group_id"))) or {} + compact_keys.append({ + "id": key.get("id"), "name": key.get("name"), "status": key.get("status"), + "groupId": key.get("group_id"), "groupName": group.get("name"), + "currentConcurrency": key.get("current_concurrency"), "keyPrinted": False, + }) + + timeline = [] + ordered = sorted([row for row in requests if isinstance(row, dict)], key=lambda row: iso_epoch(row.get("created_at")) or 0) + previous_epoch = None + for row in ordered: + matched_logs = related_logs(row, logs) + client_id = next((log.get("client_request_id") for log in matched_logs if log.get("client_request_id")), None) + account = accounts_by_id.get(str(row.get("account_id"))) or {} + proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {} + account_load = account_concurrency_map.get(str(row.get("account_id"))) or {} + queued = int(account_load.get("waiting_in_queue") or 0) + epoch = iso_epoch(row.get("created_at")) + gap = round(epoch - previous_epoch, 1) if epoch is not None and previous_epoch is not None else None + previous_epoch = epoch if epoch is not None else previous_epoch + timeline.append({ + "at": row.get("created_at"), "gapSeconds": gap, "kind": row.get("kind"), + "requestId": row.get("request_id"), "clientRequestId": client_id, + "model": row.get("model"), "statusCode": row.get("status_code"), "durationMs": row.get("duration_ms"), + "apiKeyId": row.get("api_key_id"), "apiKeyName": (keys_by_id.get(str(row.get("api_key_id"))) or {}).get("name"), + "accountId": row.get("account_id"), "accountName": account.get("name"), + "proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"), + "attribution": classify(row, matched_logs, account, proxy, queued), + "evidence": {"systemLogCount": len(matched_logs), "accountStatus": account.get("status"), "accountSchedulable": account.get("schedulable"), "proxyStatus": proxy.get("status")}, + }) + + all_timeline = timeline + max_gap = max([row.get("gapSeconds") for row in all_timeline if isinstance(row.get("gapSeconds"), (int, float))], default=None) + next_page_token = None + more_available = False + if REQUEST_SELECTOR: + selected_ids = {REQUEST_SELECTOR} + for row in all_timeline: + if row.get("requestId") == REQUEST_SELECTOR or row.get("clientRequestId") == REQUEST_SELECTOR: + selected_ids.update(value for value in (row.get("requestId"), row.get("clientRequestId")) if isinstance(value, str) and value) + timeline = [row for row in all_timeline if any(isinstance(value, str) and value in selected_ids for value in (row.get("requestId"), row.get("clientRequestId")))] + detail_logs = [compact_log(row) for row in logs if any(isinstance(value, str) and value in selected_ids for value in (row.get("request_id"), row.get("client_request_id")))] + else: + detail_logs = [] + ordered_desc = list(reversed(all_timeline)) + cursor = decode_page_token(PAGE_TOKEN) + start = 0 + if cursor: + start = next((index + 1 for index, row in enumerate(ordered_desc) if row.get("at") == cursor.get("at") and row.get("requestId") == cursor.get("requestId")), -1) + if start < 0: + raise RuntimeError("page token cursor is no longer present in the selected window") + timeline = ordered_desc[start:start + PAGE_SIZE] + more_available = start + len(timeline) < len(ordered_desc) + if more_available and timeline: + next_page_token = encode_page_token(timeline[-1]) + if not requests and user_waiting > 0: + conclusion = "sub2api-queue" + elif not requests and user_current > 0: + conclusion = "in-flight" + elif not requests: + conclusion = "client-before-submit-or-no-inbound-evidence" + elif timeline: + conclusion = timeline[0].get("attribution") + else: + conclusion = "request-selector-not-found" + + related_account_ids = {str(row.get("accountId")) for row in timeline if row.get("accountId") is not None} + account_evidence = [] + for account_id in sorted(related_account_ids): + account = accounts_by_id.get(account_id) or {} + proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {} + load = account_concurrency_map.get(account_id) or {} + available = account_availability_map.get(account_id) or {} + account_evidence.append({ + "id": account.get("id"), "name": account.get("name"), "status": account.get("status"), "schedulable": account.get("schedulable"), + "currentInUse": load.get("current_in_use"), "waitingInQueue": load.get("waiting_in_queue"), "maxCapacity": load.get("max_capacity"), + "available": available.get("is_available"), "rateLimited": available.get("is_rate_limited"), "overloaded": available.get("is_overloaded"), + "proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"), "proxyStatus": proxy.get("status"), + }) + + return { + "ok": True, + "mode": "feedback", + "selector": {"user": USER_SELECTOR, "requestId": REQUEST_SELECTOR, "pageToken": PAGE_TOKEN, "window": WINDOW}, + "user": {"id": user_id, "email": user.get("email"), "username": user.get("username"), "status": user.get("status"), "maxConcurrency": user.get("concurrency"), "currentInUse": user_current, "waitingInQueue": user_waiting}, + "apiKeys": compact_keys, + "lifecycle": {"notInbound": not requests and user_current == 0 and user_waiting == 0, "queued": user_waiting, "inFlight": user_current, "completed": len(requests)}, + "timeline": timeline, + "timelineSummary": {"returned": len(timeline), "total": len(all_timeline), "moreAvailable": more_available, "nextPageToken": next_page_token, "pageSize": PAGE_SIZE, "systemLogCount": len(logs), "maxGapSeconds": max_gap}, + "requestDetail": {"logs": detail_logs, "logCount": len(detail_logs)} if REQUEST_SELECTOR else None, + "accounts": account_evidence, + "infrastructure": infrastructure, + "conclusion": {"attribution": conclusion, "boundary": "只读证据归因；无入站记录不等于客户端未调用，运行中请求仅由实时并发佐证。"}, + "toolCallReduction": {"manualInvestigationCalls": 9, "feedbackCliCalls": 1, "reducedCalls": 8, "reductionPercent": 88.9, "basis": "issue-2000: user, requests, system logs, two request-id traces, account, proxy, process/port, journal"}, + "sources": ["admin users", "admin user api-keys", "ops requests", "ops system-logs", "ops concurrency", "ops user-concurrency", "ops account-availability", "admin accounts", "admin proxies", "runtime health"], + "errors": errors, + "admin": {"email": admin_email, "tokenPrinted": False}, + "valuesPrinted": False, + } + +try: + print(json.dumps(main(), ensure_ascii=False)) +except Exception as exc: + print(json.dumps({"ok": False, "mode": "feedback", "error": redact_text(str(exc)), "valuesPrinted": False}, ensure_ascii=False)) + raise +PY +`; +} + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function records(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +} + +function renderFeedbackReport(report: Record | null, options: FeedbackOptions, remote: Record): string { + if (report === null) { + return [ + `SUB2API FEEDBACK user=${options.user} unavailable`, + `remote_exit=${remote.exitCode ?? "?"} stdout_bytes=${remote.stdoutBytes ?? "?"} stderr_bytes=${remote.stderrBytes ?? "?"}`, + textValue(remote.stderrTail ?? remote.stdoutTail), + ].join("\n"); + } + if (report.ok !== true) { + return [ + `SUB2API FEEDBACK user=${options.user} ok=false`, + `error=${textValue(report.error)}`, + "JSON: add --json for the structured failure envelope.", + ].join("\n"); + } + const user = record(report.user); + const lifecycle = record(report.lifecycle); + const summary = record(report.timelineSummary); + const conclusion = record(report.conclusion); + const reduction = record(report.toolCallReduction); + const timeline = records(report.timeline); + const accounts = records(report.accounts); + const apiKeys = records(report.apiKeys); + const detail = record(report.requestDetail); + const detailLogs = records(detail.logs); + const infrastructure = record(report.infrastructure); + const components = records(infrastructure.components); + const errors = records(report.errors); + const lines: string[] = []; + lines.push(`SUB2API FEEDBACK user=${textValue(user.email)}#${textValue(user.id)} window=${options.window} ok=true`); + lines.push(`CONCLUSION attribution=${textValue(conclusion.attribution)} boundary=${textValue(conclusion.boundary)}`); + lines.push(`LIFECYCLE not_inbound=${textValue(lifecycle.notInbound)} queued=${textValue(lifecycle.queued)} in_flight=${textValue(lifecycle.inFlight)} completed=${textValue(lifecycle.completed)} max_gap_s=${textValue(summary.maxGapSeconds)}`); + lines.push(`TOOLS manual=${textValue(reduction.manualInvestigationCalls)} cli=${textValue(reduction.feedbackCliCalls)} reduced=${textValue(reduction.reducedCalls)} reduction=${textValue(reduction.reductionPercent)}%`); + if (apiKeys.length > 0) { + lines.push(""); + lines.push("API KEYS"); + lines.push(renderTable([ + ["ID", "NAME", "GROUP", "STATUS", "CURRENT"], + ...apiKeys.map((key) => [textValue(key.id), shorten(textValue(key.name), 30), shorten(textValue(key.groupName), 24), textValue(key.status), textValue(key.currentConcurrency)]), + ])); + } + lines.push(""); + lines.push(`REQUESTS returned=${textValue(summary.returned)} total=${textValue(summary.total)} moreAvailable=${textValue(summary.moreAvailable)} system_logs=${textValue(summary.systemLogCount)}`); + if (timeline.length === 0) { + lines.push("No matching completed request records in the selected window."); + } else { + lines.push(renderTable([ + ["#", "AT", "GAP_S", "STATUS", "DURATION", "MODEL", "ACCOUNT", "ATTRIBUTION"], + ...timeline.map((item, index) => [ + String(index + 1), shortIso(item.at), textValue(item.gapSeconds), textValue(item.statusCode ?? item.kind), textValue(item.durationMs), + shorten(textValue(item.model), 18), shorten(`${textValue(item.accountName)}#${textValue(item.accountId)}`, 28), textValue(item.attribution), + ]), + ])); + lines.push(""); + lines.push("REQUEST ID INDEX"); + lines.push(renderTable([ + ["#", "REQUEST_ID", "CLIENT_ID"], + ...timeline.map((item, index) => [String(index + 1), textValue(item.requestId), textValue(item.clientRequestId)]), + ])); + } + if (accounts.length > 0 || components.length > 0) { + lines.push(""); + lines.push("ACCOUNT / INFRA"); + lines.push(renderTable([ + ["ACCOUNT", "STATUS", "SCHED", "IN_USE", "QUEUE", "CAP", "AVAILABLE", "PROXY", "P_STATUS"], + ...accounts.map((item) => [ + shorten(`${textValue(item.name)}#${textValue(item.id)}`, 32), textValue(item.status), textValue(item.schedulable), + textValue(item.currentInUse), textValue(item.waitingInQueue), textValue(item.maxCapacity), textValue(item.available), + shorten(`${textValue(item.proxyName)}#${textValue(item.proxyId)}`, 24), textValue(item.proxyStatus), + ]), + ...components.map((item) => [shorten(textValue(item.name), 32), textValue(item.status), "-", "-", "-", "-", textValue(item.running), "runtime", textValue(infrastructure.mode)]), + ])); + } + if (options.requestId !== null) { + lines.push(""); + lines.push(`REQUEST DETAIL id=${options.requestId} logs=${textValue(detail.logCount)}`); + if (detailLogs.length > 0) { + lines.push(renderTable([ + ["LOG_ID", "AT", "LEVEL", "COMPONENT", "REQUEST_ID", "CLIENT_ID", "MESSAGE"], + ...detailLogs.map((item) => [ + textValue(item.id), shortIso(item.at), textValue(item.level), shorten(textValue(item.component), 22), + shorten(textValue(item.requestId), 18), shorten(textValue(item.clientRequestId), 18), shorten(textValue(item.message), 64), + ]), + ])); + } + } + if (errors.length > 0) { + lines.push(""); + lines.push("PARTIAL EVIDENCE"); + lines.push(renderTable([["SOURCE", "ERROR"], ...errors.map((item) => [textValue(item.source), shorten(textValue(item.error), 90)])])); + } + lines.push(""); + if (summary.moreAvailable === true && typeof summary.nextPageToken === "string") { + lines.push(`NEXT_PAGE_TOKEN ${summary.nextPageToken}`); + lines.push(`Next: bun scripts/cli.ts platform-infra sub2api codex-pool feedback --target ${options.targetId} --user ${options.user} --window ${options.window} --page-token ${summary.nextPageToken}`); + } + lines.push("Disclosure: rerun with --request-id for correlated indexed logs."); + lines.push("JSON: add --json for the complete redacted report."); + return lines.join("\n"); +} diff --git a/scripts/src/platform-infra-sub2api-codex/index.ts b/scripts/src/platform-infra-sub2api-codex/index.ts index 51e6ff6f..9ec51ffa 100644 --- a/scripts/src/platform-infra-sub2api-codex/index.ts +++ b/scripts/src/platform-infra-sub2api-codex/index.ts @@ -13,3 +13,4 @@ export * from "./remote-scripts"; export * from "./accounts"; export * from "./remote-python-sync"; export * from "./remote"; +export * from "./feedback"; diff --git a/scripts/src/platform-infra-sub2api-codex/options.ts b/scripts/src/platform-infra-sub2api-codex/options.ts index a17ddade..2006d7d5 100644 --- a/scripts/src/platform-infra-sub2api-codex/options.ts +++ b/scripts/src/platform-infra-sub2api-codex/options.ts @@ -21,8 +21,9 @@ import { import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets"; import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; -import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types"; +import type { ConfirmOptions, DisclosureOptions, FeedbackOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types"; import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions"; +import { codexPoolFeedback } from "./feedback"; import { renderCodexPoolPlan } from "./render"; import { codexPoolRuntime } from "./runtime"; import { defaultCodexPoolRuntimeTargetId } from "./runtime-target"; @@ -39,6 +40,7 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]) if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1))); if (action === "runtime") return await codexPoolRuntime(config, args.slice(1)); if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1))); + if (action === "feedback") return await codexPoolFeedback(config, parseFeedbackOptions(args.slice(1))); if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1))); if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1))); if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1))); @@ -181,6 +183,7 @@ export function parseSentinelReportOptions(args: string[]): SentinelReportOption export function parseTraceOptions(args: string[]): TraceOptions { let requestId: string | null = null; + let pageToken: string | null = null; let since = "24h"; let tail = 20_000; let contextSeconds = 300; @@ -258,6 +261,64 @@ export function parseTraceOptions(args: string[]): TraceOptions { return { ...disclosure, requestId, since, tail, contextSeconds, showLines }; } +export function parseFeedbackOptions(args: string[]): FeedbackOptions { + let user: string | null = null; + let window: FeedbackOptions["window"] = "30m"; + let requestId: string | null = null; + let pageToken: string | null = null; + let json = false; + const targetArgs: string[] = []; + const windows = new Set(["5m", "30m", "1h", "6h", "24h", "7d", "30d"]); + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]!; + if (arg === "--json") { + json = true; + continue; + } + if (arg === "--target" || arg.startsWith("--target=")) { + targetArgs.push(arg); + if (arg === "--target") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); + targetArgs.push(value); + index += 1; + } + continue; + } + const readValue = (option: string): string => { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${option} requires a value`); + index += 1; + return value.trim(); + }; + if (arg === "--user") user = readValue(arg); + else if (arg.startsWith("--user=")) user = arg.slice("--user=".length).trim(); + else if (arg === "--window") { + const value = readValue(arg) as FeedbackOptions["window"]; + if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d"); + window = value; + } else if (arg.startsWith("--window=")) { + const value = arg.slice("--window=".length) as FeedbackOptions["window"]; + if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d"); + window = value; + } else if (arg === "--request-id") requestId = readValue(arg); + else if (arg.startsWith("--request-id=")) requestId = arg.slice("--request-id=".length).trim(); + else if (arg === "--page-token") pageToken = readValue(arg); + else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); + else throw new Error(`unsupported option: ${arg}`); + } + if (user === null || user.length === 0) throw new Error("feedback requires --user "); + if (user.length > 320 || /[\r\n]/u.test(user)) throw new Error("--user must be a single email or positive user id"); + if (requestId !== null && (requestId.length === 0 || requestId.length > 256 || /[\r\n]/u.test(requestId))) { + throw new Error("--request-id must be a non-empty stable request id up to 256 characters"); + } + if (pageToken !== null && (pageToken.length === 0 || pageToken.length > 1024 || !/^[A-Za-z0-9_-]+$/u.test(pageToken))) { + throw new Error("--page-token must be a URL-safe cursor token"); + } + if (requestId !== null && pageToken !== null) throw new Error("--request-id and --page-token cannot be combined"); + return { user, window, requestId, pageToken, json, targetId: parseTargetId(targetArgs) }; +} + export function parseKubectlDuration(raw: string): string { const value = raw.trim(); if (!/^[1-9][0-9]*(?:s|m|h)$/u.test(value)) throw new Error("--since must be a kubectl duration such as 24h, 90m, or 300s"); diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 347d4685..0133efe1 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -71,6 +71,15 @@ export interface TraceOptions extends DisclosureOptions { showLines: boolean; } +export interface FeedbackOptions { + user: string; + window: "5m" | "30m" | "1h" | "6h" | "24h" | "7d" | "30d"; + requestId: string | null; + pageToken: string | null; + json: boolean; + targetId: string; +} + export interface SentinelImageOptions extends DisclosureOptions { action: "status" | "build"; confirm: boolean; @@ -341,8 +350,8 @@ export function codexPoolHelp(): unknown { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(); return { - command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", - output: "json, except trace and sentinel-report default to low-noise text tables", + command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|feedback|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", + output: "json, except trace, feedback, and sentinel-report default to low-noise text tables", usage: [ "bun scripts/cli.ts platform-infra sub2api codex-pool plan", "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", @@ -354,6 +363,7 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account --template [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account --kind temp-unschedulable [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user [--window 30m] [--page-token |--request-id ] [--target PK01] [--json]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm", diff --git a/scripts/src/platform-infra/entry.ts b/scripts/src/platform-infra/entry.ts index d9116132..a62892b1 100644 --- a/scripts/src/platform-infra/entry.ts +++ b/scripts/src/platform-infra/entry.ts @@ -421,6 +421,7 @@ export function platformInfraHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool validate", "bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ", + "bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user [--window 30m] [--page-token |--request-id ] [--json]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm", "bun scripts/cli.ts platform-infra sub2rank plan [--target NC01]", @@ -503,6 +504,7 @@ export function platformInfraHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool validate", "bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ", + "bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user [--window 30m] [--page-token |--request-id ] [--json]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status", ], module: "scripts/src/platform-infra-sub2api-codex.ts", From 41c1d59ca0d17c57c51693f2b478685c05e8ae81 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:40:23 +0200 Subject: [PATCH 03/12] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20Sub2API=20?= =?UTF-8?q?=E6=95=85=E9=9A=9C=E7=AD=89=E7=BA=A7=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform-infra-sub2api-codex/faults.ts | 569 ++++++++++++++++++ .../platform-infra-sub2api-codex/options.ts | 2 + .../src/platform-infra-sub2api-codex/types.ts | 3 +- 3 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 scripts/src/platform-infra-sub2api-codex/faults.ts diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts new file mode 100644 index 00000000..8f89e410 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -0,0 +1,569 @@ +import { createHash } from "node:crypto"; + +import type { UniDeskConfig } from "../config"; +import { CliInputError, type RenderedCliResult } from "../output"; + +import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; +import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; + +type FaultLevel = "P0" | "P1" | "P2"; + +interface FaultOptions { + level: FaultLevel | null; + group: string | null; + account: string | null; + model: string | null; + stream: "sync" | "stream" | null; + endpoint: string | null; + requestId: string | null; + pageToken: string | null; + targetId: string; + json: boolean; +} + +const PAGE_SIZE = 20; + +export async function codexPoolFaults(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { + if (args.includes("--help")) return renderFaultHelp(); + const options = parseFaultOptions(args); + const target = codexPoolRuntimeTarget(options.targetId); + const scope = faultScope(options); + const offset = decodePageToken(options.pageToken, scope); + const payload = { + filters: { + level: options.level, + group: options.group, + account: options.account, + model: options.model, + stream: options.stream, + endpoint: options.endpoint, + requestId: options.requestId, + }, + offset, + pageSize: PAGE_SIZE, + }; + const result = await runRemoteCodexPoolScript(config, "faults", faultScript(payload, target), target); + const parsed = parseJsonOutput(result.stdout); + const ok = result.exitCode === 0 && boolField(parsed, "ok", false); + const data = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; + const pagination = data && typeof data.pagination === "object" && data.pagination !== null + ? data.pagination as Record + : null; + if (pagination?.hasMore === true) pagination.nextPageToken = encodePageToken(offset + PAGE_SIZE, scope); + if (pagination) { + pagination.pageSize = PAGE_SIZE; + pagination.pageToken = options.pageToken; + } + const response = { + ok, + action: "platform-infra-sub2api-codex-pool-faults", + target: { + id: target.id, + route: target.route, + runtimeMode: target.runtimeMode, + endpoint: target.serviceDns, + }, + source: { + kind: "sub2api-native-admin-ops", + versionBoundary: "Sub2API native admin Ops facts with explicit, version-neutral UniDesk CLI projection", + mutation: false, + }, + filters: payload.filters, + faults: data, + remote: compactCapture(result, { full: result.exitCode !== 0 || data === null }), + valuesPrinted: false, + }; + if (options.json) return response; + return renderFaults(response); +} + +function renderFaultHelp(): RenderedCliResult { + return { + ok: true, + command: "platform-infra sub2api codex-pool faults --help", + renderedText: [ + "SUB2API CODEX-POOL FAULTS", + "Usage:", + " bun scripts/cli.ts platform-infra sub2api codex-pool faults [options]", + "Options:", + " --level P0|P1|P2", + " --group ", + " --account ", + " --model ", + " --stream sync|stream", + " --endpoint ", + " --request-id ", + " --page-token ", + " --target ", + " --json", + "Notes:", + ` Fixed page size: ${PAGE_SIZE}; --limit is intentionally unsupported.`, + " Default output is a bounded Kubernetes-style table; JSON requires --json.", + ].join("\n"), + contentType: "text/plain", + }; +} + +function parseFaultOptions(args: string[]): FaultOptions { + let level: FaultLevel | null = null; + let group: string | null = null; + let account: string | null = null; + let model: string | null = null; + let stream: "sync" | "stream" | null = null; + let endpoint: string | null = null; + let requestId: string | null = null; + let pageToken: string | null = null; + let targetId = defaultCodexPoolRuntimeTargetId(); + let json = false; + const readValue = (index: number, name: string): [string, number] => { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); + return [validateSelector(value, name), index + 1]; + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]!; + if (arg === "--json") json = true; + else if (arg === "--level") { + const [value, next] = readValue(index, "--level"); + level = parseLevel(value); + index = next; + } else if (arg.startsWith("--level=")) level = parseLevel(arg.slice(8)); + else if (arg === "--group") [group, index] = readValue(index, "--group"); + else if (arg.startsWith("--group=")) group = validateSelector(arg.slice(8), "--group"); + else if (arg === "--account") [account, index] = readValue(index, "--account"); + else if (arg.startsWith("--account=")) account = validateSelector(arg.slice(10), "--account"); + else if (arg === "--model") [model, index] = readValue(index, "--model"); + else if (arg.startsWith("--model=")) model = validateSelector(arg.slice(8), "--model"); + else if (arg === "--stream") { + const [value, next] = readValue(index, "--stream"); + stream = parseStream(value); + index = next; + } else if (arg.startsWith("--stream=")) stream = parseStream(arg.slice(9)); + else if (arg === "--endpoint") [endpoint, index] = readValue(index, "--endpoint"); + else if (arg.startsWith("--endpoint=")) endpoint = validateSelector(arg.slice(11), "--endpoint"); + else if (arg === "--request-id") [requestId, index] = readValue(index, "--request-id"); + else if (arg.startsWith("--request-id=")) requestId = validateSelector(arg.slice(13), "--request-id"); + else if (arg === "--page-token") [pageToken, index] = readValue(index, "--page-token"); + else if (arg.startsWith("--page-token=")) pageToken = validateSelector(arg.slice(13), "--page-token"); + else if (arg === "--target") [targetId, index] = readValue(index, "--target"); + else if (arg.startsWith("--target=")) targetId = validateSelector(arg.slice(9), "--target"); + else if (arg === "--limit" || arg.startsWith("--limit=")) throw new Error("faults uses a fixed page size; use --page-token for progressive disclosure"); + else throw new Error(`unsupported faults option: ${arg}`); + } + if (level !== null && group === null) { + throw new CliInputError("--level requires --group; use the default command for the all-groups overview", { + code: "missing-required-option", + argument: "--group", + usage: [ + "bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01", + "bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01 --level P0 --group ", + ], + hint: "Run without --level for the all-groups overview, then drill down with --level and --group.", + }); + } + return { level, group, account, model, stream, endpoint, requestId, pageToken, targetId, json }; +} + +function parseLevel(value: string): FaultLevel { + const normalized = value.toUpperCase(); + if (normalized !== "P0" && normalized !== "P1" && normalized !== "P2") throw new Error("--level must be P0, P1, or P2"); + return normalized; +} + +function parseStream(value: string): "sync" | "stream" { + if (value !== "sync" && value !== "stream") throw new Error("--stream must be sync or stream"); + return value; +} + +function validateSelector(value: string, name: string): string { + const normalized = value.trim(); + if (!normalized || normalized.length > 512 || /[\r\n\0]/u.test(normalized)) throw new Error(`${name} has an unsupported format`); + return normalized; +} + +function faultScope(options: FaultOptions): string { + return createHash("sha256").update(JSON.stringify({ + level: options.level, + group: options.group, + account: options.account, + model: options.model, + stream: options.stream, + endpoint: options.endpoint, + requestId: options.requestId, + targetId: options.targetId, + })).digest("hex").slice(0, 16); +} + +function encodePageToken(offset: number, scope: string): string { + return Buffer.from(JSON.stringify({ v: 1, offset, scope }), "utf8").toString("base64url"); +} + +function decodePageToken(token: string | null, scope: string): number { + if (token === null) return 0; + try { + const parsed = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as Record; + if (parsed.v !== 1 || parsed.scope !== scope || !Number.isInteger(parsed.offset) || Number(parsed.offset) < 0) throw new Error("invalid"); + return Number(parsed.offset); + } catch { + throw new Error("--page-token is invalid or belongs to different filters"); + } +} + +function renderFaults(response: Record): RenderedCliResult { + const faults = response.faults && typeof response.faults === "object" ? response.faults as Record : {}; + const lines = ["PLATFORM-INFRA SUB2API FAULTS"]; + const window = record(faults.window); + lines.push(`WINDOW ${text(window.timeRange)} SOURCE native-admin-ops PROJECTION unidesk-cli`); + lines.push(""); + const summary = arrayOfRecords(faults.summary); + if (summary.length > 0) { + lines.push("GROUPS"); + lines.push(table(["GROUP", "P0", "CUSTOMER_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERR%", "ABSORB%"], summary.map((row) => [ + `${text(row.groupName)} (${text(row.groupId)})`, + text(row.p0), percent(row.customerErrorRatePercent), + text(row.p1), milliseconds(row.ttftP99Ms), + text(row.p2), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), + ]))); + } + const details = arrayOfRecords(faults.details); + if (details.length > 0) { + lines.push("", `DETAILS ${text(faults.level ?? "SUMMARY")}`); + const columns = Array.isArray(faults.detailColumns) ? faults.detailColumns.map(String) : []; + lines.push(table(columns, details.map((row) => columns.map((column) => text(row[column]))))); + } + const boundary = record(faults.boundary); + lines.push("", "BOUNDARY"); + for (const value of Array.isArray(boundary.notes) ? boundary.notes : []) lines.push(`- ${String(value)}`); + const pagination = record(faults.pagination); + if (pagination.nextPageToken) lines.push("", `NEXT_PAGE_TOKEN ${text(pagination.nextPageToken)}`); + if (faults.next) lines.push("", "NEXT", ...String(faults.next).split("\n").map((line) => ` ${line}`)); + return { + ok: response.ok === true, + command: "platform-infra sub2api codex-pool faults", + renderedText: lines.join("\n"), + contentType: "text/plain", + projection: response, + }; +} + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function arrayOfRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +} + +function text(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + if (typeof value === "boolean") return value ? "yes" : "no"; + return String(value).replace(/[\r\n\t]+/gu, " "); +} + +function percent(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : text(value); +} + +function milliseconds(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}ms` : text(value); +} + +function table(headers: string[], rows: string[][]): string { + const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length))); + return [headers, ...rows].map((row) => row.map((cell, index) => cell.padEnd(widths[index]!)).join(" ").trimEnd()).join("\n"); +} + +function pyJson(value: unknown): string { + return `json.loads(${JSON.stringify(JSON.stringify(value))})`; +} + +function faultScript(payload: unknown, target: ReturnType): string { + return ` +set -u +python3 - <<'PY' +import base64 +import json +import math +import subprocess +import sys +from urllib.parse import urlencode + +RUNTIME_MODE = ${pyJson(target.runtimeMode)} +NAMESPACE = ${pyJson(target.namespace)} +HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} +APP_SECRET_NAME = ${pyJson(target.appSecretName)} +PAYLOAD = ${pyJson(payload)} +APP_POD = None + +def run(cmd, input_bytes=None): + return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +def docker(args): + proc = run(["docker", *args]) + if proc.returncode == 0: + return proc + sudo_proc = run(["sudo", "-n", "docker", *args]) + return sudo_proc if sudo_proc.returncode == 0 else proc + +def kubectl(args, input_bytes=None): + return run(["kubectl", *args], input_bytes) + +def kube_json(args, label): + proc = kubectl([*args, "-o", "json"]) + if proc.returncode != 0: + raise RuntimeError(label + " failed") + return json.loads(proc.stdout.decode("utf-8")) + +if RUNTIME_MODE != "host-docker": + pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app=sub2api"], "list pods").get("items") or [] + ready = [item for item in pods if item.get("status", {}).get("phase") == "Running"] + if not ready: + raise RuntimeError("sub2api app pod not found") + APP_POD = ready[0]["metadata"]["name"] + +def config_value(name, key, default=None): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", "sub2api-app", "--format", "{{json .Config.Env}}"]) + if proc.returncode != 0: + return default + for item in json.loads(proc.stdout.decode("utf-8")): + if item.startswith(key + "="): + return item.split("=", 1)[1] + return default + data = kube_json(["-n", NAMESPACE, "get", "configmap", name], "configmap/" + name).get("data") or {} + return data.get(key, default) + +def secret_value(name, key): + if RUNTIME_MODE == "host-docker": + return config_value("", key) + data = kube_json(["-n", NAMESPACE, "get", "secret", name], "secret/" + name).get("data") or {} + return base64.b64decode(data[key]).decode("utf-8") if key in data else None + +def curl_api(method, path, bearer=None, payload=None): + body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") + script = r'''set -eu +method="$1"; url="$2"; token="\${3:-}"; tmp="$(mktemp)"; trap 'rm -f "$tmp"' EXIT; cat > "$tmp" +args=""; [ -n "$token" ] && args="Authorization: Bearer $token" +if [ -n "$args" ] && [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" +elif [ -n "$args" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" "$url" +elif [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" +else curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"; fi''' + base = f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}" if RUNTIME_MODE == "host-docker" else "http://127.0.0.1:8080" + cmd = ["sh", "-c", script, "sh", method, base + path, bearer or ""] + proc = run(cmd, body) if RUNTIME_MODE == "host-docker" else run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", *cmd], body) + output = proc.stdout.decode("utf-8", errors="replace") + marker = "\\n__HTTP_CODE__:" + pos = output.rfind(marker) + status = int(output[pos + len(marker):].strip()[-3:]) if pos >= 0 else 0 + raw = output[:pos] if pos >= 0 else output + try: + parsed = json.loads(raw) if raw.strip() else None + except json.JSONDecodeError: + parsed = None + return {"ok": proc.returncode == 0 and 200 <= status < 300, "status": status, "json": parsed, "body": raw[:300]} + +def data_of(response, label): + parsed = response.get("json") + code = parsed.get("code") if isinstance(parsed, dict) else None + if response.get("ok") is not True or (code is not None and code != 0): + message = parsed.get("message") if isinstance(parsed, dict) else response.get("body") + raise RuntimeError(f"{label} failed: http={response.get('status')} message={message}") + return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed + +def login(): + email = config_value("sub2api-config", "ADMIN_EMAIL", "admin@example.com") + password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") + if not password: + raise RuntimeError("ADMIN_PASSWORD missing") + data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login") + token = data.get("access_token") or data.get("token") if isinstance(data, dict) else None + if not token: + raise RuntimeError("admin login response has no token") + return token + +def get(token, path, params=None, label=None): + suffix = "?" + urlencode({key: value for key, value in (params or {}).items() if value is not None}) if params else "" + return data_of(curl_api("GET", path + suffix, bearer=token), label or path) + +def items_of(data): + if isinstance(data, list): + return data + if isinstance(data, dict): + for key in ("items", "groups", "accounts"): + if isinstance(data.get(key), list): + return data[key] + return [] + +def total_of(data): + return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data)) + +def percent(value): + if not isinstance(value, (int, float)): + return None + return round(value * 100 if value <= 1 else value, 4) + +def selector_id(value): + return int(value) if isinstance(value, str) and value.isdigit() else None + +def filters_for(group_id=None): + filters = PAYLOAD["filters"] + result = {"time_range": "24h", "platform": "openai", "group_id": group_id} + account = filters.get("account") + if account: + result["account_id" if selector_id(account) is not None else "account_name"] = selector_id(account) if selector_id(account) is not None else account + if filters.get("model"): + result["model"] = filters["model"] + if filters.get("stream"): + result["stream"] = "true" if filters["stream"] == "stream" else "false" + if filters.get("endpoint"): + result["inbound_endpoint"] = filters["endpoint"] + if filters.get("requestId"): + result["request_id"] = filters["requestId"] + return result + +def selected_groups(groups): + selector = PAYLOAD["filters"].get("group") + if not selector: + return groups + selected = [item for item in groups if str(item.get("id")) == str(selector) or item.get("name") == selector] + if len(selected) != 1: + raise RuntimeError("group-not-found-or-ambiguous: " + str(selector)) + return selected + +def threshold_state(value, threshold): + if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)): + return "unavailable" + return "breach" if value >= threshold else "ok" + +def availability_by_account(token, group_id): + data = get(token, "/api/v1/admin/ops/account-availability", {"platform": "openai", "group_id": group_id}, "account availability") + raw = data.get("account") if isinstance(data, dict) else None + return raw if isinstance(raw, dict) else {} + +def detail_row(item, group_metrics, availability, level, absorbed=None): + account_id = item.get("account_id") + state = availability.get(str(account_id), {}) if isinstance(availability, dict) else {} + endpoint = item.get("inbound_endpoint") or item.get("request_path") or item.get("path") + base = { + "GROUP": f"{item.get('group_name') or group_metrics.get('groupName')} ({item.get('group_id') or group_metrics.get('groupId')})", + "ACCOUNT": f"{item.get('account_name') or '-'} ({account_id or '-'})", + "MODEL": item.get("requested_model") or item.get("model") or "-", + "MODE": "stream" if item.get("stream") is True else "sync" if item.get("stream") is False else "-", + "ENDPOINT": endpoint or "-", + "STATUS": item.get("status_code") or "-", + "REQUEST_ID": item.get("request_id") or item.get("client_request_id") or "-", + } + if level == "P0": + base.update({"CUSTOMER_ERR%": group_metrics.get("customerErrorRatePercent"), "ABSORBED": False}) + else: + unavailable_reason = str(state.get("unavailable_reason") or state.get("reason") or "").lower() + if state.get("is_available") is False and ("temporary" in unavailable_reason or "cooldown" in unavailable_reason): + temp_unsched = "active" + elif state.get("is_available") is True: + temp_unsched = "none" + else: + temp_unsched = "unavailable" + base.update({"ROOT": item.get("error_source") or item.get("message") or "upstream", "TEMP_UNSCHED": temp_unsched, "ABSORBED": absorbed}) + return base + +def execute(): + token = login() + thresholds = get(token, "/api/v1/admin/ops/settings/metric-thresholds", label="metric thresholds") + groups = selected_groups(items_of(get(token, "/api/v1/admin/groups/all", {"platform": "openai"}, "list groups"))) + summaries = [] + group_data = [] + for group in groups: + group_id = group.get("id") + native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview") + request_errors = get(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") + upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") + customer_total = total_of(request_errors) + upstream_total = total_of(upstream_errors) + absorbed = max(0, upstream_total - customer_total) + absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None + customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None) + upstream_rate = percent(native.get("upstream_error_rate") if isinstance(native, dict) else None) + ttft = native.get("ttft") if isinstance(native, dict) and isinstance(native.get("ttft"), dict) else {} + metrics = { + "groupId": group_id, + "groupName": group.get("name"), + "customerErrorCount": customer_total, + "customerErrorRatePercent": customer_rate, + "upstreamErrorCount": upstream_total, + "upstreamErrorRatePercent": upstream_rate, + "absorbedCountProjection": absorbed, + "absorbedPercent": absorbed_percent, + "ttftP99Ms": ttft.get("p99_ms"), + } + metrics["p0"] = "active" if customer_total > 0 else "clear" + metrics["p1"] = threshold_state(metrics["ttftP99Ms"], thresholds.get("ttft_p99_ms_max")) + metrics["p2"] = threshold_state(upstream_rate, thresholds.get("upstream_error_rate_percent_max")) + summaries.append(metrics) + group_data.append((group, metrics)) + level = PAYLOAD["filters"].get("level") + offset = int(PAYLOAD.get("offset") or 0) + page = offset // int(PAYLOAD["pageSize"]) + 1 + details = [] + detail_columns = [] + total = 0 + if level in ("P0", "P2"): + for group, metrics in group_data: + group_id = group.get("id") + endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" + data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + total += total_of(data) + availability = availability_by_account(token, group_id) if level == "P2" else {} + for item in items_of(data): + absorbed = None + if level == "P2" and item.get("request_id"): + correlated = get(token, "/api/v1/admin/ops/request-errors", {"time_range": "24h", "request_id": item.get("request_id"), "page": 1, "page_size": 1, "view": "all"}, "correlated request error") + absorbed = total_of(correlated) == 0 + details.append(detail_row(item, metrics, availability, level, absorbed)) + detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"] + elif level == "P1": + total = len(group_data) * 2 + endpoint_names = ["/v1/responses", "/v1/responses/compact"] + for group, metrics in group_data: + for endpoint in endpoint_names: + if PAYLOAD["filters"].get("endpoint") and PAYLOAD["filters"].get("endpoint") != endpoint: + continue + details.append({ + "GROUP": f"{metrics.get('groupName')} ({metrics.get('groupId')})", + "ENDPOINT": endpoint, + "SAMPLES": "unavailable", + "P50": "unavailable", + "P95": "unavailable", + "P99": "unavailable", + "MAX": "unavailable", + "OUTLIER_REQUEST_ID": "unavailable", + "BOUNDARY": "native endpoint/request TTFT unavailable; total duration not substituted", + }) + detail_columns = ["GROUP", "ENDPOINT", "SAMPLES", "P50", "P95", "P99", "MAX", "OUTLIER_REQUEST_ID", "BOUNDARY"] + return { + "ok": True, + "level": level, + "window": {"timeRange": "24h"}, + "thresholds": thresholds, + "summary": summaries, + "details": details[:PAYLOAD["pageSize"]], + "detailColumns": detail_columns, + "pagination": {"offset": offset, "total": total, "hasMore": offset + PAYLOAD["pageSize"] < total}, + "boundary": {"notes": [ + "P0/P2 facts come from native admin ops request-errors, upstream-errors, overview, and account-availability.", + "P0/P1/P2 labels and failover/retry absorption are UniDesk CLI projections; Sub2API native severity is not rewritten.", + "P1 endpoint-level samples and request-level TTFT are unavailable in the observed native Ops response; total duration is never used as TTFT.", + "Absorption summary is a bounded count projection from native upstream/customer totals; P2 detail absorption is correlated by stable request ID.", + ]}, + "next": "Use --level P0|P1|P2 for details; reuse filters with --page-token for the next fixed page; use codex-pool trace --request-id for trace disclosure.", + "valuesPrinted": False, + } + +try: + output = execute() +except Exception as exc: + output = {"ok": False, "error": str(exc), "valuesPrinted": False} +print(json.dumps(output, ensure_ascii=False, indent=2)) +sys.exit(0 if output.get("ok") else 1) +PY +`; +} diff --git a/scripts/src/platform-infra-sub2api-codex/options.ts b/scripts/src/platform-infra-sub2api-codex/options.ts index a17ddade..64ce9e00 100644 --- a/scripts/src/platform-infra-sub2api-codex/options.ts +++ b/scripts/src/platform-infra-sub2api-codex/options.ts @@ -25,6 +25,7 @@ import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelP import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions"; import { renderCodexPoolPlan } from "./render"; import { codexPoolRuntime } from "./runtime"; +import { codexPoolFaults } from "./faults"; import { defaultCodexPoolRuntimeTargetId } from "./runtime-target"; import { codexPoolHelp } from "./types"; @@ -38,6 +39,7 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]) if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1))); if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1))); if (action === "runtime") return await codexPoolRuntime(config, args.slice(1)); + if (action === "faults") return await codexPoolFaults(config, args.slice(1)); if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1))); if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1))); if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1))); diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 347d4685..5c54219c 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -341,7 +341,7 @@ export function codexPoolHelp(): unknown { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(); return { - command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", + command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|faults|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", output: "json, except trace and sentinel-report default to low-noise text tables", usage: [ "bun scripts/cli.ts platform-infra sub2api codex-pool plan", @@ -351,6 +351,7 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account ] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool faults [--level P0|P1|P2] [--group ] [--account ] [--model ] [--stream sync|stream] [--endpoint ] [--request-id ] [--page-token ] [--target PK01] [--json]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account --template [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account --kind temp-unschedulable [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", From 8a944d886463123a85bb01a6039b44fa0517b747 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:42:02 +0200 Subject: [PATCH 04/12] feat: expose Sub2API monitor record diagnostics --- .../platform-infra-sub2api-ops/projection.ts | 100 ++++++++++-- .../src/platform-infra-sub2api-ops/query.ts | 143 +++++++++++++++++- .../src/platform-infra-sub2api-ops/render.ts | 23 +++ 3 files changed, 256 insertions(+), 10 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-ops/projection.ts b/scripts/src/platform-infra-sub2api-ops/projection.ts index c1c22fed..606027c3 100644 --- a/scripts/src/platform-infra-sub2api-ops/projection.ts +++ b/scripts/src/platform-infra-sub2api-ops/projection.ts @@ -4,6 +4,8 @@ const upstreamVersion = "v0.1.155"; const diagnosisRuleSource = "frontend/src/views/admin/ops/components/OpsDashboardHeader.vue"; const diagnosisTextSource = "frontend/src/i18n/locales/zh/admin/ops.ts"; const channelHistoryPageSize = 20; +const monitorResponseHeaderTimeoutMs = 30_000; +const monitorTotalRequestTimeoutMs = 45_000; export function projectSub2ApiOps(raw: Record, options: Sub2ApiOpsOptions): Record { const target = record(raw.target); @@ -25,6 +27,7 @@ export function projectSub2ApiOps(raw: Record, options: Sub2Api } function projectDiagnosis(target: Record, data: Record, options: Sub2ApiOpsOptions): Record { + const runtimeVersion = record(data.runtimeVersion); const groups = arrayRecords(data.groups).map((entry) => { const group = record(entry.group); const overview = record(entry.overview); @@ -53,7 +56,7 @@ function projectDiagnosis(target: Record, data: Record): Record, data: Record, options: Sub2ApiOpsOptions): Record { + const runtimeVersion = record(data.runtimeVersion); const channels = arrayRecords(data.channels).map((entry) => projectChannel(entry, options.window)); const operational = channels.every((channel) => channel.status === "OPERATIONAL"); const history = arrayRecords(data.history) @@ -157,14 +161,15 @@ function projectChannels(target: Record, data: Record String(left.checkedAt).localeCompare(String(right.checkedAt))); - const historyProjection = options.channel === null ? null : paginateHistory(history, options); + const historyProjection = options.channel === null ? null : paginateHistory(history, options, record(data.correlation), runtimeVersion); const historyError = historyProjection === null ? null : historyProjection.error; return { ok: historyError === null, action: "platform-infra-sub2api-ops-channels", target, query: querySummary(options), - source: sourceSummary("channels", "available"), + source: sourceSummary("channels", "available", runtimeVersion), + timeoutPolicy: monitorTimeoutPolicy(runtimeVersion), overallStatus: operational ? "OPERATIONAL" : "DEGRADED", channels, history: historyProjection, @@ -173,9 +178,10 @@ function projectChannels(target: Record, data: Record[], options: Sub2ApiOpsOptions): Record { +function paginateHistory(history: Record[], options: Sub2ApiOpsOptions, correlation: Record, runtimeVersion: Record): Record { if (options.recordId !== null) { - const selectedRecord = history.find((item) => String(item.id) === options.recordId) ?? null; + const selectedRecordRaw = history.find((item) => String(item.id) === options.recordId) ?? null; + const selectedRecord = selectedRecordRaw === null ? null : projectSelectedRecord(selectedRecordRaw, correlation, runtimeVersion); return { order: "PAST_TO_NOW", pageSize: channelHistoryPageSize, @@ -221,6 +227,53 @@ function paginateHistory(history: Record[], options: Sub2ApiOps }; } +function projectSelectedRecord(item: Record, correlation: Record, runtimeVersion: Record): Record { + const latencyMs = typeof item.latencyMs === "number" ? item.latencyMs : null; + const selected = record(correlation.selected); + const final = record(selected.final); + const failovers = arrayRecords(selected.failovers); + const selectFailures = arrayRecords(selected.selectFailures); + const totalLatencyMs = typeof final.latencyMs === "number" ? final.latencyMs : null; + const runtimeTag = typeof runtimeVersion.version === "string" && runtimeVersion.version.length > 0 ? runtimeVersion.version : null; + const runtimeVersionMismatch = runtimeTag?.replace(/^v/u, "") !== upstreamVersion.replace(/^v/u, ""); + return { + ...item, + timeout: { + observedKind: String(item.message ?? "").toLowerCase().includes("timeout awaiting response headers") ? "response-header" : "unknown", + responseHeaderTimeoutMs: monitorResponseHeaderTimeoutMs, + totalRequestTimeoutMs: monitorTotalRequestTimeoutMs, + sourceReferenceVersion: upstreamVersion, + runtimeVersion: runtimeTag, + runtimeVersionMismatch, + runtimePolicyStatus: runtimeVersionMismatch ? "unverified" : "source-version-verified", + remainingAfterResponseHeaderMs: monitorTotalRequestTimeoutMs - monitorResponseHeaderTimeoutMs, + observedLatencyDeltaMs: latencyMs === null ? null : latencyMs - monitorResponseHeaderTimeoutMs, + perUpstreamTimeout: { status: "unsupported", reason: "native monitor history does not expose per-upstream timeout" }, + overallGatewayDeadline: { status: "unsupported", reason: "native monitor history does not expose the gateway request deadline" }, + failoverReserve: { status: "unsupported", reason: "native monitor history does not expose a dedicated failover reserve" }, + }, + correlation: { + status: correlation.status ?? "unsupported", + reason: correlation.reason ?? null, + requestId: selected.requestId ?? null, + match: selected.match ?? null, + apiKeyName: selected.apiKeyName ?? null, + model: selected.model ?? null, + accounts: selected.accounts ?? [], + upstreamErrors: selected.upstreamErrors ?? [], + failovers, + selectFailures, + final: Object.keys(final).length === 0 ? null : final, + switchingObserved: failovers.length > 0, + selectionFailureObserved: selectFailures.length > 0, + sufficientForObservedCompletion: runtimeVersionMismatch || totalLatencyMs === null ? null : monitorTotalRequestTimeoutMs >= totalLatencyMs, + sufficiencyStatus: runtimeVersionMismatch ? "unverified-runtime-version-mismatch" : totalLatencyMs === null ? "unsupported-no-final-latency" : "evaluated-against-verified-source-version", + traceNext: selected.requestId ? `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ${String(selected.requestId)}` : null, + candidates: correlation.status === "ambiguous" ? correlation.candidates ?? [] : [], + }, + }; +} + function projectChannel(entry: Record, window: "7d" | "15d" | "30d"): Record { const summary = record(entry.summary); const detail = record(entry.detail); @@ -261,7 +314,33 @@ function projectChannel(entry: Record, window: "7d" | "15d" | " collectedAt: timeline.length === 0 ? null : timeline[timeline.length - 1]?.checkedAt ?? null, refreshRemainingSeconds: null, refreshSourceStatus: "unsupported-by-native-response", - timeline: { order: "PAST_TO_NOW", records: timeline }, + timeline: { + order: "PAST_TO_NOW", + recordCount: timeline.length, + firstAt: timeline[0]?.checkedAt ?? null, + lastAt: timeline[timeline.length - 1]?.checkedAt ?? null, + records: [], + detail: "use --channel for native history pagination", + }, + }; +} + +function monitorTimeoutPolicy(runtimeVersion: Record): Record { + const runtimeTag = typeof runtimeVersion.version === "string" && runtimeVersion.version.length > 0 ? runtimeVersion.version : null; + const runtimeVersionMismatch = runtimeTag?.replace(/^v/u, "") !== upstreamVersion.replace(/^v/u, ""); + return { + runtime: { image: runtimeVersion.image ?? null, version: runtimeTag }, + sourceReferenceVersion: upstreamVersion, + runtimeVersionMismatch, + runtimePolicyStatus: runtimeVersionMismatch ? "unverified" : "source-version-verified", + sourceReference: { + responseHeaderTimeoutMs: monitorResponseHeaderTimeoutMs, + totalRequestTimeoutMs: monitorTotalRequestTimeoutMs, + sourceStatus: "official-source-constant-verified-for-reference-version", + }, + perUpstreamTimeout: { status: "unsupported", reason: "native monitor API does not expose this field" }, + overallGatewayDeadline: { status: "unsupported", reason: "native monitor API does not expose this field" }, + failoverReserve: { status: "unsupported", reason: "native monitor API does not expose this field" }, }; } @@ -269,11 +348,12 @@ function availabilityFor(model: Record, window: "7d" | "15d" | return model[`availability_${window}`] ?? model.availability ?? null; } -function sourceSummary(action: "diagnosis" | "channels", status: string): Record { +function sourceSummary(action: "diagnosis" | "channels", status: string, runtimeVersion: Record = {}): Record { if (action === "diagnosis") { return { status, - upstreamVersion, + sourceReferenceVersion: upstreamVersion, + runtimeVersion, metricsEndpoint: "/api/v1/admin/ops/dashboard/overview", nativeDiagnosisEndpoint: { status: "unsupported", reason: "v0.1.155 has no diagnosis/advice response schema" }, projectionKind: "native-frontend-projection", @@ -284,10 +364,12 @@ function sourceSummary(action: "diagnosis" | "channels", status: string): Record } return { status, - upstreamVersion, + sourceReferenceVersion: upstreamVersion, + runtimeVersion, listEndpoint: "/api/v1/channel-monitors", detailEndpoint: "/api/v1/channel-monitors/:id/status", historyEndpoint: "/api/v1/admin/channel-monitors/:id/history", + correlationSource: "heuristic projection by native record checked_at, latency_ms and model; request/failover facts come from request rows and bounded runtime logs when available", windows: ["7d", "15d", "30d"], overallStatusKind: "native-frontend-projection", refreshRemaining: { status: "unsupported", reason: "native response does not include a refresh countdown" }, diff --git a/scripts/src/platform-infra-sub2api-ops/query.ts b/scripts/src/platform-infra-sub2api-ops/query.ts index a71e12ef..e5ea7017 100644 --- a/scripts/src/platform-infra-sub2api-ops/query.ts +++ b/scripts/src/platform-infra-sub2api-ops/query.ts @@ -51,6 +51,7 @@ import base64 import json import subprocess import sys +from datetime import datetime from urllib.parse import urlencode TARGET = ${pyJson(target)} @@ -59,6 +60,18 @@ OPTIONS = ${pyJson(options)} def run(command, input_bytes=None): return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +def docker(args): + proc = run(["docker", *args]) + if proc.returncode == 0: + return proc + sudo_proc = run(["sudo", "-n", "docker", *args]) + return sudo_proc if sudo_proc.returncode == 0 else proc + +def runtime_logs(since="24h", tail=20000): + if TARGET["runtimeMode"] == "host-docker": + return docker(["logs", f"--since={since}", f"--tail={tail}", "sub2api-app"]) + return run(["kubectl", "-n", TARGET["namespace"], "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) + def tail_text(value, limit=800): if isinstance(value, bytes): value = value.decode("utf-8", errors="replace") @@ -103,6 +116,17 @@ def select_app_pod(): APP_POD = select_app_pod() +def runtime_version(): + if TARGET["runtimeMode"] == "host-docker": + proc = docker(["inspect", "--format", "{{.Config.Image}}", "sub2api-app"]) + image = proc.stdout.decode("utf-8", errors="replace").strip() if proc.returncode == 0 else "" + else: + pod = kube_json(["-n", TARGET["namespace"], "get", "pod", APP_POD], "sub2api pod") + containers = ((pod.get("spec") or {}).get("containers") or []) + image = containers[0].get("image") if containers else "" + tag = image.rsplit(":", 1)[1] if isinstance(image, str) and ":" in image else None + return {"image": image or None, "version": tag} + def config_value(key): if TARGET["runtimeMode"] == "host-docker": return read_host_env().get(key) @@ -199,6 +223,116 @@ def find_named(items, selector, label): raise RuntimeError(f"unknown {label} {selector}; available={available}") return matches +def log_item(line): + json_start = line.find("{") + if json_start < 0: + return None + try: + item = json.loads(line[json_start:]) + except Exception: + return None + if not isinstance(item, dict): + return None + prefix = line[:json_start].strip().split() + item["_at"] = prefix[0] if prefix else None + item["_message"] = " ".join(prefix[3:]) if len(prefix) >= 4 else " ".join(prefix[2:]) + return item + +def epoch_of(value): + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except Exception: + return None + +def account_names(token, platform): + params = {"page": 1, "page_size": 500} + if platform: + params["platform"] = platform + rows = items_of(get(token, "/api/v1/admin/accounts", params, "list accounts for monitor correlation")) + return {item.get("id"): item.get("name") for item in rows if isinstance(item, dict) and item.get("id") is not None} + +def correlate_record(token, record, monitor): + checked_at = record.get("checked_at") + checked_epoch = epoch_of(checked_at) + latency_ms = record.get("latency_ms") + if checked_epoch is None or not isinstance(latency_ms, (int, float)): + return {"status": "unsupported", "reason": "native record has no usable checked_at/latency_ms", "candidates": []} + expected_start = checked_epoch - latency_ms / 1000.0 + record_model = str(record.get("model") or "") + request_rows = items_of(get(token, "/api/v1/admin/ops/requests", {"time_range": "24h", "page": 1, "page_size": 500, "kind": "all"}, "list request correlation candidates")) + request_candidates = [] + for item in request_rows: + request_id = item.get("request_id") + created_epoch = epoch_of(item.get("created_at")) + model_match = not record_model or str(item.get("model") or "") == record_model + if not isinstance(request_id, str) or not request_id or created_epoch is None or not model_match: + continue + start_delta_ms = round(abs(created_epoch - expected_start) * 1000) + if start_delta_ms <= 15000: + request_candidates.append((request_id, item, start_delta_ms)) + proc = runtime_logs() + names = account_names(token, monitor.get("provider")) + grouped = {} + candidate_ids = {item[0] for item in request_candidates} + for line in proc.stdout.decode("utf-8", errors="replace").splitlines() if proc.returncode == 0 else []: + item = log_item(line) + if item is None: + continue + request_id = item.get("request_id") + at_epoch = epoch_of(item.get("_at")) + if not isinstance(request_id, str) or not request_id: + continue + if request_id not in candidate_ids and (at_epoch is None or at_epoch < expected_start - 15 or at_epoch > checked_epoch + 90): + continue + grouped.setdefault(request_id, []).append(item) + candidates = [] + rows_by_id = {item[0]: (item[1], item[2]) for item in request_candidates} + all_request_ids = set(grouped.keys()) | set(rows_by_id.keys()) + for request_id in all_request_ids: + events = grouped.get(request_id, []) + events.sort(key=lambda item: epoch_of(item.get("_at")) or 0) + request_row, row_delta_ms = rows_by_id.get(request_id, ({}, None)) + first = events[0] if events else None + models = [str(item.get("model")) for item in events if item.get("model")] + first_epoch = epoch_of(first.get("_at")) if first is not None else epoch_of(request_row.get("created_at")) + start_delta_ms = row_delta_ms if row_delta_ms is not None else round(abs((first_epoch or expected_start) - expected_start) * 1000) + model_match = not record_model or record_model in models or str(request_row.get("model") or "") == record_model + if start_delta_ms > 15000 or not model_match: + continue + failovers = [item for item in events if "upstream_failover_switching" in str(item.get("_message") or "")] + select_failures = [item for item in events if "account_select_failed" in str(item.get("_message") or "")] + upstream_errors = [item for item in events if "account_upstream_error" in str(item.get("_message") or "")] + finals = [item for item in events if "http request completed" in str(item.get("_message") or "")] + final = finals[-1] if finals else None + account_ids = [] + for item in events: + account_id = item.get("account_id") + if isinstance(account_id, str) and account_id.isdigit(): + account_id = int(account_id) + if isinstance(account_id, int) and account_id not in account_ids: + account_ids.append(account_id) + candidates.append({ + "requestId": request_id, + "match": {"kind": "native-request-start-window", "startDeltaMs": start_delta_ms, "modelMatched": model_match}, + "firstAt": first.get("_at") if first is not None else request_row.get("created_at"), + "lastAt": events[-1].get("_at") if events else request_row.get("created_at"), + "model": next((item for item in models if item), request_row.get("model")), + "apiKeyName": next((item.get("api_key_name") for item in events if item.get("api_key_name")), None), + "accounts": [{"id": account_id, "name": names.get(account_id)} for account_id in account_ids] or ([{"id": request_row.get("account_id"), "name": request_row.get("account_name") or names.get(request_row.get("account_id"))}] if request_row.get("account_id") is not None else []), + "upstreamErrors": [{"at": item.get("_at"), "accountId": item.get("account_id"), "error": item.get("error")} for item in upstream_errors], + "failovers": [{"at": item.get("_at"), "accountId": item.get("account_id"), "upstreamStatus": item.get("upstream_status"), "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches")} for item in failovers], + "selectFailures": [{"at": item.get("_at"), "error": item.get("error"), "excludedAccountCount": item.get("excluded_account_count")} for item in select_failures], + "final": {"at": request_row.get("created_at"), "statusCode": request_row.get("status_code") or request_row.get("upstream_status_code"), "accountId": request_row.get("account_id"), "latencyMs": request_row.get("duration_ms"), "path": request_row.get("path")} if final is None else {"at": final.get("_at"), "statusCode": final.get("status_code"), "accountId": final.get("account_id"), "latencyMs": final.get("latency_ms"), "path": final.get("path")}, + }) + candidates.sort(key=lambda item: item["match"]["startDeltaMs"]) + if not candidates: + return {"status": "unsupported", "reason": "no unique native request_id is available in the record; bounded log correlation found no candidate", "candidates": []} + best_delta = candidates[0]["match"]["startDeltaMs"] + best = [item for item in candidates if item["match"]["startDeltaMs"] == best_delta] + return {"status": "inferred" if len(best) == 1 else "ambiguous", "reason": "heuristic correlation by record start window and model; the native monitor record has no request_id" if len(best) == 1 else "multiple requests have the same nearest start time", "selected": best[0] if len(best) == 1 else None, "candidates": candidates[:5]} + def diagnosis(token): group_params = {"platform": OPTIONS["platform"]} if OPTIONS.get("platform") else None groups = items_of(get(token, "/api/v1/admin/groups/all", group_params, "list groups")) @@ -247,17 +381,24 @@ def channels(token): detail = get(token, f"/api/v1/channel-monitors/{monitor['id']}/status", None, "channel monitor status") output.append({"summary": monitor, "detail": detail}) history = None + correlation = None if OPTIONS.get("channel") and output: monitor_id = output[0]["summary"]["id"] params = {"limit": 1000} if OPTIONS.get("model"): params["model"] = OPTIONS["model"] history = items_of(get(token, f"/api/v1/admin/channel-monitors/{monitor_id}/history", params, "channel monitor history")) - return {"channels": output, "history": history} + record_id = OPTIONS.get("recordId") + if record_id: + selected = next((item for item in history if str(item.get("id")) == str(record_id)), None) + if selected is not None: + correlation = correlate_record(token, selected, output[0]["summary"]) + return {"channels": output, "history": history, "correlation": correlation} try: token = login() data = diagnosis(token) if OPTIONS["action"] == "diagnosis" else channels(token) + data["runtimeVersion"] = runtime_version() print(json.dumps({"ok": True, "sourceStatus": "available", "data": data}, ensure_ascii=False, separators=(",", ":"))) except Exception as error: print(json.dumps({"ok": False, "sourceStatus": "unavailable", "error": str(error), "data": None}, ensure_ascii=False, separators=(",", ":"))) diff --git a/scripts/src/platform-infra-sub2api-ops/render.ts b/scripts/src/platform-infra-sub2api-ops/render.ts index bf583153..c2d8fed9 100644 --- a/scripts/src/platform-infra-sub2api-ops/render.ts +++ b/scripts/src/platform-infra-sub2api-ops/render.ts @@ -125,6 +125,12 @@ function renderChannels(result: Record): string[] { } const selectedRecord = record(history.selectedRecord); if (Object.keys(selectedRecord).length > 0) { + const timeout = record(selectedRecord.timeout); + const correlation = record(selectedRecord.correlation); + const final = record(correlation.final); + const accounts = arrayRecords(correlation.accounts); + const failovers = arrayRecords(correlation.failovers); + const selectFailures = arrayRecords(correlation.selectFailures); lines.push( "", "RECORD", @@ -136,8 +142,25 @@ function renderChannels(result: Record): string[] { ["pingLatencyMs", stringValue(selectedRecord.pingLatencyMs)], ["checkedAt", stringValue(selectedRecord.checkedAt)], ["message", stringValue(selectedRecord.message)], + ["runtimeVersion", stringValue(timeout.runtimeVersion)], + ["sourceReferenceVersion", stringValue(timeout.sourceReferenceVersion)], + ["runtimePolicyStatus", stringValue(timeout.runtimePolicyStatus)], + ["sourceResponseHeaderMs", stringValue(timeout.responseHeaderTimeoutMs)], + ["sourceTotalRequestMs", stringValue(timeout.totalRequestTimeoutMs)], + ["remainingAfterHeaderMs", stringValue(timeout.remainingAfterResponseHeaderMs)], + ["correlationStatus", stringValue(correlation.status)], + ["requestId", stringValue(correlation.requestId)], + ["accounts", accounts.map((item) => `${stringValue(item.id)}:${stringValue(item.name)}`).join(",") || "-"], + ["failoverCount", String(failovers.length)], + ["selectFailureCount", String(selectFailures.length)], + ["finalStatus", stringValue(final.statusCode)], + ["finalLatencyMs", stringValue(final.latencyMs)], + ["timeoutSufficient", stringValue(correlation.sufficientForObservedCompletion)], + ["sufficiencyStatus", stringValue(correlation.sufficiencyStatus)], ]), ); + if (correlation.traceNext) lines.push(`Trace: ${stringValue(correlation.traceNext)}`); + if (correlation.reason) lines.push(`Correlation: ${stringValue(correlation.reason)}`); } lines.push( "", From 0affaa4827c761a9987e4c081c87bd78d86615d2 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:42:05 +0200 Subject: [PATCH 05/12] =?UTF-8?q?fix:=20=E5=85=BC=E5=AE=B9=20Sub2API=200.1?= =?UTF-8?q?.147=20=E6=95=85=E9=9A=9C=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform-infra-sub2api-codex/faults.ts | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts index 8f89e410..61e276bd 100644 --- a/scripts/src/platform-infra-sub2api-codex/faults.ts +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -397,6 +397,19 @@ def items_of(data): def total_of(data): return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data)) +def all_items(token, path, params, label): + page = 1 + page_size = 500 + rows = [] + while True: + data = get(token, path, {**params, "page": page, "page_size": page_size}, label) + batch = items_of(data) + rows.extend(batch) + total = total_of(data) + if not batch or len(rows) >= total: + return rows + page += 1 + def percent(value): if not isinstance(value, (int, float)): return None @@ -417,8 +430,6 @@ def filters_for(group_id=None): result["stream"] = "true" if filters["stream"] == "stream" else "false" if filters.get("endpoint"): result["inbound_endpoint"] = filters["endpoint"] - if filters.get("requestId"): - result["request_id"] = filters["requestId"] return result def selected_groups(groups): @@ -510,14 +521,30 @@ def execute(): for group, metrics in group_data: group_id = group.get("id") endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" - data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") - total += total_of(data) + request_id = PAYLOAD["filters"].get("requestId") + if request_id: + rows = [ + item for item in all_items(token, endpoint, {**filters_for(group_id), "view": "all", "include_detail": 1}, level + " request lookup") + if item.get("request_id") == request_id or item.get("client_request_id") == request_id + ] + total += len(rows) + rows = rows[offset:offset + PAYLOAD["pageSize"]] + else: + data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + total += total_of(data) + rows = items_of(data) availability = availability_by_account(token, group_id) if level == "P2" else {} - for item in items_of(data): + customer_request_ids = set() + if level == "P2": + customer_request_ids = { + item.get("request_id") or item.get("client_request_id") + for item in all_items(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "view": "all"}, "customer error correlation") + if item.get("request_id") or item.get("client_request_id") + } + for item in rows: absorbed = None if level == "P2" and item.get("request_id"): - correlated = get(token, "/api/v1/admin/ops/request-errors", {"time_range": "24h", "request_id": item.get("request_id"), "page": 1, "page_size": 1, "view": "all"}, "correlated request error") - absorbed = total_of(correlated) == 0 + absorbed = item.get("request_id") not in customer_request_ids details.append(detail_row(item, metrics, availability, level, absorbed)) detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"] elif level == "P1": From cfba88f0c0677041fca0b968aa855b98237aa766 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:44:34 +0200 Subject: [PATCH 06/12] =?UTF-8?q?fix:=20=E7=B2=BE=E7=A1=AE=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=20Sub2API=200.1.147=20=E6=95=85=E9=9A=9C=E8=AE=B0?= =?UTF-8?q?=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform-infra-sub2api-codex/faults.ts | 72 ++++++++++++------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts index 61e276bd..57e72890 100644 --- a/scripts/src/platform-infra-sub2api-codex/faults.ts +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -218,11 +218,11 @@ function renderFaults(response: Record): RenderedCliResult { const summary = arrayOfRecords(faults.summary); if (summary.length > 0) { lines.push("GROUPS"); - lines.push(table(["GROUP", "P0", "CUSTOMER_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERR%", "ABSORB%"], summary.map((row) => [ + lines.push(table(["GROUP", "P0", "CUSTOMER_ERRS", "GROUP_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERRS", "GROUP_UP_ERR%", "ABSORB%"], summary.map((row) => [ `${text(row.groupName)} (${text(row.groupId)})`, - text(row.p0), percent(row.customerErrorRatePercent), + text(row.p0), text(row.customerErrorCount), percent(row.customerErrorRatePercent), text(row.p1), milliseconds(row.ttftP99Ms), - text(row.p2), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), + text(row.p2), text(row.upstreamErrorCount), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), ]))); } const details = arrayOfRecords(faults.details); @@ -418,19 +418,43 @@ def percent(value): def selector_id(value): return int(value) if isinstance(value, str) and value.isdigit() else None -def filters_for(group_id=None): - filters = PAYLOAD["filters"] +def scope_filters(group_id=None): result = {"time_range": "24h", "platform": "openai", "group_id": group_id} + return result + +def matches_detail_filters(item): + filters = PAYLOAD["filters"] account = filters.get("account") if account: - result["account_id" if selector_id(account) is not None else "account_name"] = selector_id(account) if selector_id(account) is not None else account - if filters.get("model"): - result["model"] = filters["model"] + account_match = str(item.get("account_id") or "") == str(account) or str(item.get("account_name") or "") == str(account) + if not account_match: + return False + if filters.get("model") and str(item.get("requested_model") or item.get("model") or "") != str(filters["model"]): + return False if filters.get("stream"): - result["stream"] = "true" if filters["stream"] == "stream" else "false" - if filters.get("endpoint"): - result["inbound_endpoint"] = filters["endpoint"] - return result + expected_stream = filters["stream"] == "stream" + if item.get("stream") is not expected_stream: + return False + if filters.get("endpoint") and str(item.get("inbound_endpoint") or item.get("request_path") or item.get("path") or "") != str(filters["endpoint"]): + return False + if filters.get("requestId"): + request_id = filters["requestId"] + if item.get("request_id") != request_id and item.get("client_request_id") != request_id: + return False + return True + +def has_detail_filters(): + filters = PAYLOAD["filters"] + return any(filters.get(key) for key in ("account", "model", "stream", "endpoint", "requestId")) + +DETAIL_ROWS = {} + +def error_rows(token, path, group_id, apply_filters): + cache_key = (path, group_id, apply_filters) + if cache_key not in DETAIL_ROWS: + rows = all_items(token, path, {**scope_filters(group_id), "view": "all", "include_detail": 1}, "fault detail scan") + DETAIL_ROWS[cache_key] = [item for item in rows if matches_detail_filters(item)] if apply_filters else rows + return DETAIL_ROWS[cache_key] def selected_groups(groups): selector = PAYLOAD["filters"].get("group") @@ -486,10 +510,14 @@ def execute(): for group in groups: group_id = group.get("id") native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview") - request_errors = get(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") - upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") - customer_total = total_of(request_errors) - upstream_total = total_of(upstream_errors) + if has_detail_filters(): + customer_total = len(error_rows(token, "/api/v1/admin/ops/request-errors", group_id, True)) + upstream_total = len(error_rows(token, "/api/v1/admin/ops/upstream-errors", group_id, True)) + else: + request_errors = get(token, "/api/v1/admin/ops/request-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") + upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") + customer_total = total_of(request_errors) + upstream_total = total_of(upstream_errors) absorbed = max(0, upstream_total - customer_total) absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None) @@ -521,16 +549,12 @@ def execute(): for group, metrics in group_data: group_id = group.get("id") endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" - request_id = PAYLOAD["filters"].get("requestId") - if request_id: - rows = [ - item for item in all_items(token, endpoint, {**filters_for(group_id), "view": "all", "include_detail": 1}, level + " request lookup") - if item.get("request_id") == request_id or item.get("client_request_id") == request_id - ] + if has_detail_filters(): + rows = error_rows(token, endpoint, group_id, True) total += len(rows) rows = rows[offset:offset + PAYLOAD["pageSize"]] else: - data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + data = get(token, endpoint, {**scope_filters(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") total += total_of(data) rows = items_of(data) availability = availability_by_account(token, group_id) if level == "P2" else {} @@ -538,7 +562,7 @@ def execute(): if level == "P2": customer_request_ids = { item.get("request_id") or item.get("client_request_id") - for item in all_items(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "view": "all"}, "customer error correlation") + for item in error_rows(token, "/api/v1/admin/ops/request-errors", group_id, False) if item.get("request_id") or item.get("client_request_id") } for item in rows: From 0cfbdea0314091ea1c1b402121f06144980c3260 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:47:33 +0200 Subject: [PATCH 07/12] =?UTF-8?q?fix:=20=E5=AE=8C=E6=95=B4=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=20Sub2API=20=E4=B8=8A=E6=B8=B8=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/src/platform-infra-sub2api-codex/feedback.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-codex/feedback.ts b/scripts/src/platform-infra-sub2api-codex/feedback.ts index d2e9f6e3..57ddea77 100644 --- a/scripts/src/platform-infra-sub2api-codex/feedback.ts +++ b/scripts/src/platform-infra-sub2api-codex/feedback.ts @@ -541,7 +541,7 @@ function renderFeedbackReport(report: Record | null, options: F ["#", "AT", "GAP_S", "STATUS", "DURATION", "MODEL", "ACCOUNT", "ATTRIBUTION"], ...timeline.map((item, index) => [ String(index + 1), shortIso(item.at), textValue(item.gapSeconds), textValue(item.statusCode ?? item.kind), textValue(item.durationMs), - shorten(textValue(item.model), 18), shorten(`${textValue(item.accountName)}#${textValue(item.accountId)}`, 28), textValue(item.attribution), + shorten(textValue(item.model), 18), `${textValue(item.accountName)}#${textValue(item.accountId)}`, textValue(item.attribution), ]), ])); lines.push(""); @@ -557,7 +557,7 @@ function renderFeedbackReport(report: Record | null, options: F lines.push(renderTable([ ["ACCOUNT", "STATUS", "SCHED", "IN_USE", "QUEUE", "CAP", "AVAILABLE", "PROXY", "P_STATUS"], ...accounts.map((item) => [ - shorten(`${textValue(item.name)}#${textValue(item.id)}`, 32), textValue(item.status), textValue(item.schedulable), + `${textValue(item.name)}#${textValue(item.id)}`, textValue(item.status), textValue(item.schedulable), textValue(item.currentInUse), textValue(item.waitingInQueue), textValue(item.maxCapacity), textValue(item.available), shorten(`${textValue(item.proxyName)}#${textValue(item.proxyId)}`, 24), textValue(item.proxyStatus), ]), From 020c208920ed89806dde130c02edb087b0b23e54 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:52:20 +0200 Subject: [PATCH 08/12] =?UTF-8?q?fix:=20=E4=BF=9D=E6=8C=81=20Sub2API=20?= =?UTF-8?q?=E4=B8=83=E6=9D=A1=E6=95=85=E9=9A=9C=E5=86=B7=E5=8D=B4=E8=A7=84?= =?UTF-8?q?=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/platform-infra/sub2api-codex-pool.yaml | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 31a138f0..d1c1f343 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -12,22 +12,38 @@ runtime: templates: - id: codex-upstream-failover kind: temp-unschedulable - description: Reusable Sub2API request-path cooling and failover policy for Codex-compatible API-key accounts. + description: Codex 兼容 API-key 账号的请求路径短时冷却与换号策略。 spec: enabled: true rules: + - statusCode: 429 + keywords: [concurrency limit exceeded, upstream rate limit exceeded] + durationMinutes: 1 + description: 明确命中上游并发或速率限制时短时冷却当前账号。 + - statusCode: 400 + keywords: [input must be a list] + durationMinutes: 1 + description: 明确命中 Responses input 列表兼容错误时短时冷却并切换账号。 + - statusCode: 500 + keywords: [failed to validate api key, do request failed] + durationMinutes: 1 + description: 上游 API-key 查询依赖或请求传输内部失败时短时冷却当前账号。 - statusCode: 502 - keywords: [upstream service temporarily unavailable, upstream request failed, context window, stream usage incomplete, missing terminal event] + keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 502 upstream availability, context-window, and incomplete-stream failures briefly cool down the selected account. + description: 明确命中 502 上游不可用、过载或并发限制时短时冷却当前账号。 - statusCode: 503 - keywords: [upstream service temporarily unavailable, upstream request failed, temporarily unavailable] + keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 503 temporary upstream availability failures briefly cool down the selected account. + description: 明确命中 503 上游暂时不可用、过载或并发限制时短时冷却当前账号。 + - statusCode: 504 + keywords: [concurrency limit exceeded, upstream service temporarily unavailable, input must be a list] + durationMinutes: 1 + description: 明确命中 504 上游并发限制、暂时不可用或 Responses input 列表兼容错误时短时冷却当前账号。 - statusCode: 524 - keywords: [upstream request failed, missing_required_parameter, encrypted_content] + keywords: [upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 524 upstream request and encrypted-content failures briefly cool down the selected account. + description: 明确命中 524 上游不可用、过载或并发限制时短时冷却当前账号。 pool: groupName: unidesk-codex-pool From 80379a20122d121fa7f167fdaa7fa21e42dc9e0c Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:55:30 +0200 Subject: [PATCH 09/12] perf: reuse selfmedia build environment --- config/selfmedia.yaml | 10 +++ scripts/native/cicd/pac-status-evaluator.cjs | 13 +++- ...platform-infra-pipelines-as-code-remote.sh | 10 ++- .../src/platform-infra-pipelines-as-code.ts | 10 +++ scripts/src/selfmedia-config.ts | 19 ++++++ scripts/src/selfmedia-delivery-renderer.ts | 63 ++++++++++++++++++- 6 files changed, 118 insertions(+), 7 deletions(-) diff --git a/config/selfmedia.yaml b/config/selfmedia.yaml index ec4d9586..3529f125 100644 --- a/config/selfmedia.yaml +++ b/config/selfmedia.yaml @@ -34,6 +34,16 @@ delivery: dockerfile: deploy/Dockerfile imageRepository: 127.0.0.1:5000/selfmedia/newsroom networkMode: host + envReuse: + enabled: true + mode: buildkit-state-host-path + statePath: /var/lib/unidesk/selfmedia-ci/buildkit-state + identityFiles: + - package.json + - bun.lock + - web/package.json + - web/bun.lock + - deploy/Dockerfile proxy: http: http://127.0.0.1:10808 https: http://127.0.0.1:10808 diff --git a/scripts/native/cicd/pac-status-evaluator.cjs b/scripts/native/cicd/pac-status-evaluator.cjs index f15cf7a7..f1a0db17 100644 --- a/scripts/native/cicd/pac-status-evaluator.cjs +++ b/scripts/native/cicd/pac-status-evaluator.cjs @@ -505,6 +505,9 @@ function digestOf(item) { function extractPacArtifactEvidence(recordsInput, logText) { const records = Array.isArray(recordsInput) ? recordsInput.map(record) : []; + const envReuseRecord = [...records].reverse().find((item) => item.phase === "env-reuse") || null; + const imageBuildRecord = [...records].reverse().find((item) => item.phase === "image-build" || item.phase === "image-build-complete") || null; + const stageTimings = Object.assign({}, ...records.map((item) => record(item.stageTimings))); const sourceObservation = extractPacSourceObservation(records); const catalog = [...records].reverse().find((item) => item.status === "published" && Array.isArray(item.services) @@ -562,10 +565,13 @@ function extractPacArtifactEvidence(recordsInput, logText) { } if (image !== null) { const env = record(image.envReuse); + const reuseEnv = record(envReuseRecord?.envReuse); return { - imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status) || (stringOrNull(image.digestRef) !== null ? "built" : null), - envIdentity: stringOrNull(image.envIdentity), - envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.mode) || stringOrNull(image.envReuseStatus), + imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status) + || (imageBuildRecord !== null || stringOrNull(image.digestRef) !== null ? "built" : null), + envIdentity: stringOrNull(image.envIdentity) || stringOrNull(envReuseRecord?.envIdentity), + envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.status) || stringOrNull(image.envReuseStatus) + || stringOrNull(reuseEnv.dependencyReuse) || stringOrNull(reuseEnv.status) || stringOrNull(envReuseRecord?.envReuseStatus), nodeDepsReuse: null, buildCache: null, digest: digestOf(image) || (digests.length === 1 ? digests[0] : null), @@ -576,6 +582,7 @@ function extractPacArtifactEvidence(recordsInput, logText) { baselineSourceCommit: stringOrNull(image.baselineSourceCommit), action: stringOrNull(image.action), reason: stringOrNull(image.reason), + stageTimings, sourceObservation, valuesPrinted: false, }; diff --git a/scripts/src/platform-infra-pipelines-as-code-remote.sh b/scripts/src/platform-infra-pipelines-as-code-remote.sh index 4ac6f318..14287745 100644 --- a/scripts/src/platform-infra-pipelines-as-code-remote.sh +++ b/scripts/src/platform-infra-pipelines-as-code-remote.sh @@ -703,6 +703,12 @@ function recursiveEnvReuse(value, state) { if (typeof value.reusedServiceCount === 'number') state.serviceReusedCount = value.reusedServiceCount; if (typeof value.envReuse === 'string') state.status = value.envReuse; if (typeof value.envReuseStatus === 'string') state.status = value.envReuseStatus; + if (typeof value.envIdentity === 'string') state.identity = value.envIdentity; + if (value.stageTimings && typeof value.stageTimings === 'object' && !Array.isArray(value.stageTimings)) { + for (const [key, timing] of Object.entries(value.stageTimings)) { + if (typeof timing === 'number' && Number.isFinite(timing)) state.stageTimings[key] = timing; + } + } if (value.envReuse && typeof value.envReuse === 'object') { const env = value.envReuse; if (typeof env.mode === 'string') state.status = env.mode; @@ -742,7 +748,7 @@ function boundedFailureRecord(value) { } function envReuseForPipelineRun(namespace, name, taskRuns) { - const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null }; + const state = { status: null, identity: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, stageTimings: {}, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null }; const matching = (taskRuns.items || []).filter((item) => item.metadata?.labels?.['tekton.dev/pipelineRun'] === name); const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null); const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean); @@ -889,9 +895,11 @@ function rowFor(consumer, item, taskRuns) { classification, envReuse: { status: evidence.status, + identity: evidence.identity, buildSkippedCount: evidence.buildSkippedCount, serviceReusedCount: evidence.serviceReusedCount, buildCache: evidence.buildCache, + stageTimings: evidence.stageTimings, source: evidence.source, }, sourceObservation, diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index 8946986c..41102446 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -2318,6 +2318,7 @@ export function renderStatus(result: Record): RenderedCliResult "", "IMAGE / GITOPS", ...table(["IMAGE_STATUS", "ENV_REUSE", "ENV_ID", "DIGEST", "GITOPS"], [[stringValue(artifact.imageStatus), stringValue(artifact.envReuse), stringValue(artifact.envIdentity), short(stringValue(artifact.digest), 18), short(stringValue(artifact.gitopsCommit))]]), + ` stage-timings: ${stageTimingsText(record(artifact.stageTimings))}`, "", "ARGO", ...table(["SYNC", "HEALTH", "REVISION", "RELATION", "TARGET_REVISION"], [[ @@ -2880,6 +2881,13 @@ function envReuseText(envReuse: Record): string { return parts.length === 0 ? "-" : parts.join(","); } +function stageTimingsText(timings: Record): string { + const parts = Object.entries(timings) + .filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1])) + .map(([name, milliseconds]) => `${name}=${Math.round(milliseconds)}ms`); + return parts.length === 0 ? "-" : parts.join(","); +} + function registryText(registry: Record): string { if (Object.keys(registry).length === 0) return "-"; if (registry.status === "not-configured" || registry.applicability === "not-configured") return "N/A"; @@ -2941,6 +2949,8 @@ function renderHistoryDetail(row: Record): string[] { ["errorExitCode", stringValue(typedError.exitCode)], ["errorSummary", compactLine(stringValue(typedError.stderrSummary ?? typedError.message))], ["envReuseSource", stringValue(envReuse.source)], + ["envIdentity", short(stringValue(envReuse.identity), 24)], + ["stageTimings", stageTimingsText(record(envReuse.stageTimings))], ["deliveryMode", stringValue(sourceObservation.mode)], ["deliveryValid", stringValue(sourceObservation.valid)], ["deliverySourceMatched", stringValue(sourceObservation.sourceMatched)], diff --git a/scripts/src/selfmedia-config.ts b/scripts/src/selfmedia-config.ts index 7cbb60e5..e1d6a412 100644 --- a/scripts/src/selfmedia-config.ts +++ b/scripts/src/selfmedia-config.ts @@ -33,6 +33,12 @@ export interface SelfMediaDeliveryTarget { readonly dockerfile: string; readonly imageRepository: string; readonly networkMode: string; + readonly envReuse: { + readonly enabled: true; + readonly mode: "buildkit-state-host-path"; + readonly statePath: string; + readonly identityFiles: readonly string[]; + }; readonly proxy: Record; }; readonly gitops: Record & { @@ -144,6 +150,7 @@ function parseDeliveryTarget(id: string, value: Record): SelfMe const ci = record(value.ci, `${path}.ci`); const source = record(value.source, `${path}.source`); const build = record(value.build, `${path}.build`); + const envReuse = record(build.envReuse, `${path}.build.envReuse`); const proxy = record(build.proxy, `${path}.build.proxy`); const gitops = record(value.gitops, `${path}.gitops`); const deployment = record(value.deployment, `${path}.deployment`); @@ -161,6 +168,12 @@ function parseDeliveryTarget(id: string, value: Record): SelfMe validateAccessSecret(deployment, `${path}.deployment`); const gitopsWriteUrl = text(gitops.writeUrl, `${path}.gitops.writeUrl`); if (!gitopsWriteUrl.startsWith("http://gitea-http.")) throw new Error(`${path}.gitops.writeUrl must use internal Gitea authority`); + if (boolean(envReuse.enabled, `${path}.build.envReuse.enabled`) !== true) throw new Error(`${path}.build.envReuse.enabled must be true`); + const envReuseMode = text(envReuse.mode, `${path}.build.envReuse.mode`); + if (envReuseMode !== "buildkit-state-host-path") throw new Error(`${path}.build.envReuse.mode must be buildkit-state-host-path`); + const identityFiles = stringArray(envReuse.identityFiles, `${path}.build.envReuse.identityFiles`) + .map((item, index) => relativePath(item, `${path}.build.envReuse.identityFiles[${index}]`)); + if (identityFiles.length === 0 || new Set(identityFiles).size !== identityFiles.length) throw new Error(`${path}.build.envReuse.identityFiles must be non-empty and unique`); return { id, node: id, @@ -190,6 +203,12 @@ function parseDeliveryTarget(id: string, value: Record): SelfMe dockerfile: relativePath(build.dockerfile, `${path}.build.dockerfile`), imageRepository: text(build.imageRepository, `${path}.build.imageRepository`), networkMode: text(build.networkMode, `${path}.build.networkMode`), + envReuse: { + enabled: true, + mode: "buildkit-state-host-path", + statePath: absolutePath(envReuse.statePath, `${path}.build.envReuse.statePath`), + identityFiles, + }, proxy: structuredClone(proxy), }, gitops: { diff --git a/scripts/src/selfmedia-delivery-renderer.ts b/scripts/src/selfmedia-delivery-renderer.ts index acffbfe9..bab3d09a 100644 --- a/scripts/src/selfmedia-delivery-renderer.ts +++ b/scripts/src/selfmedia-delivery-renderer.ts @@ -98,7 +98,10 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin params: taskParams, volumes: [ { name: "workspace", emptyDir: { sizeLimit: target.ci.workspaceSize } }, - { name: "buildkit-state", emptyDir: { sizeLimit: "12Gi" } }, + { + name: "buildkit-state", + hostPath: { path: target.build.envReuse.statePath, type: "DirectoryOrCreate" }, + }, { name: "tmp", emptyDir: { sizeLimit: "4Gi" } }, { name: "gitops-token", @@ -112,6 +115,7 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin steps: [ sourceStep(target, effectiveDeploymentB64), prepareBuildkitStep(target), + envReuseStep(target), imageBuildStep(target), gitOpsPublishStep(target), ], @@ -136,6 +140,7 @@ function sourceStep(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: str ], script: `#!/bin/sh set -eu +stage_started_ms=$(date +%s%3N) source_commit="$(params.revision)" snapshot_prefix="$(params.source-snapshot-prefix)" rm -rf /workspace/source /workspace/release @@ -166,6 +171,8 @@ bun deploy/scripts/prepare-build.ts \\ --source-commit "$source_commit" \\ --source-snapshot-prefix "$snapshot_prefix" \\ --output-dir /workspace/release +stage_finished_ms=$(date +%s%3N) +printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))" `, securityContext: nonRootSecurity(), volumeMounts: [ @@ -181,12 +188,48 @@ function prepareBuildkitStep(target: SelfMediaDeliveryTarget): Record { + const identityFiles = target.build.envReuse.identityFiles.map((path) => `/workspace/source/${path}`); + const identityInputs = [...identityFiles, "/workspace/release/codex-version"] + .map(shellSingleQuote) + .join(" "); + return { + name: "env-reuse", + image: target.ci.toolImage, + imagePullPolicy: "IfNotPresent", + script: `#!/bin/sh +set -eu +stage_started_ms=$(date +%s%3N) +identity_file=/home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity +for input in ${identityInputs}; do test -f "$input"; done +env_identity="sha256:$(sha256sum ${identityInputs} | sha256sum | awk '{print $1}')" +previous_identity="" +if [ -f "$identity_file" ]; then previous_identity=$(cat "$identity_file"); fi +if [ "$previous_identity" = "$env_identity" ]; then + reuse_status=hit + previous_present=true +else + reuse_status=miss + if [ -n "$previous_identity" ]; then previous_present=true; else previous_present=false; fi +fi +printf '%s\n' "$env_identity" > /workspace/env-identity +stage_finished_ms=$(date +%s%3N) +printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentityMs":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_ms-stage_started_ms))" +`, + securityContext: nonRootSecurity(), + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + ], + }; +} + function imageBuildStep(target: SelfMediaDeliveryTarget): Record { return { name: "image-build", @@ -197,7 +240,14 @@ function imageBuildStep(target: SelfMediaDeliveryTarget): Record"$askpass" <<'ASKPASS' #!/bin/sh @@ -236,6 +287,8 @@ bun /workspace/source/deploy/scripts/publish-gitops.ts \\ --source-commit "$SOURCE_COMMIT" \\ --worktree /workspace/selfmedia-gitops rm -f "$askpass" +stage_finished_ms=$(date +%s%3N) +printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))" `, securityContext: nonRootSecurity(), volumeMounts: [ @@ -354,3 +407,7 @@ function requiredStringArray(value: unknown, path: string): string[] { if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path} must be an array of non-empty strings`); return value as string[]; } + +function shellSingleQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} From 56b3b344b205d6a7ea59007e6b576c683b371bf7 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 13:01:45 +0200 Subject: [PATCH 10/12] fix: use portable selfmedia stage timing --- .../src/platform-infra-pipelines-as-code.ts | 2 +- scripts/src/selfmedia-delivery-renderer.ts | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index 41102446..a6ff3c87 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -2884,7 +2884,7 @@ function envReuseText(envReuse: Record): string { function stageTimingsText(timings: Record): string { const parts = Object.entries(timings) .filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1])) - .map(([name, milliseconds]) => `${name}=${Math.round(milliseconds)}ms`); + .map(([name, value]) => `${name}=${Math.round(value)}${name.endsWith("Seconds") ? "s" : "ms"}`); return parts.length === 0 ? "-" : parts.join(","); } diff --git a/scripts/src/selfmedia-delivery-renderer.ts b/scripts/src/selfmedia-delivery-renderer.ts index bab3d09a..75ca0ea7 100644 --- a/scripts/src/selfmedia-delivery-renderer.ts +++ b/scripts/src/selfmedia-delivery-renderer.ts @@ -140,7 +140,7 @@ function sourceStep(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: str ], script: `#!/bin/sh set -eu -stage_started_ms=$(date +%s%3N) +stage_started_seconds=$(date +%s) source_commit="$(params.revision)" snapshot_prefix="$(params.source-snapshot-prefix)" rm -rf /workspace/source /workspace/release @@ -171,8 +171,8 @@ bun deploy/scripts/prepare-build.ts \\ --source-commit "$source_commit" \\ --source-snapshot-prefix "$snapshot_prefix" \\ --output-dir /workspace/release -stage_finished_ms=$(date +%s%3N) -printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))" +stage_finished_seconds=$(date +%s) +printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareSeconds":%s},"valuesPrinted":false}\n' "$((stage_finished_seconds-stage_started_seconds))" `, securityContext: nonRootSecurity(), volumeMounts: [ @@ -205,7 +205,7 @@ function envReuseStep(target: SelfMediaDeliveryTarget): Record imagePullPolicy: "IfNotPresent", script: `#!/bin/sh set -eu -stage_started_ms=$(date +%s%3N) +stage_started_seconds=$(date +%s) identity_file=/home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity for input in ${identityInputs}; do test -f "$input"; done env_identity="sha256:$(sha256sum ${identityInputs} | sha256sum | awk '{print $1}')" @@ -219,8 +219,8 @@ else if [ -n "$previous_identity" ]; then previous_present=true; else previous_present=false; fi fi printf '%s\n' "$env_identity" > /workspace/env-identity -stage_finished_ms=$(date +%s%3N) -printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentityMs":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_ms-stage_started_ms))" +stage_finished_seconds=$(date +%s) +printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentitySeconds":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_seconds-stage_started_seconds))" `, securityContext: nonRootSecurity(), volumeMounts: [ @@ -242,11 +242,11 @@ function imageBuildStep(target: SelfMediaDeliveryTarget): Record"$askpass" <<'ASKPASS' #!/bin/sh @@ -287,8 +287,8 @@ bun /workspace/source/deploy/scripts/publish-gitops.ts \\ --source-commit "$SOURCE_COMMIT" \\ --worktree /workspace/selfmedia-gitops rm -f "$askpass" -stage_finished_ms=$(date +%s%3N) -printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))" +stage_finished_seconds=$(date +%s) +printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishSeconds":%s},"valuesPrinted":false}\n' "$((stage_finished_seconds-stage_started_seconds))" `, securityContext: nonRootSecurity(), volumeMounts: [ From 3ae9495209239ba840b562d30bf6a2bff50a916b Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 13:04:07 +0200 Subject: [PATCH 11/12] =?UTF-8?q?docs:=20=E8=B7=9F=E8=B8=AA=E9=99=88?= =?UTF-8?q?=E6=97=A7=20turn=20admission=20=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/MDTODO/agentrun-runtime-reliability.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/MDTODO/agentrun-runtime-reliability.md b/docs/MDTODO/agentrun-runtime-reliability.md index 2d031ec3..54e1df06 100644 --- a/docs/MDTODO/agentrun-runtime-reliability.md +++ b/docs/MDTODO/agentrun-runtime-reliability.md @@ -121,6 +121,9 @@ ### R5.8 [in_progress] 解决 [UniDesk #2014](https://github.com/pikasTech/unidesk/issues/2014)：把 `config/unidesk-cli.yaml#github.auth.repositoryOverrides` 的既有私有仓凭据通过 YAML-first tool credential/resource bundle 投影给 Artificer，使受控 `unidesk-gh` 可读取、创建和评论任务私有仓 PR；Secret 只披露 SecretRef presence/fingerprint/valuesPrinted=false，不新增第二 GitHub 客户端、权限契约、租约或围栏，并与 PikaOA MVP 主线并行且不作为业务门禁，完成任务后将详细报告写入[任务报告](./details/agentrun-runtime-reliability/R5.8_Task_Report.md)。 +### R5.9 [in_progress] + +解决 [AgentRun #356](https://github.com/pikasTech/agentrun/issues/356)：修复 runner Job 已不存在且 lease 过期后，非终态 active turn admission 仍以 `session-turn-admission-active` 阻塞同 session `send`；按既有 SPEC 在 memory/PostgreSQL authority 内原子终结或隔离陈旧 admission 并创建新 run/turn/runner，保留 fresh active steer/并发拒绝，不新增第二 session authority、租约、围栏或客户端 retry，并与 PikaOA MVP 并行且不作为业务门禁，完成任务后将详细报告写入[任务报告](./details/agentrun-runtime-reliability/R5.9_Task_Report.md)。 ## R6 [completed] 将“先恢复运行面再工程化”固化为 unidesk-subagent 的泛化主线规则：运行面恢复关键路径由主代理控制，低耦合工程化任务及时并行，恢复后继续完成工程化交付；将单 PR 收口、复用 guarded merge 内建 preflight、避免仅补 merge SHA 的重复 PR 和减少碎片化 GitHub 查询固化到 unidesk-gh 及相关 reference，并让 merge 默认摘要直接披露 merge commit 与 mergedAt，完成任务后将详细报告写入[任务报告](./details/agentrun-runtime-reliability/R6_Task_Report.md)。 From 2412f47e95cb94d00926c3201dc5cdbba17d8ff1 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 13:13:04 +0200 Subject: [PATCH 12/12] refactor: split Sub2API remote Python modules --- .../remote-python-core.ts | 1219 ++++++ .../remote-python-sentinel-probe.ts | 189 + .../remote-python-sentinel.ts | 614 +++ .../remote-python-sync-validate.ts | 142 + .../remote-python-sync.ts | 3321 +---------------- .../remote-python-trace.ts | 345 ++ .../remote-python-validation.ts | 809 ++++ 7 files changed, 3332 insertions(+), 3307 deletions(-) create mode 100644 scripts/src/platform-infra-sub2api-codex/remote-python-core.ts create mode 100644 scripts/src/platform-infra-sub2api-codex/remote-python-sentinel-probe.ts create mode 100644 scripts/src/platform-infra-sub2api-codex/remote-python-sentinel.ts create mode 100644 scripts/src/platform-infra-sub2api-codex/remote-python-sync-validate.ts create mode 100644 scripts/src/platform-infra-sub2api-codex/remote-python-trace.ts create mode 100644 scripts/src/platform-infra-sub2api-codex/remote-python-validation.ts diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-core.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-core.ts new file mode 100644 index 00000000..fd7de0fa --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-core.ts @@ -0,0 +1,1219 @@ +// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote-python-sync module for scripts/src/platform-infra-sub2api-codex.ts. + +// Moved mechanically from scripts/src/platform-infra-sub2api-codex.ts:4265-5400 for #903. + +import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { UniDeskConfig } from "../config"; +import { rootPath } from "../config"; +import type { RenderedCliResult } from "../output"; +import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "../platform-infra-public-service"; +import { shortSha256Fingerprint } from "../platform-infra-ops-library"; +import { + codexPoolSentinelSummary, + codexPoolSentinelRuntimeImage, + readCodexPoolSentinelConfig, + renderCodexPoolSentinelManifest, + type CodexPoolSentinelConfig, + type CodexPoolSentinelProfileSecret, +} from "../platform-infra-sub2api-codex-sentinel"; +import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets"; +import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; + +import type { CodexPoolConfig, CodexPoolRuntimeTarget } from "./types"; +import { desiredAccountCapacityMap, desiredAccountLoadFactorMap, desiredAccountTempUnschedulableMap, desiredAccountWebSocketsV2ModeMap, desiredDefaultAccountTempUnschedulable } from "./accounts"; +import { resolvedManualAccountProtections } from "./public-exposure"; +import { fieldManager } from "./types"; + +function pyJson(value: unknown): string { + return `json.loads(${JSON.stringify(JSON.stringify(value))})`; +} + +export function remotePythonCoreScript(mode: "sync" | "validate" | "trace" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { + const hostDockerEnvPath = target.runtimeMode === "host-docker" ? target.hostDockerEnvPath : null; + return ` +set -u +python3 - <<'PY' +import base64 +import hashlib +import json +import os +import re +import secrets +import string +import subprocess +import sys +import time +from datetime import datetime, timezone, timedelta +from urllib.parse import quote + +TARGET_ID = ${pyJson(target.id)} +RUNTIME_MODE = ${pyJson(target.runtimeMode)} +NAMESPACE = ${pyJson(target.namespace)} +SERVICE_NAME = ${pyJson(target.serviceName)} +SERVICE_DNS = ${pyJson(target.serviceDns)} +HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} +HOST_DOCKER_ENV_PATH = ${pyJson(hostDockerEnvPath)} +HOST_DOCKER_APP_CONTAINER = "sub2api-app" +FIELD_MANAGER = "${fieldManager}" +APP_SECRET_NAME = ${pyJson(target.appSecretName)} +POOL_GROUP_NAME = "${pool.groupName}" +POOL_GROUP_DESCRIPTION = ${pyJson(pool.groupDescription)} +POOL_API_KEY_NAME = "${pool.apiKeyName}" +POOL_API_KEY_SECRET_NAME = "${pool.apiKeySecretName}" +POOL_API_KEY_SECRET_KEY = "${pool.apiKeySecretKey}" +POOL_ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)} +MIN_OWNER_BALANCE_USD = ${pyJson(pool.minOwnerBalanceUsd)} +MIN_OWNER_CONCURRENCY = ${pyJson(pool.minOwnerConcurrency)} +MIN_OWNER_CONCURRENCY_SOURCE = ${pyJson(pool.minOwnerConcurrencySource)} +POOL_DEFAULT_ACCOUNT_PRIORITY = ${pyJson(pool.defaultAccountPriority)} +POOL_DEFAULT_ACCOUNT_CAPACITY = ${pyJson(pool.defaultAccountCapacity)} +POOL_DEFAULT_ACCOUNT_LOAD_FACTOR = ${pyJson(pool.defaultAccountLoadFactor)} +RESPONSES_SMOKE_MODEL = ${pyJson(pool.localCodex.responsesSmokeModel)} +EXPECTED_ACCOUNT_CAPACITIES = ${pyJson(desiredAccountCapacityMap(pool))} +EXPECTED_ACCOUNT_LOAD_FACTORS = ${pyJson(desiredAccountLoadFactorMap(pool))} +EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))}) +EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))}) +EXPECTED_DEFAULT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredDefaultAccountTempUnschedulable(pool)))}) +MANUAL_ACCOUNT_PROTECTIONS = json.loads(${JSON.stringify(JSON.stringify(resolvedManualAccountProtections(pool, target)))}) +SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))}) +TARGET_EGRESS_PROXY = json.loads(${JSON.stringify(JSON.stringify(target.egressProxy))}) +TARGET_SENTINEL_ENABLED = ${target.sentinelEnabled ? "True" : "False"} +MODE = "${mode}" +PAYLOAD_B64 = "${encodedPayload}" + +def run(cmd, input_bytes=None): + return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +def text(data, limit=4000): + if isinstance(data, bytes): + data = data.decode("utf-8", errors="replace") + return data[-limit:] + +def read_host_env(): + if RUNTIME_MODE != "host-docker": + return {} + if not isinstance(HOST_DOCKER_ENV_PATH, str) or not HOST_DOCKER_ENV_PATH: + raise RuntimeError("host-docker env source path missing") + values = {} + lines = read_host_env_lines() + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + key = key.strip() + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + if key: + values[key] = value + return values + +def read_host_env_lines(): + try: + with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle: + return handle.read().splitlines() + except FileNotFoundError: + raise RuntimeError(f"host-docker env source missing: {HOST_DOCKER_ENV_PATH}") + except PermissionError: + proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH]) + if proc.returncode != 0: + raise RuntimeError("read host-docker env source failed: " + text(proc.stderr, 1000)) + return proc.stdout.decode("utf-8", errors="replace").splitlines() + +def write_host_env_value(key, value): + if RUNTIME_MODE != "host-docker": + raise RuntimeError("write_host_env_value is only valid for host-docker") + if not isinstance(HOST_DOCKER_ENV_PATH, str) or not HOST_DOCKER_ENV_PATH: + raise RuntimeError("host-docker env source path missing") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + raise RuntimeError(f"unsupported env key: {key}") + os.makedirs(os.path.dirname(HOST_DOCKER_ENV_PATH), exist_ok=True) + try: + lines = read_host_env_lines() + except RuntimeError as exc: + if "missing" not in str(exc): + raise + lines = [] + next_lines = [] + replaced = False + for line in lines: + stripped = line.strip() + if stripped.startswith("#") or "=" not in stripped: + next_lines.append(line) + continue + current_key = stripped.split("=", 1)[0].strip() + if current_key == key: + next_lines.append(f"{key}={value}") + replaced = True + else: + next_lines.append(line) + if not replaced: + next_lines.append(f"{key}={value}") + content = "\\n".join(next_lines).rstrip() + "\\n" + tmp_path = HOST_DOCKER_ENV_PATH + ".tmp" + try: + with open(tmp_path, "w", encoding="utf-8") as handle: + handle.write(content) + os.chmod(tmp_path, 0o600) + os.replace(tmp_path, HOST_DOCKER_ENV_PATH) + except PermissionError: + try: + os.unlink(tmp_path) + except Exception: + pass + script = r''' +set -eu +path="$1" +dir="$(dirname "$path")" +mkdir -p "$dir" +tmp="$path.tmp.$$" +umask 077 +cat > "$tmp" +mv "$tmp" "$path" +chmod 600 "$path" +''' + proc = run(["sudo", "-n", "sh", "-c", script, "sh", HOST_DOCKER_ENV_PATH], content.encode("utf-8")) + if proc.returncode != 0: + raise RuntimeError("write host-docker env source failed: " + text(proc.stderr, 1000)) + return "updated" if replaced else "created" + +def docker(args): + proc = run(["docker", *args]) + if proc.returncode == 0: + return proc + sudo_proc = run(["sudo", "-n", "docker", *args]) + return sudo_proc if sudo_proc.returncode == 0 else proc + +def runtime_logs(since, tail): + if RUNTIME_MODE == "host-docker": + return docker(["logs", f"--since={since}", f"--tail={tail}", HOST_DOCKER_APP_CONTAINER]) + return kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) + +def kubectl(args, input_obj=None): + if isinstance(input_obj, str): + input_bytes = input_obj.encode("utf-8") + else: + input_bytes = input_obj + return run(["kubectl", *args], input_bytes) + +def require_kubectl(args, input_obj=None, label="kubectl"): + proc = kubectl(args, input_obj) + if proc.returncode != 0: + raise RuntimeError(f"{label} failed: {text(proc.stderr, 1000)}") + return proc.stdout + +def kube_json(args, label): + raw = require_kubectl([*args, "-o", "json"], label=label) + return json.loads(raw.decode("utf-8")) + +def decode_secret_value(name, key): + if RUNTIME_MODE == "host-docker": + return read_host_env().get(key) + data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {} + if key not in data: + return None + return base64.b64decode(data[key]).decode("utf-8") + +def get_config_value(name, key): + if RUNTIME_MODE == "host-docker": + return read_host_env().get(key) + data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {} + value = data.get(key) + return value if isinstance(value, str) and value else None + +def select_app_pod(): + if RUNTIME_MODE == "host-docker": + return HOST_DOCKER_APP_CONTAINER + pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or [] + for pod in pods: + status = pod.get("status") or {} + if status.get("phase") != "Running": + continue + statuses = status.get("containerStatuses") or [] + if statuses and all(item.get("ready") is True for item in statuses): + return pod["metadata"]["name"] + if pods: + return pods[0]["metadata"]["name"] + raise RuntimeError("sub2api app pod not found") + +APP_POD = select_app_pod() + +def parse_curl_output(proc): + stdout = proc.stdout.decode("utf-8", errors="replace") + marker = "\\n__HTTP_CODE__:" + pos = stdout.rfind(marker) + if pos < 0: + return { + "ok": False, + "httpStatus": 0, + "json": None, + "body": stdout, + "stderr": text(proc.stderr, 1000), + "transportExitCode": proc.returncode, + } + body = stdout[:pos] + status_text = stdout[pos + len(marker):].strip() + try: + http_status = int(status_text[-3:]) + except ValueError: + http_status = 0 + try: + parsed = json.loads(body) if body.strip() else None + except json.JSONDecodeError: + parsed = None + return { + "ok": proc.returncode == 0 and 200 <= http_status < 300, + "httpStatus": http_status, + "json": parsed, + "body": body, + "stderr": text(proc.stderr, 1000), + "transportExitCode": proc.returncode, + } + +def utc_iso(offset_seconds=0): + return (datetime.now(timezone.utc) + timedelta(seconds=offset_seconds)).strftime("%Y-%m-%dT%H:%M:%SZ") + +def normalize_runtime_base_url(value): + if not isinstance(value, str): + return None + value = value.strip().rstrip("/") + return value or None + +def empty_to_none(value): + return value if isinstance(value, str) and value else None + +def sentinel_quality_gate_enabled(): + return TARGET_SENTINEL_ENABLED and (SENTINEL_CONFIG.get("monitor") or {}).get("enabled") is True and (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is True + +def account_notes_fingerprint(account): + notes = account.get("notes") if isinstance(account, dict) else None + if not isinstance(notes, str): + return None + match = re.search(r"fingerprint=([A-Za-z0-9_-]+)", notes) + return match.group(1) if match else None + +def runtime_account_credentials(account): + credentials = account.get("credentials") if isinstance(account, dict) and isinstance(account.get("credentials"), dict) else {} + return credentials + +def runtime_account_extra(account): + extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {} + return extra + +def sentinel_probe_change_reasons(current, profile): + if not isinstance(current, dict) or current.get("id") is None: + return ["created"] + credentials = runtime_account_credentials(current) + extra = runtime_account_extra(current) + expected_base_url = normalize_runtime_base_url(profile.get("baseUrl")) + runtime_base_url = normalize_runtime_base_url(credentials.get("base_url")) + expected_user_agent = empty_to_none(profile.get("upstreamUserAgent")) + runtime_user_agent = empty_to_none(credentials.get("user_agent")) + expected_ws_mode = empty_to_none(profile.get("openaiResponsesWebSocketsV2Mode")) + runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode")) + expected_trust_upstream = profile.get("trustUpstream") is True + runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True + expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} + runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False} + reasons = [] + if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"): + reasons.append("profile") + if runtime_base_url != expected_base_url: + reasons.append("base-url") + if account_notes_fingerprint(current) != profile.get("apiKeyFingerprint"): + reasons.append("api-key-fingerprint") + if runtime_user_agent != expected_user_agent: + reasons.append("upstream-user-agent") + if runtime_ws_mode != expected_ws_mode: + reasons.append("responses-websockets-v2-mode") + if runtime_trust_upstream != expected_trust_upstream: + reasons.append("trust-upstream") + if runtime_protect != expected_protect: + reasons.append("sentinel-protect") + return reasons + +def curl_api(method, path, bearer=None, payload=None): + body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") + script = r''' +set -eu +method="$1" +url="$2" +token="\${3:-}" +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +cat > "$tmp" +if [ -n "$token" ]; then + if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then + curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url" + else + curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url" + fi +else + if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then + curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" "$url" + else + curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" + fi +fi +''' + if RUNTIME_MODE == "host-docker": + if not isinstance(HOST_DOCKER_APP_PORT, int): + raise RuntimeError("host-docker app port missing") + proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body) + else: + proc = run([ + "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, + "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or "", + ], body) + return parse_curl_output(proc) + +def envelope_data(parsed): + if isinstance(parsed, dict) and "data" in parsed: + return parsed.get("data") + return parsed + +def ensure_success(resp, label): + parsed = resp.get("json") + code = parsed.get("code") if isinstance(parsed, dict) else None + if not resp.get("ok") or (code is not None and code != 0): + message = parsed.get("message") if isinstance(parsed, dict) else text(resp.get("body", ""), 500) + raise RuntimeError(f"{label} failed: http={resp.get('httpStatus')} message={message}") + return envelope_data(parsed) + +def ensure_admin_compliance(token): + status_resp = curl_api("GET", "/api/v1/admin/compliance", bearer=token) + if status_resp.get("httpStatus") == 404: + return { + "ok": True, + "action": "not-supported-by-runtime", + "valuesPrinted": False, + } + status = ensure_success(status_resp, "get admin compliance") + if not isinstance(status, dict): + return { + "ok": True, + "action": "status-unstructured", + "valuesPrinted": False, + } + version = status.get("version") + required = status.get("required") is True + if not required: + return { + "ok": True, + "action": "already-acknowledged", + "version": version, + "requiredBefore": False, + "requiredAfter": False, + "phrasePrinted": False, + "valuesPrinted": False, + } + phrase = status.get("ack_phrase_zh") if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else status.get("ack_phrase_en") + language = "zh" if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else "en" + if not isinstance(phrase, str) or not phrase: + raise RuntimeError("admin compliance acknowledgement phrase missing") + accepted = ensure_success( + curl_api("POST", "/api/v1/admin/compliance/accept", bearer=token, payload={"phrase": phrase, "language": language}), + "accept admin compliance", + ) + required_after = accepted.get("required") if isinstance(accepted, dict) else None + return { + "ok": required_after is False, + "action": "accepted", + "version": version, + "requiredBefore": True, + "requiredAfter": required_after, + "phrasePrinted": False, + "valuesPrinted": False, + } + +def extract_items(data): + if isinstance(data, list): + return data + if isinstance(data, dict): + if isinstance(data.get("items"), list): + return data["items"] + for key in ("groups", "accounts", "api_keys", "keys"): + if isinstance(data.get(key), list): + return data[key] + return [] + +def find_access_token(data): + if isinstance(data, dict): + for key in ("access_token", "token"): + if isinstance(data.get(key), str) and data[key]: + return data[key] + for value in data.values(): + token = find_access_token(value) + if token: + return token + return None + +def login(): + admin_email = get_config_value("sub2api-config", "ADMIN_EMAIL") or POOL_ADMIN_EMAIL_DEFAULT + admin_password = decode_secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") + if not admin_password: + raise RuntimeError("ADMIN_PASSWORD missing from sub2api-secrets") + data = ensure_success(curl_api("POST", "/api/v1/auth/login", payload={"email": admin_email, "password": admin_password}), "admin login") + token = find_access_token(data) + if not token: + raise RuntimeError("admin login response did not contain access_token") + compliance = ensure_admin_compliance(token) + if compliance.get("ok") is not True: + raise RuntimeError("admin compliance acknowledgement failed") + return admin_email, token, compliance + +def group_payload(): + return { + "name": POOL_GROUP_NAME, + "description": POOL_GROUP_DESCRIPTION, + "platform": "openai", + "rate_multiplier": 1, + "is_exclusive": False, + "subscription_type": "standard", + "allow_messages_dispatch": True, + "require_oauth_only": False, + "require_privacy_set": False, + "rpm_limit": 0, + } + +def list_groups(token): + data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups") + return extract_items(data) + +def ensure_group(token): + existing = next((item for item in list_groups(token) if item.get("name") == POOL_GROUP_NAME), None) + payload = group_payload() + if existing is None: + created = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=payload), "create group") + return created, "created" + group_id = existing.get("id") + if group_id is None: + raise RuntimeError("existing group has no id") + payload["status"] = "active" + updated = ensure_success(curl_api("PUT", f"/api/v1/admin/groups/{group_id}", bearer=token, payload=payload), "update group") + return updated if isinstance(updated, dict) else existing, "updated" + +def list_accounts_for_group(token, group_id): + path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500&platform=openai" + data = ensure_success(curl_api("GET", path, bearer=token), f"list accounts for group {group_id}") + return extract_items(data) + +def account_group_ids(token, account): + if not isinstance(account, dict) or account.get("id") is None: + return [] + account_id = account.get("id") + account_name = account.get("name") + ids = [] + for group in list_groups(token): + group_id = group.get("id") if isinstance(group, dict) else None + if group_id is None: + continue + members = list_accounts_for_group(token, group_id) + if any(item.get("id") == account_id or item.get("name") == account_name for item in members if isinstance(item, dict)): + ids.append(group_id) + return sorted(set(ids)) + +def list_accounts(token): + path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-codex-") + data = ensure_success(curl_api("GET", path, bearer=token), "list accounts") + return extract_items(data) + +def find_account_by_name(token, name): + path = "/api/v1/admin/accounts?page=1&page_size=50&platform=openai&search=" + quote(str(name)) + data = ensure_success(curl_api("GET", path, bearer=token), "find account " + str(name)) + for item in extract_items(data): + if isinstance(item, dict) and item.get("name") == name: + return item + return None + +def list_proxy_candidates(token, search=""): + path = "/api/v1/admin/proxies?page=1&page_size=200" + if search: + path += "&search=" + quote(str(search)) + data = ensure_success(curl_api("GET", path, bearer=token), "list proxies") + return extract_items(data) + +def find_proxy_by_name(token, name): + for item in list_proxy_candidates(token, name): + if isinstance(item, dict) and item.get("name") == name: + return item + return None + +def manual_binding_plan(binding, expected_kind): + if not isinstance(binding, dict): + return None + plan = binding.get("sourcePlan") + if not isinstance(plan, dict): + raise RuntimeError("manual account binding is missing resolved sourcePlan") + if plan.get("enabled") is not True: + raise RuntimeError(f"manual account binding source {plan.get('id')} is disabled") + if plan.get("kind") != expected_kind: + raise RuntimeError(f"manual account binding source {plan.get('id')} kind must be {expected_kind}") + return plan + +def desired_manual_proxy_payload(protection): + binding = protection.get("proxyBinding") if isinstance(protection, dict) else None + if not isinstance(binding, dict) or binding.get("enabled") is not True: + return None + plan = manual_binding_plan(binding, "proxy") + if plan.get("provider") not in ("target-egress-proxy", "fixed-http-proxy"): + raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}") + if plan.get("provider") == "fixed-http-proxy": + fixed_proxy = plan.get("fixedProxy") + if not isinstance(fixed_proxy, dict): + raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} requires fixedProxy") + proxy_name = binding.get("proxyName") + protocol = fixed_proxy.get("protocol") + host = fixed_proxy.get("host") + port = fixed_proxy.get("port") + if not isinstance(proxy_name, str) or not proxy_name: + raise RuntimeError("manual account proxyBinding proxyName is missing") + if protocol != "http": + raise RuntimeError("fixed proxy protocol must be http") + if not isinstance(host, str) or not host: + raise RuntimeError("fixed proxy host is missing") + if not isinstance(port, int) or port <= 0: + raise RuntimeError("fixed proxy port is missing") + return { + "name": proxy_name, + "protocol": protocol, + "host": host, + "port": port, + "fallback_mode": "none", + "expiry_warn_days": 0, + } + target_proxy = plan.get("targetEgressProxy") + if not isinstance(target_proxy, dict): + raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} requires an enabled target egressProxy") + service_name = target_proxy.get("serviceName") + listen_port = target_proxy.get("listenPort") + namespace = target_proxy.get("namespace") if isinstance(target_proxy.get("namespace"), str) and target_proxy.get("namespace") else NAMESPACE + proxy_name = binding.get("proxyName") + if not isinstance(service_name, str) or not service_name: + raise RuntimeError("target egressProxy serviceName is missing") + if not isinstance(listen_port, int) or listen_port <= 0: + raise RuntimeError("target egressProxy listenPort is missing") + if not isinstance(proxy_name, str) or not proxy_name: + raise RuntimeError("manual account proxyBinding proxyName is missing") + return { + "name": proxy_name, + "protocol": "http", + "host": f"{service_name}.{namespace}.svc.cluster.local", + "port": listen_port, + "fallback_mode": "none", + "expiry_warn_days": 0, + } + +def proxy_needs_update(proxy, payload): + if not isinstance(proxy, dict): + return True + for key in ("name", "protocol", "host", "port", "fallback_mode", "expiry_warn_days"): + if proxy.get(key) != payload.get(key): + return True + return proxy.get("status") != "active" + +def ensure_manual_proxy(token, payload): + current = find_proxy_by_name(token, payload["name"]) + if current is None: + created = ensure_success(curl_api("POST", "/api/v1/admin/proxies", bearer=token, payload=payload), f"create proxy {payload['name']}") + return created, "created" + if proxy_needs_update(current, payload): + update_payload = dict(payload) + update_payload["status"] = "active" + updated = ensure_success(curl_api("PUT", f"/api/v1/admin/proxies/{current['id']}", bearer=token, payload=update_payload), f"update proxy {payload['name']}") + return updated if isinstance(updated, dict) else current, "updated" + return current, "unchanged" + +def manual_proxy_status(token, account, protection): + payload = desired_manual_proxy_payload(protection) + if payload is None: + return { + "enabled": False, + "ok": True, + "action": "not-configured", + "valuesPrinted": False, + } + binding = protection.get("proxyBinding") if isinstance(protection, dict) else None + plan = manual_binding_plan(binding, "proxy") + proxy = find_proxy_by_name(token, payload["name"]) + runtime_proxy = account.get("proxy") if isinstance(account, dict) and isinstance(account.get("proxy"), dict) else None + binding_aligned = ( + isinstance(account, dict) + and isinstance(proxy, dict) + and account.get("proxy_id") == proxy.get("id") + ) + return { + "enabled": True, + "ok": binding_aligned, + "action": "validate", + "source": plan.get("id"), + "sourceProvider": plan.get("provider"), + "expectedProxyName": payload["name"], + "expectedProtocol": payload["protocol"], + "expectedHost": payload["host"], + "expectedPort": payload["port"], + "proxyRecordExists": isinstance(proxy, dict), + "proxyId": proxy.get("id") if isinstance(proxy, dict) else None, + "runtimeProxyId": account.get("proxy_id") if isinstance(account, dict) else None, + "runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None, + "bindingAligned": binding_aligned, + "valuesPrinted": False, + } + +def ensure_manual_account_proxy_bindings(token): + items = [] + for protection in MANUAL_ACCOUNT_PROTECTIONS: + if not isinstance(protection, dict): + continue + name = protection.get("accountName") + if not isinstance(name, str) or not name: + continue + account = find_account_by_name(token, name) + payload = desired_manual_proxy_payload(protection) + if payload is None: + items.append({ + "accountName": name, + "enabled": False, + "action": "not-configured", + "ok": True, + "valuesPrinted": False, + }) + continue + if not isinstance(account, dict): + items.append({ + "accountName": name, + "enabled": True, + "action": "account-missing", + "ok": False, + "expectedProxyName": payload["name"], + "valuesPrinted": False, + }) + continue + proxy, proxy_action = ensure_manual_proxy(token, payload) + binding = protection.get("proxyBinding") if isinstance(protection, dict) else None + plan = manual_binding_plan(binding, "proxy") + proxy_id = proxy.get("id") if isinstance(proxy, dict) else None + if proxy_id is None: + raise RuntimeError(f"proxy {payload['name']} has no id") + action = "unchanged" + if account.get("proxy_id") != proxy_id: + updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"proxy_id": proxy_id}), f"bind manual account proxy {name}") + account = updated if isinstance(updated, dict) else account + action = "bound" + runtime_proxy = account.get("proxy") if isinstance(account.get("proxy"), dict) else None + items.append({ + "accountName": name, + "accountId": account.get("id"), + "enabled": True, + "action": action, + "proxyAction": proxy_action, + "source": plan.get("id"), + "sourceProvider": plan.get("provider"), + "ok": account.get("proxy_id") == proxy_id, + "expectedProxyName": payload["name"], + "expectedProtocol": payload["protocol"], + "expectedHost": payload["host"], + "expectedPort": payload["port"], + "proxyId": proxy_id, + "runtimeProxyId": account.get("proxy_id"), + "runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None, + "bindingAligned": account.get("proxy_id") == proxy_id, + "controlPolicy": "manual-protected: only proxy_id binding is YAML-controlled; credentials/status/schedulable are untouched", + "valuesPrinted": False, + }) + return { + "ok": all(item.get("ok") is True for item in items), + "itemCount": len(items), + "items": items, + "valuesPrinted": False, + } + +def manual_group_binding_plan(protection): + binding = protection.get("groupBinding") if isinstance(protection, dict) else None + if not isinstance(binding, dict) or binding.get("enabled") is not True: + return None + plan = manual_binding_plan(binding, "group") + if plan.get("provider") != "pool-group": + raise RuntimeError(f"manual account groupBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}") + return plan + +def manual_group_binding_enabled(protection): + return manual_group_binding_plan(protection) is not None + +def manual_group_status(token, account, protection, group_id): + plan = manual_group_binding_plan(protection) + if plan is None: + return { + "enabled": False, + "ok": True, + "action": "not-configured", + "valuesPrinted": False, + } + group_accounts = list_accounts_for_group(token, group_id) + account_id = account.get("id") if isinstance(account, dict) else None + account_name = account.get("name") if isinstance(account, dict) else None + binding_aligned = any( + item.get("id") == account_id or item.get("name") == account_name + for item in group_accounts + if isinstance(item, dict) + ) + return { + "enabled": True, + "ok": binding_aligned, + "action": "validate", + "source": plan.get("id"), + "sourceProvider": plan.get("provider"), + "poolGroupName": POOL_GROUP_NAME, + "poolGroupId": group_id, + "bindingAligned": binding_aligned, + "valuesPrinted": False, + } + +def ensure_manual_account_group_bindings(token, group_id): + items = [] + for protection in MANUAL_ACCOUNT_PROTECTIONS: + if not isinstance(protection, dict): + continue + name = protection.get("accountName") + if not isinstance(name, str) or not name: + continue + if not manual_group_binding_enabled(protection): + items.append({ + "accountName": name, + "enabled": False, + "action": "not-configured", + "ok": True, + "valuesPrinted": False, + }) + continue + plan = manual_group_binding_plan(protection) + account = find_account_by_name(token, name) + if not isinstance(account, dict): + items.append({ + "accountName": name, + "enabled": True, + "action": "account-missing", + "ok": False, + "poolGroupName": POOL_GROUP_NAME, + "poolGroupId": group_id, + "valuesPrinted": False, + }) + continue + existing_group_ids = account_group_ids(token, account) + desired_group_ids = sorted(set(existing_group_ids + [group_id])) + action = "unchanged" + if group_id not in existing_group_ids: + updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"group_ids": desired_group_ids}), f"bind manual account group {name}") + account = updated if isinstance(updated, dict) else account + action = "bound" + binding_aligned = any( + item.get("id") == account.get("id") or item.get("name") == name + for item in list_accounts_for_group(token, group_id) + if isinstance(item, dict) + ) + items.append({ + "accountName": name, + "accountId": account.get("id"), + "enabled": True, + "ok": binding_aligned, + "action": action, + "source": plan.get("id") if isinstance(plan, dict) else None, + "sourceProvider": plan.get("provider") if isinstance(plan, dict) else None, + "poolGroupName": POOL_GROUP_NAME, + "poolGroupId": group_id, + "previousGroupIds": existing_group_ids, + "desiredGroupIds": desired_group_ids, + "bindingAligned": binding_aligned, + "controlPolicy": "manual-protected: only pool group membership is YAML-controlled; credentials/status/schedulable are untouched and sentinel does not probe it", + "valuesPrinted": False, + }) + return { + "ok": all(item.get("ok") is True for item in items), + "itemCount": len(items), + "items": items, + "valuesPrinted": False, + } + +def manual_account_protection_status(token, group_id=None): + items = [] + desired_names = set(EXPECTED_ACCOUNT_CAPACITIES.keys()) + for protection in MANUAL_ACCOUNT_PROTECTIONS: + if not isinstance(protection, dict): + continue + name = protection.get("accountName") + if not isinstance(name, str) or not name: + continue + account = find_account_by_name(token, name) + extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {} + proxy_status = manual_proxy_status(token, account, protection) + group_status = manual_group_status(token, account, protection, group_id) if group_id is not None else {"enabled": False, "ok": True, "action": "not-checked", "valuesPrinted": False} + items.append({ + "accountName": name, + "reason": protection.get("reason") if isinstance(protection.get("reason"), str) else None, + "exists": isinstance(account, dict), + "accountId": account.get("id") if isinstance(account, dict) else None, + "status": account.get("status") if isinstance(account, dict) else None, + "schedulable": account.get("schedulable") if isinstance(account, dict) else None, + "inYamlProfiles": name in desired_names, + "runtimeMarkedUnideskManaged": extra.get("unidesk_managed") is True, + "proxyBinding": proxy_status, + "groupBinding": group_status, + "ok": proxy_status.get("ok") is True and group_status.get("ok") is True, + "controlPolicy": "manual-protected: no create/update/prune/probe/freeze; optional proxy_id and pool group membership binding only when configured", + "valuesPrinted": False, + }) + return { + "ok": all(item.get("ok") is True for item in items), + "protectedCount": len(items), + "items": items, + "valuesPrinted": False, + } + +def list_probe_accounts(token): + path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-probe-") + data = ensure_success(curl_api("GET", path, bearer=token), "list probe accounts") + return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")] + +def list_probe_groups(token): + data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups") + return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")] + +def cleanup_probe_resources(token): + api_key_results = [] + account_results = [] + group_results = [] + for item in list_user_keys(token): + name = item.get("name") + key_id = item.get("id") + if not isinstance(name, str) or not name.startswith("unidesk-probe-") or key_id is None: + continue + api_key_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{key_id}", "api-key")) + for item in list_probe_accounts(token): + account_id = item.get("id") + if account_id is None: + continue + account_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account")) + for item in list_probe_groups(token): + group_id = item.get("id") + if group_id is None: + continue + group_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group")) + return { + "ok": all(item.get("ok") is True for item in api_key_results + account_results + group_results), + "apiKeysDeleted": len(api_key_results), + "accountsDeleted": len(account_results), + "groupsDeleted": len(group_results), + "items": { + "apiKeys": api_key_results, + "accounts": account_results, + "groups": group_results, + }, + "valuesPrinted": False, + } + +def temp_unschedulable_credentials(profile): + credentials = profile.get("tempUnschedulableCredentials") + if not isinstance(credentials, dict): + credentials = {} + return normalize_temp_unschedulable_credentials(credentials) + +def account_payload(profile, group_id): + extra = { + "openai_responses_mode": "force_responses", + "unidesk_codex_profile": profile["profile"], + "unidesk_managed": True, + "unidesk_trust_upstream": profile.get("trustUpstream") is True, + "unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, + } + ws_mode = profile.get("openaiResponsesWebSocketsV2Mode") + if ws_mode: + extra["openai_apikey_responses_websockets_v2_mode"] = ws_mode + extra["openai_apikey_responses_websockets_v2_enabled"] = ws_mode != "off" + credentials = { + "api_key": profile["apiKey"], + "base_url": profile["baseUrl"], + } + upstream_user_agent = profile.get("upstreamUserAgent") + if upstream_user_agent: + credentials["user_agent"] = upstream_user_agent + temp_unschedulable = temp_unschedulable_credentials(profile) + if temp_unschedulable["enabled"]: + credentials["temp_unschedulable_enabled"] = True + credentials["temp_unschedulable_rules"] = temp_unschedulable["rules"] + return { + "name": profile["accountName"], + "notes": f"UniDesk-managed Codex profile {profile['profile']} from {profile['configFile']} and {profile['authFile']}; secret source={profile['apiKeySource']}; fingerprint={profile['apiKeyFingerprint']}.", + "platform": "openai", + "type": "apikey", + "credentials": credentials, + "extra": extra, + "concurrency": int(profile.get("capacity", 5) or 5), + "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY), + "rate_multiplier": 1, + "load_factor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR), + "group_ids": [group_id], + "confirm_mixed_channel_risk": True, + **({"proxy_id": int(profile["proxyId"])} if profile.get("proxyId") is not None else {}), + } + +def planned_sentinel_account_results(profiles, existing_accounts): + existing = {item.get("name"): item for item in existing_accounts if isinstance(item, dict)} + results = [] + for profile in profiles: + change_reasons = sentinel_probe_change_reasons(existing.get(profile["accountName"]), profile) + quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0 + results.append({ + "profile": profile["profile"], + "accountName": profile["accountName"], + "profileConfig": { + "accountName": profile["accountName"], + "trustUpstream": profile.get("trustUpstream") is True, + "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, + }, + "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), + "sentinelProbeRequired": quality_gate_required, + "sentinelChangeReasons": change_reasons if quality_gate_required else [], + "sentinelProbePending": quality_gate_required, + "sentinelDefaultFrozen": False, + "valuesPrinted": False, + }) + return results + +def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_frozen_names=None, existing_accounts=None): + if not isinstance(protected_frozen_names, set): + protected_frozen_names = set() + if not isinstance(existing_accounts, list): + existing_accounts = list_accounts(token) + existing = {item.get("name"): item for item in existing_accounts} + desired_names = {profile["accountName"] for profile in profiles} + results = [] + for profile in profiles: + payload = account_payload(profile, group_id) + current = existing.get(profile["accountName"]) + change_reasons = sentinel_probe_change_reasons(current, profile) + quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0 + keep_frozen = profile["accountName"] in protected_frozen_names + if current and current.get("id") is not None: + account_id = current["id"] + update_payload = dict(payload) + update_payload.pop("platform", None) + update_payload["status"] = "active" + data = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account_id}", bearer=token, payload=update_payload), f"update account {profile['accountName']}") + action = "updated" + else: + data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}") + action = "created" + results.append({ + "profile": profile["profile"], + "accountName": profile["accountName"], + "profileConfig": { + "accountName": profile["accountName"], + "trustUpstream": profile.get("trustUpstream") is True, + "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, + }, + "accountId": data.get("id") if isinstance(data, dict) else None, + "action": action, + "baseUrl": profile["baseUrl"], + "apiKeySource": profile["apiKeySource"], + "apiKeyFingerprint": profile["apiKeyFingerprint"], + "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), + "sentinelProbeRequired": quality_gate_required, + "sentinelChangeReasons": change_reasons if quality_gate_required else [], + "sentinelProbePending": quality_gate_required, + "sentinelDefaultFrozen": False, + "sentinelFreezeProtected": keep_frozen, + "schedulableControl": "sentinel-marker-restore", + "openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"), + "trustUpstream": profile.get("trustUpstream") is True, + "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY), + "capacity": int(profile.get("capacity", 5) or 5), + "loadFactor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR), + "runtimeConcurrency": data.get("concurrency") if isinstance(data, dict) else None, + "runtimeLoadFactor": data.get("load_factor") if isinstance(data, dict) else None, + "runtimeSchedulable": data.get("schedulable") if isinstance(data, dict) else None, + "tempUnschedulableConfigured": bool(payload["credentials"].get("temp_unschedulable_enabled")), + "tempUnschedulableRuleCount": len(payload["credentials"].get("temp_unschedulable_rules") or []), + "upstreamUserAgentConfigured": bool(profile.get("upstreamUserAgent")), + "valuesPrinted": False, + }) + prune_results = prune_removed_accounts(token, existing_accounts, desired_names) if prune_removed else [] + return results, prune_results + +def prune_removed_accounts(token, existing_accounts, desired_names): + results = [] + for account in existing_accounts: + name = account.get("name") + account_id = account.get("id") + extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} + if not isinstance(name, str) or not name.startswith("unidesk-codex-"): + continue + if extra.get("unidesk_managed") is not True: + continue + if name in desired_names: + continue + if account_id is None: + raise RuntimeError(f"removed account {name} has no id") + ensure_success(curl_api("DELETE", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"delete removed account {name}") + results.append({ + "accountName": name, + "accountId": account_id, + "profile": extra.get("unidesk_codex_profile"), + "action": "deleted", + "reason": "removed-from-yaml", + "valuesPrinted": False, + }) + return results + +def generate_api_key(): + alphabet = string.ascii_letters + string.digits + return "sk-unidesk-codex-" + "".join(secrets.choice(alphabet) for _ in range(48)) + +def sub2api_key_value(item): + if not isinstance(item, dict): + return None + key = item.get("key") + return key if isinstance(key, str) and key else None + +def find_pool_key_by_name(keys): + return next((item for item in keys if isinstance(item, dict) and item.get("name") == POOL_API_KEY_NAME), None) + +def existing_sub2api_pool_api_key(token): + try: + existing = find_pool_key_by_name(list_user_keys(token)) + except Exception: + return None + return sub2api_key_value(existing) + +def ensure_pool_api_key_for_validate(token, secret_value, source_action): + group, group_action = ensure_group(token) + group_id = group.get("id") if isinstance(group, dict) else None + if group_id is None: + raise RuntimeError("pool group id missing while ensuring validate API key") + api_key = secret_value or generate_api_key() + api_key_result = ensure_sub2api_api_key(token, api_key, group_id) + host_env_action = write_host_env_value(POOL_API_KEY_SECRET_KEY, api_key) + key_item = { + "id": api_key_result.get("id"), + "name": api_key_result.get("name") or POOL_API_KEY_NAME, + "group_id": api_key_result.get("groupId"), + "user_id": api_key_result.get("userId"), + "key": api_key, + } + return api_key, key_item, { + "ok": True, + "source": "sub2api-admin-api", + "sourceAction": source_action, + "secretPresent": bool(secret_value), + "lookup": "ensured-by-admin-api", + "groupAction": group_action, + "sub2apiAction": api_key_result.get("action"), + "sub2apiId": api_key_result.get("id"), + "hostEnvAction": host_env_action, + "hostEnvPath": HOST_DOCKER_ENV_PATH, + "hostEnvKey": POOL_API_KEY_SECRET_KEY, + "valuesPrinted": False, + } + +def resolve_pool_api_key_for_validate(token): + secret_value = None + secret_error = None + try: + secret_value = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY) + except Exception as exc: + if RUNTIME_MODE != "host-docker": + raise + secret_error = str(exc) + keys = list_user_keys(token) + matched_item = None + if secret_value: + matched_item = next((item for item in keys if isinstance(item, dict) and item.get("key") == secret_value), None) + if isinstance(matched_item, dict) and matched_item.get("name") == POOL_API_KEY_NAME: + return secret_value, matched_item, { + "ok": True, + "source": pool_api_key_secret_location(), + "sourceAction": "kept-existing", + "secretPresent": True, + "lookup": "matched-by-value", + "hostEnvAction": "none", + "valuesPrinted": False, + } + if RUNTIME_MODE == "host-docker": + named_item = find_pool_key_by_name(keys) + named_key = sub2api_key_value(named_item) + if named_key: + host_env_action = write_host_env_value(POOL_API_KEY_SECRET_KEY, named_key) + return named_key, named_item, { + "ok": True, + "source": "sub2api-admin-api", + "sourceAction": "recovered-missing" if not secret_value else "repaired-mismatch", + "secretPresent": bool(secret_value), + "lookup": "matched-by-name", + "hostEnvAction": host_env_action, + "hostEnvPath": HOST_DOCKER_ENV_PATH, + "hostEnvKey": POOL_API_KEY_SECRET_KEY, + "valuesPrinted": False, + } + if not secret_value: + return ensure_pool_api_key_for_validate(token, secret_value, "created-missing") + matched_name = matched_item.get("name") if isinstance(matched_item, dict) else None + return ensure_pool_api_key_for_validate(token, secret_value, "repaired-mismatch:" + (matched_name or "-")) + if not secret_value: + raise RuntimeError(f"{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY} missing") + return secret_value, matched_item, { + "ok": isinstance(matched_item, dict) and matched_item.get("name") == POOL_API_KEY_NAME, + "source": pool_api_key_secret_location(), + "sourceAction": "kept-existing", + "secretPresent": True, + "lookup": "matched-by-value" if matched_item else "missing-in-admin-api", + "hostEnvAction": "not-applicable", + "valuesPrinted": False, + } + +def ensure_api_key_secret(group_id, token): + existing = None + try: + existing = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY) + except Exception: + existing = None + reused = None if existing else existing_sub2api_pool_api_key(token) + api_key = existing or reused or generate_api_key() + if existing: + secret_action = "kept-existing" + elif reused: + secret_action = "reused-existing-sub2api-key" + else: + secret_action = "created" + if RUNTIME_MODE == "host-docker": + env_action = "kept-existing" if existing else write_host_env_value(POOL_API_KEY_SECRET_KEY, api_key) + return api_key, secret_action, f"host-docker-env:{env_action};source={HOST_DOCKER_ENV_PATH};key={POOL_API_KEY_SECRET_KEY};valuesPrinted=false" + manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": POOL_API_KEY_SECRET_NAME, + "namespace": NAMESPACE, + "labels": { + "app.kubernetes.io/name": "sub2api", + "app.kubernetes.io/part-of": "platform-infra", + "app.kubernetes.io/managed-by": "unidesk", + "unidesk.ai/secret-purpose": "sub2api-codex-pool-api-key", + }, + }, + "type": "Opaque", + "stringData": { + POOL_API_KEY_SECRET_KEY: api_key, + "GROUP_ID": str(group_id), + "GROUP_NAME": POOL_GROUP_NAME, + "SERVICE_DNS": SERVICE_DNS, + }, + } + proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) + if proc.returncode != 0: + raise RuntimeError(f"apply API key secret failed: {text(proc.stderr, 1000)}") + return api_key, secret_action, text(proc.stdout, 1000) + +`; +} diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sentinel-probe.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sentinel-probe.ts new file mode 100644 index 00000000..a5428ab8 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sentinel-probe.ts @@ -0,0 +1,189 @@ +export const remotePythonSentinelProbeScript = `def parse_embedded_json(stdout): + if not isinstance(stdout, str) or not stdout.strip(): + return None + start = stdout.find("{") + end = stdout.rfind("}") + if start < 0 or end <= start: + return None + try: + return json.loads(stdout[start:end + 1]) + except Exception: + return None + +def inject_env(template, name, value): + spec = template.setdefault("spec", {}) + containers = spec.setdefault("containers", []) + if not containers: + raise RuntimeError("sentinel job template has no containers") + container = containers[0] + env = container.setdefault("env", []) + env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)] + env.append({"name": name, "value": value}) + +def sentinel_probe_job_manifest(accounts): + cronjob_name = SENTINEL_CONFIG.get("cronJobName") + if not isinstance(cronjob_name, str) or not cronjob_name: + raise RuntimeError("sentinel cronJobName missing from config") + cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}") + job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {} + job_spec = copy_json(job_template.get("spec") or {}) + if not isinstance(job_spec, dict) or not job_spec: + raise RuntimeError("sentinel CronJob jobTemplate.spec missing") + template = job_spec.get("template") + if not isinstance(template, dict): + raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing") + inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts)) + job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600) + suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5)) + job_name = ("sub2api-sentinel-probe-" + suffix)[:63] + return job_name, { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": job_name, + "namespace": NAMESPACE, + "labels": { + "app.kubernetes.io/name": cronjob_name, + "app.kubernetes.io/part-of": "platform-infra", + "app.kubernetes.io/managed-by": "unidesk", + "unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe", + }, + }, + "spec": job_spec, + } + +def copy_json(value): + return json.loads(json.dumps(value)) + +def job_condition(job, cond_type): + for item in ((job.get("status") or {}).get("conditions") or []): + if item.get("type") == cond_type and item.get("status") == "True": + return item + return None + +def wait_sentinel_probe_job(job_name, timeout_seconds): + deadline = time.time() + timeout_seconds + latest = None + while time.time() < deadline: + latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}") + if isinstance(latest, dict): + complete = job_condition(latest, "Complete") + failed = job_condition(latest, "Failed") + if complete is not None: + return "succeeded", latest, complete + if failed is not None: + return "failed", latest, failed + time.sleep(2) + return "timeout", latest, None + +def job_logs(job_name): + proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"]) + return { + "exitCode": proc.returncode, + "stdout": proc.stdout.decode("utf-8", errors="replace"), + "stderr": text(proc.stderr, 4000), + } + +def run_sentinel_probe(): + payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} + accounts = payload.get("accounts") if isinstance(payload, dict) else None + if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts): + raise RuntimeError("sentinel-probe payload requires non-empty accounts") + job_name, manifest = sentinel_probe_job_manifest(accounts) + proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) + if proc.returncode != 0: + raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}") + timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240) + status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds) + logs = job_logs(job_name) + parsed = parse_embedded_json(logs.get("stdout") or "") + results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else [] + state_summary = sentinel_state_summary() + last_run = state_summary.get("lastRun") if isinstance(state_summary, dict) and isinstance(state_summary.get("lastRun"), dict) else {} + if not results and isinstance(last_run.get("results"), list): + results = last_run.get("results") + requested = set(accounts) + measured = {item.get("accountName") for item in results if isinstance(item, dict)} + missing = sorted(name for name in requested if name not in measured) + marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested) + if not results and isinstance(last_run, dict): + selected = int(last_run.get("selected") or 0) + ok_count = int(last_run.get("okCount") or 0) + marker_mismatch_count = int(last_run.get("markerMismatchCount") or 0) + transport_failure_count = int(last_run.get("transportFailureCount") or 0) + if selected >= len(requested) and ok_count >= len(requested) and marker_mismatch_count == 0 and transport_failure_count == 0: + missing = [] + marker_ok = True + job_ok = status == "succeeded" and logs.get("exitCode") == 0 + return { + "ok": job_ok and marker_ok, + "jobExecutionOk": job_ok, + "markerOk": marker_ok, + "mode": "sentinel-probe", + "namespace": NAMESPACE, + "requestedAccounts": accounts, + "missingAccounts": missing, + "job": { + "name": job_name, + "status": status, + "condition": condition, + "timeoutSeconds": timeout_seconds, + "applyStdoutTail": text(proc.stdout, 1200), + "logsExitCode": logs.get("exitCode"), + "logsStderrTail": logs.get("stderr"), + }, + "probe": parsed, + "summary": { + "at": last_run.get("at"), + "selected": last_run.get("selected"), + "okCount": last_run.get("okCount"), + "markerMismatchCount": last_run.get("markerMismatchCount"), + "transportFailureCount": last_run.get("transportFailureCount"), + "actionsTaken": last_run.get("actionsTaken"), + "selection": last_run.get("selection"), + }, + "results": results, + "sentinelState": state_summary, + "valuesPrinted": False, + } + +def run_cleanup_probes(): + admin_email, token, admin_compliance = login() + cleanup = cleanup_probe_resources(token) + return { + "ok": cleanup.get("ok") is True, + "mode": "cleanup-probes", + "namespace": NAMESPACE, + "serviceDns": SERVICE_DNS, + "appPod": APP_POD, + "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, + "cleanup": cleanup, + "valuesPrinted": False, + } + +try: + if MODE == "sync": + result = run_sync() + elif MODE == "trace": + result = run_trace() + elif MODE == "cleanup-probes": + result = run_cleanup_probes() + elif MODE == "sentinel-probe": + result = run_sentinel_probe() + else: + result = run_validate() +except Exception as exc: + result = { + "ok": False, + "mode": MODE, + "namespace": NAMESPACE, + "serviceDns": SERVICE_DNS, + "appPod": globals().get("APP_POD"), + "error": str(exc), + "valuesPrinted": False, + } + +print(json.dumps(result, ensure_ascii=False, indent=2)) +sys.exit(0 if result.get("ok") else 1) +PY +`; diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sentinel.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sentinel.ts new file mode 100644 index 00000000..0073f922 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sentinel.ts @@ -0,0 +1,614 @@ +export const remotePythonSentinelScript = `def pool_api_key_secret_location(): + if RUNTIME_MODE == "host-docker": + return f"{HOST_DOCKER_ENV_PATH}.{POOL_API_KEY_SECRET_KEY}" + return f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}" + +def apply_sentinel_manifest(manifest): + if not TARGET_SENTINEL_ENABLED: + return { + "ok": True, + "action": "skipped-target-disabled", + "valuesPrinted": False, + } + if not isinstance(manifest, str) or not manifest.strip(): + return { + "ok": False, + "action": "missing-manifest", + "valuesPrinted": False, + } + proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest) + if proc.returncode != 0: + return { + "ok": False, + "action": "apply-failed", + "stdoutTail": text(proc.stdout, 2000), + "stderrTail": text(proc.stderr, 4000), + "valuesPrinted": False, + } + status = sentinel_runtime_status() + return { + "ok": status.get("ok") is True, + "action": "applied", + "stdoutTail": text(proc.stdout, 2000), + "runtime": status, + "valuesPrinted": False, + } + +def safe_kube_json(args, label): + proc = kubectl([*args, "-o", "json"]) + if proc.returncode != 0: + return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)} + try: + return json.loads(proc.stdout.decode("utf-8")), None + except Exception as exc: + return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)} + +def sentinel_runtime_status(): + if not TARGET_SENTINEL_ENABLED: + return { + "ok": True, + "action": "skipped-target-disabled", + "desired": { + "monitorEnabled": SENTINEL_CONFIG.get("monitor", {}).get("enabled"), + "actionsEnabled": SENTINEL_CONFIG.get("actions", {}).get("enabled"), + }, + "valuesPrinted": False, + } + cfg = SENTINEL_CONFIG + cronjob_name = cfg.get("cronJobName") + secret_name = cfg.get("credentialsSecretName") + configmap_name = cfg.get("configMapName") + state_name = cfg.get("stateConfigMapName") + cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}") + secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}") + configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}") + state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}") + state = None + if isinstance(state_cm, dict): + raw_state = (state_cm.get("data") or {}).get("state.json") + if isinstance(raw_state, str) and raw_state: + try: + state = json.loads(raw_state) + except Exception as exc: + state = {"parseError": str(exc)} + accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {} + quarantined = [] + recent_accounts = [] + for name, account_state in accounts.items(): + if not isinstance(account_state, dict): + continue + quarantine = account_state.get("quarantine") + if isinstance(quarantine, dict) and quarantine.get("active") is True: + quarantined.append({ + "accountName": name, + "until": quarantine.get("until"), + "applied": quarantine.get("applied"), + "reason": quarantine.get("reason"), + "failureKind": quarantine.get("failureKind"), + "errorDetails": quarantine.get("errorDetails"), + "intervalMinutes": quarantine.get("intervalMinutes"), + "sentinelProtect": quarantine.get("sentinelProtect"), + }) + last_probe = account_state.get("lastProbe") + if isinstance(last_probe, dict): + recent_accounts.append({ + "accountName": name, + "lastProbeAt": account_state.get("lastProbeAt"), + "lastStatus": account_state.get("lastStatus"), + "nextProbeAfter": account_state.get("nextProbeAfter"), + "ok": last_probe.get("ok"), + "purpose": last_probe.get("purpose"), + "httpStatus": last_probe.get("httpStatus"), + "durationMs": last_probe.get("durationMs"), + "markerMatched": last_probe.get("markerMatched"), + "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), + "outputHash": last_probe.get("outputHash"), + "outputPreview": last_probe.get("outputPreview"), + "responseBodyHash": last_probe.get("responseBodyHash"), + "responseBodyPreview": last_probe.get("responseBodyPreview"), + "error": last_probe.get("error"), + "errorDetails": last_probe.get("errorDetails"), + "usage": last_probe.get("usage"), + "failureKind": last_probe.get("failureKind"), + "requestShape": last_probe.get("requestShape"), + "action": last_probe.get("action"), + }) + recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "") + last_run = state.get("lastRun") if isinstance(state, dict) else None + cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {} + secret_data = secret.get("data") if isinstance(secret, dict) else {} + configmap_data = configmap.get("data") if isinstance(configmap, dict) else {} + ok = cronjob is not None and secret is not None and configmap is not None + return { + "ok": ok, + "desired": { + "monitorEnabled": cfg.get("monitor", {}).get("enabled"), + "actionsEnabled": cfg.get("actions", {}).get("enabled"), + "schedule": cfg.get("schedule"), + "cronJobName": cronjob_name, + "configMapName": configmap_name, + "credentialsSecretName": secret_name, + "stateConfigMapName": state_name, + }, + "cronJob": { + "exists": cronjob is not None, + "schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None, + "suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None, + "lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None, + "active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None, + "error": cronjob_error, + }, + "secret": { + "exists": secret is not None, + "profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data, + "valuesPrinted": False, + "error": secret_error, + }, + "configMap": { + "exists": configmap is not None, + "configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data, + "runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data, + "error": configmap_error, + }, + "state": { + "exists": state_cm is not None, + "accountCount": len(accounts), + "quarantinedCount": len(quarantined), + "quarantined": quarantined[-10:], + "recentAccounts": recent_accounts[-12:], + "lastRun": last_run, + "error": state_error, + }, + "valuesPrinted": False, + } + +def parse_epoch_z(value): + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except Exception: + return None + +def sentinel_state_object(): + if not TARGET_SENTINEL_ENABLED: + return None, None + state_name = SENTINEL_CONFIG.get("stateConfigMapName") + if not state_name: + return None, None + obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}") + if not isinstance(obj, dict): + return None, None + raw_state = (obj.get("data") or {}).get("state.json") + if not isinstance(raw_state, str) or not raw_state: + return obj, None + try: + return obj, json.loads(raw_state) + except Exception: + return obj, None + +def active_sentinel_quarantine_names(): + if not TARGET_SENTINEL_ENABLED: + return set() + _, state = sentinel_state_object() + if not isinstance(state, dict): + return set() + accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} + names = set() + for name, account_state in accounts_state.items(): + if not isinstance(name, str) or not isinstance(account_state, dict): + continue + quarantine = account_state.get("quarantine") + if isinstance(quarantine, dict) and quarantine.get("active") is True and quarantine.get("applied") is True: + names.add(name) + return names + +def default_sentinel_state(): + return {"version": 1, "accounts": {}, "ledger": {}, "history": []} + +def clamp_sentinel_freezes_for_config(state, now): + freeze_config = SENTINEL_CONFIG.get("freeze") if isinstance(SENTINEL_CONFIG.get("freeze"), dict) else {} + try: + max_interval = int(freeze_config.get("maxTtlMinutes") or 10) + except Exception: + max_interval = 10 + accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} + now_epoch = time.time() + items = [] + for name, account_state in accounts_state.items(): + if not isinstance(name, str) or not isinstance(account_state, dict): + continue + quarantine = account_state.get("quarantine") + if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True: + continue + try: + interval = int(quarantine.get("intervalMinutes") or 0) + except Exception: + interval = 0 + until_epoch = parse_epoch_z(quarantine.get("until")) + old_until = quarantine.get("until") + if interval <= max_interval and (until_epoch is None or until_epoch <= now_epoch + max_interval * 60): + continue + quarantine["previousIntervalMinutes"] = interval + quarantine["intervalMinutes"] = max_interval + quarantine["until"] = now + quarantine["clampedAt"] = now + quarantine["clampedBy"] = "sync-freeze-max-ttl" + account_state["nextProbeAfter"] = now + items.append({ + "accountName": name, + "previousIntervalMinutes": interval, + "maxIntervalMinutes": max_interval, + "previousUntil": old_until, + "nextProbeAfter": now, + }) + return items + +def parse_iso_epoch(value): + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except Exception: + return None + +def profile_success_max_interval(profile): + cadence = SENTINEL_CONFIG.get("cadence") if isinstance(SENTINEL_CONFIG.get("cadence"), dict) else {} + legacy = cadence.get("successMaxIntervalMinutes") + if legacy is None: + legacy = cadence.get("trustedSuccessMaxIntervalMinutes") or cadence.get("untrustedSuccessMaxIntervalMinutes") or 1 + if profile.get("trustUpstream") is True: + value = cadence.get("trustedSuccessMaxIntervalMinutes") or legacy + else: + value = cadence.get("untrustedSuccessMaxIntervalMinutes") or legacy + try: + return int(value) + except Exception: + return int(legacy) + +def clamp_sentinel_success_cadence_for_config(state, profiles, now): + accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} + profile_map = {item.get("accountName"): item for item in profiles if isinstance(item, dict) and isinstance(item.get("accountName"), str)} + now_epoch = time.time() + items = [] + for name, profile in profile_map.items(): + account_state = accounts_state.get(name) + if not isinstance(account_state, dict): + continue + quarantine = account_state.get("quarantine") + if isinstance(quarantine, dict) and quarantine.get("active") is True: + account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} + account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile) + continue + try: + interval = int(account_state.get("successIntervalMinutes") or 0) + except Exception: + interval = 0 + next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter")) + max_interval = profile_success_max_interval(profile) + account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} + account_state["successMaxIntervalMinutes"] = max_interval + if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60): + continue + old_next = account_state.get("nextProbeAfter") + account_state["previousSuccessIntervalMinutes"] = interval + account_state["successIntervalMinutes"] = min(interval, max_interval) if interval > 0 else interval + account_state["nextProbeAfter"] = now + account_state["cadenceClampedAt"] = now + account_state["cadenceClampedBy"] = "sync-success-max-interval" + items.append({ + "accountName": name, + "trustUpstream": profile.get("trustUpstream") is True, + "previousSuccessIntervalMinutes": interval, + "maxIntervalMinutes": max_interval, + "previousNextProbeAfter": old_next, + "nextProbeAfter": now, + }) + return items + +def update_sentinel_state_configmap(obj, state): + state_name = SENTINEL_CONFIG.get("stateConfigMapName") + if not state_name: + return {"ok": False, "reason": "state-configmap-missing"} + state_json = json.dumps(state, ensure_ascii=False, indent=2) + manifest = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": state_name, + "namespace": NAMESPACE, + "labels": { + "app.kubernetes.io/name": SERVICE_NAME, + "app.kubernetes.io/component": "account-sentinel", + "app.kubernetes.io/managed-by": "unidesk-platform-infra", + }, + }, + "data": {"state.json": state_json}, + } + proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) + action = "applied" if isinstance(obj, dict) else "created" + if proc.returncode != 0: + return {"ok": False, "reason": f"{action}-failed", "stderrTail": text(proc.stderr, 2000)} + return {"ok": True, "action": action} + +def ensure_sentinel_state_for_sync(account_results, pending_only=False): + if not sentinel_quality_gate_enabled(): + return {"ok": True, "skipped": True, "reason": "sentinel-quality-gate-disabled", "changedCount": 0, "items": [], "valuesPrinted": False} + state_obj, state = sentinel_state_object() + if not isinstance(state, dict): + state = default_sentinel_state() + state.setdefault("version", 1) + accounts_state = state.setdefault("accounts", {}) + if not isinstance(accounts_state, dict): + accounts_state = {} + state["accounts"] = accounts_state + now = utc_iso() + items = [] + clamped_items = [] if pending_only else clamp_sentinel_freezes_for_config(state, now) + cadence_clamped_items = [] if pending_only else clamp_sentinel_success_cadence_for_config(state, [item.get("profileConfig") for item in account_results if isinstance(item.get("profileConfig"), dict)], now) + changed_count = 0 + fingerprint_only_count = 0 + for item in account_results: + name = item.get("accountName") + if not isinstance(name, str) or not name: + continue + account_state = accounts_state.setdefault(name, {}) + if not isinstance(account_state, dict): + account_state = {} + accounts_state[name] = account_state + fingerprint_value = item.get("sentinelProbeConfigFingerprint") + if isinstance(fingerprint_value, str) and fingerprint_value: + account_state["probeConfigFingerprint"] = fingerprint_value + if item.get("sentinelProbeRequired") is not True: + fingerprint_only_count += 1 + continue + changed_count += 1 + reasons = item.get("sentinelChangeReasons") if isinstance(item.get("sentinelChangeReasons"), list) else [] + quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else None + if not (isinstance(quarantine, dict) and quarantine.get("active") is True): + account_state["quarantine"] = { + "active": False, + "reason": "yaml-account-change-pending-sentinel-probe", + "lastPendingAt": now, + "changeReasons": reasons, + } + account_state["nextProbeAfter"] = now + account_state["successStreak"] = 0 + account_state["successIntervalMinutes"] = 0 + profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {} + account_state["trustUpstream"] = profile_config.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False} + account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config) + account_state["lastStatus"] = "pending-sentinel-quality-gate" + account_state["qualityGate"] = { + "pending": True, + "reason": "yaml-account-change", + "changeReasons": reasons, + "markedAt": now, + "pendingOnly": pending_only, + "defaultFrozen": False, + } + items.append({"accountName": name, "changeReasons": reasons, "nextProbeAfter": now, "defaultFrozen": False, "defaultSchedulable": True, "pendingOnly": pending_only}) + if changed_count <= 0 and len(clamped_items) <= 0 and len(cadence_clamped_items) <= 0: + return {"ok": True, "skipped": False, "reason": "no-new-or-changed-accounts", "changedCount": 0, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": 0, "cadenceClampedCount": 0, "items": [], "valuesPrinted": False} + update = update_sentinel_state_configmap(state_obj, state) + if pending_only and changed_count > 0: + reason = "new-or-changed-accounts-pending-probe-prepared-default-schedulable" + elif changed_count > 0 and (len(clamped_items) > 0 or len(cadence_clamped_items) > 0): + reason = "new-or-changed-accounts-default-schedulable-and-sentinel-cadence-clamped" + elif changed_count > 0: + reason = "new-or-changed-accounts-default-schedulable" + elif len(cadence_clamped_items) > 0: + reason = "success-cadence-clamped-to-current-config" + else: + reason = "freeze-backoff-clamped-to-current-config" + return { + "ok": update.get("ok") is True, + "skipped": False, + "reason": reason, + "changedCount": changed_count, + "fingerprintOnlyCount": fingerprint_only_count, + "clampedCount": len(clamped_items), + "cadenceClampedCount": len(cadence_clamped_items), + "pendingOnly": pending_only, + "items": items, + "clampedItems": clamped_items, + "cadenceClampedItems": cadence_clamped_items, + "update": update, + "valuesPrinted": False, + } + +def sentinel_state_summary(): + _, state = sentinel_state_object() + if not isinstance(state, dict): + return {"exists": False, "valuesPrinted": False} + accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} + quarantined = [] + recent_accounts = [] + for name, account_state in accounts.items(): + if not isinstance(account_state, dict): + continue + quarantine = account_state.get("quarantine") + if isinstance(quarantine, dict) and quarantine.get("active") is True: + quarantined.append({ + "accountName": name, + "until": quarantine.get("until"), + "applied": quarantine.get("applied"), + "reason": quarantine.get("reason"), + "failureKind": quarantine.get("failureKind"), + "errorDetails": quarantine.get("errorDetails"), + "intervalMinutes": quarantine.get("intervalMinutes"), + "sentinelProtect": quarantine.get("sentinelProtect"), + }) + last_probe = account_state.get("lastProbe") + if isinstance(last_probe, dict): + recent_accounts.append({ + "accountName": name, + "lastProbeAt": account_state.get("lastProbeAt"), + "lastStatus": account_state.get("lastStatus"), + "nextProbeAfter": account_state.get("nextProbeAfter"), + "ok": last_probe.get("ok"), + "purpose": last_probe.get("purpose"), + "httpStatus": last_probe.get("httpStatus"), + "durationMs": last_probe.get("durationMs"), + "markerMatched": last_probe.get("markerMatched"), + "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), + "usage": last_probe.get("usage"), + "responseBodyHash": last_probe.get("responseBodyHash"), + "responseBodyPreview": last_probe.get("responseBodyPreview"), + "error": last_probe.get("error"), + "errorDetails": last_probe.get("errorDetails"), + "failureKind": last_probe.get("failureKind"), + "requestShape": last_probe.get("requestShape"), + "action": last_probe.get("action"), + }) + recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "") + return { + "exists": True, + "accountCount": len(accounts), + "quarantinedCount": len(quarantined), + "quarantined": quarantined[-10:], + "recentAccounts": recent_accounts[-12:], + "lastRun": state.get("lastRun"), + "valuesPrinted": False, + } + +def reassert_sentinel_freezes_after_sync(token): + if not TARGET_SENTINEL_ENABLED: + return {"ok": True, "skipped": True, "reason": "target-disabled", "items": [], "valuesPrinted": False} + if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True: + return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False} + _, state = sentinel_state_object() + if not isinstance(state, dict): + return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False} + accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} + accounts = list_accounts(token) + by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} + items = [] + for name, account_state in accounts_state.items(): + if not isinstance(account_state, dict): + continue + quarantine = account_state.get("quarantine") + if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True: + continue + account = by_name.get(name) + if not account or account.get("id") is None: + items.append({"accountName": name, "ok": False, "reason": "account-not-found"}) + continue + try: + ensure_success( + curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}), + f"reassert sentinel freeze for {name}", + ) + items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")}) + except Exception as exc: + items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)}) + return { + "ok": all(item.get("ok") is True for item in items), + "skipped": False, + "items": items, + "valuesPrinted": False, + } + +def list_user_keys(token): + data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys") + return extract_items(data) + +def ensure_sub2api_api_key(token, api_key, group_id): + keys = list_user_keys(token) + existing = next((item for item in keys if item.get("key") == api_key), None) + action = "kept-existing" + if existing is None: + existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None) + if existing is None or existing.get("key") != api_key: + payload = { + "name": POOL_API_KEY_NAME, + "group_id": group_id, + "custom_key": api_key, + "quota": 0, + "rate_limit_5h": 0, + "rate_limit_1d": 0, + "rate_limit_7d": 0, + } + created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key") + existing = created if isinstance(created, dict) else existing + action = "created" + elif existing.get("id") is not None and (existing.get("group_id") != group_id or existing.get("name") != POOL_API_KEY_NAME): + updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group") + existing = updated if isinstance(updated, dict) else existing + action = "updated-name-group" + return { + "action": action, + "id": existing.get("id") if isinstance(existing, dict) else None, + "name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME, + "groupId": existing.get("group_id") if isinstance(existing, dict) else group_id, + "userId": existing.get("user_id") if isinstance(existing, dict) else None, + } + +def get_admin_user(token, user_id): + data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner") + if not isinstance(data, dict): + raise RuntimeError("API key owner response is not an object") + return data + +def ensure_pool_owner_balance(token, user_id): + user = get_admin_user(token, user_id) + current = float(user.get("balance") or 0) + if current >= MIN_OWNER_BALANCE_USD: + return { + "action": "kept-existing", + "userId": user_id, + "balanceBefore": current, + "balanceAfter": current, + "minimumBalanceUsd": MIN_OWNER_BALANCE_USD, + } + updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={ + "balance": MIN_OWNER_BALANCE_USD, + "operation": "set", + "notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.", + }), "set API key owner balance") + after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD + return { + "action": "set", + "userId": user_id, + "balanceBefore": current, + "balanceAfter": after, + "minimumBalanceUsd": MIN_OWNER_BALANCE_USD, + } + +def ensure_pool_owner_concurrency(token, user_id): + user = get_admin_user(token, user_id) + try: + current = int(user.get("concurrency") or 0) + except Exception: + current = 0 + if current >= MIN_OWNER_CONCURRENCY: + return { + "ok": True, + "action": "kept-existing", + "userId": user_id, + "concurrencyBefore": current, + "concurrencyAfter": current, + "minimumConcurrency": MIN_OWNER_CONCURRENCY, + "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, + } + updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={ + "concurrency": MIN_OWNER_CONCURRENCY, + }), "set API key owner concurrency") + try: + after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY + except Exception: + after = MIN_OWNER_CONCURRENCY + return { + "ok": after >= MIN_OWNER_CONCURRENCY, + "action": "set", + "userId": user_id, + "concurrencyBefore": current, + "concurrencyAfter": after, + "minimumConcurrency": MIN_OWNER_CONCURRENCY, + "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, + } + +`; diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync-validate.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync-validate.ts new file mode 100644 index 00000000..7f16afc4 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync-validate.ts @@ -0,0 +1,142 @@ +export const remotePythonSyncValidateScript = `def run_sync(): + global MANUAL_ACCOUNT_PROTECTIONS + payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) + manual_accounts_payload = payload.get("manualAccounts") if isinstance(payload.get("manualAccounts"), dict) else {} + resolved_manual_protections = manual_accounts_payload.get("protected") + if not isinstance(resolved_manual_protections, list): + raise RuntimeError("sync payload has no manualAccounts.protected binding plan") + MANUAL_ACCOUNT_PROTECTIONS = resolved_manual_protections + profiles = payload.get("profiles") or [] + prune_removed = bool(payload.get("pruneRemoved")) + sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {} + if not profiles and not MANUAL_ACCOUNT_PROTECTIONS: + raise RuntimeError("sync payload has no profiles and no manualAccounts.protected binding plan") + admin_email, token, admin_compliance = login() + group, group_action = ensure_group(token) + group_id = group.get("id") if isinstance(group, dict) else None + if group_id is None: + raise RuntimeError("pool group id missing after ensure") + existing_accounts = list_accounts(token) + planned_account_results = planned_sentinel_account_results(profiles, existing_accounts) + sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True) + if sentinel_quality_prepare.get("ok") is not True: + raise RuntimeError("prepare sentinel pending probe failed: " + json.dumps(sentinel_quality_prepare, ensure_ascii=False)) + protected_frozen_names = active_sentinel_quarantine_names() + account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed, protected_frozen_names, existing_accounts) + manual_account_proxy_bindings = ensure_manual_account_proxy_bindings(token) + manual_account_group_bindings = ensure_manual_account_group_bindings(token, group_id) + manual_account_protections = manual_account_protection_status(token, group_id) + capacity_status = account_capacity_status(token) + load_factor_status = account_load_factor_status(token) + ws_v2_status = account_ws_v2_status(token) + temp_unschedulable_status = account_temp_unschedulable_status(token) + api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id, token) + api_key_result = ensure_sub2api_api_key(token, api_key, group_id) + owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"]) + owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"]) + gateway = validate_gateway(api_key) + responses_smoke = validate_gateway_responses(api_key) + compact_evidence = recent_compact_gateway_evidence() + responses_evidence = recent_responses_gateway_evidence() + runtime_capabilities = validate_runtime_capabilities(token) + sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest")) + sentinel_quality = ensure_sentinel_state_for_sync(account_results) + sentinel_reassert = reassert_sentinel_freezes_after_sync(token) + return { + "ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_proxy_bindings.get("ok") is True and manual_account_group_bindings.get("ok") is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and sentinel_quality_prepare.get("ok") is True and sentinel_quality.get("ok") is True and sentinel_reassert.get("ok") is True and runtime_capabilities.get("ok") is True, + "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True, + "mode": "sync", + "namespace": NAMESPACE, + "serviceDns": SERVICE_DNS, + "appPod": APP_POD, + "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, + "pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"}, + "accounts": { + "desired": len(profiles), + "created": sum(1 for item in account_results if item["action"] == "created"), + "updated": sum(1 for item in account_results if item["action"] == "updated"), + "pruned": len(pruned_account_results), + "pruneMode": "explicit" if prune_removed else "disabled-by-default", + "items": account_results, + "prunedItems": pruned_account_results, + "processControl": {"schedulableRestore": "sentinel marker probe only; sync does not restore schedulable for existing accounts", "durableConfig": False}, + "valuesPrinted": False, + }, + "manualAccounts": {**manual_account_protections, "proxySync": manual_account_proxy_bindings, "groupSync": manual_account_group_bindings}, + "capacity": capacity_status, + "loadFactor": load_factor_status, + "webSocketsV2": ws_v2_status, + "tempUnschedulable": temp_unschedulable_status, + "apiKey": { + "ok": True, + "name": POOL_API_KEY_NAME, + "secret": pool_api_key_secret_location(), + "secretAction": secret_action, + "secretApply": secret_apply_stdout, + "sub2apiAction": api_key_result["action"], + "sub2apiId": api_key_result["id"], + "groupId": api_key_result["groupId"], + "userId": api_key_result["userId"], + "apiKeyFingerprint": secret_fingerprint(api_key), + "valuesPrinted": False, + }, + "ownerBalance": owner_balance, + "ownerConcurrency": owner_concurrency, + "sentinel": {**sentinel, "qualityGatePrepare": sentinel_quality_prepare, "qualityGate": sentinel_quality, "freezeReassert": sentinel_reassert}, + "runtimeCapabilities": runtime_capabilities, + "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, + } + +def run_validate(): + admin_email, token, admin_compliance = login() + api_key, key_item, api_key_source = resolve_pool_api_key_for_validate(token) + api_key_ok = isinstance(key_item, dict) and key_item.get("name") == POOL_API_KEY_NAME + owner_balance = None + owner_concurrency = None + if key_item is not None and key_item.get("user_id") is not None: + owner_balance = ensure_pool_owner_balance(token, key_item["user_id"]) + owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"]) + capacity_status = account_capacity_status(token) + load_factor_status = account_load_factor_status(token) + ws_v2_status = account_ws_v2_status(token) + temp_unschedulable_status = account_temp_unschedulable_status(token) + pool_group_id = key_item.get("group_id") if isinstance(key_item, dict) else None + manual_account_protections = manual_account_protection_status(token, pool_group_id) + gateway = validate_gateway(api_key) + responses_smoke = validate_gateway_responses(api_key) + compact_evidence = recent_compact_gateway_evidence() + responses_evidence = recent_responses_gateway_evidence() + runtime_capabilities = validate_runtime_capabilities(token) + sentinel = sentinel_runtime_status() + return { + "ok": api_key_ok is True and gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and runtime_capabilities.get("ok") is True, + "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True, + "mode": "validate", + "namespace": NAMESPACE, + "serviceDns": SERVICE_DNS, + "appPod": APP_POD, + "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, + "apiKey": { + "ok": api_key_ok, + "name": POOL_API_KEY_NAME, + "secret": pool_api_key_secret_location(), + "source": api_key_source, + "sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None, + "userId": key_item.get("user_id") if isinstance(key_item, dict) else None, + "groupId": key_item.get("group_id") if isinstance(key_item, dict) else None, + "apiKeyFingerprint": secret_fingerprint(api_key), + "valuesPrinted": False, + }, + "ownerBalance": owner_balance, + "ownerConcurrency": owner_concurrency, + "capacity": capacity_status, + "loadFactor": load_factor_status, + "webSocketsV2": ws_v2_status, + "tempUnschedulable": temp_unschedulable_status, + "manualAccounts": manual_account_protections, + "sentinel": sentinel, + "runtimeCapabilities": runtime_capabilities, + "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, + } + +`; diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts index 4facf494..5f8766ec 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts @@ -1,3313 +1,20 @@ // SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote-python-sync module for scripts/src/platform-infra-sub2api-codex.ts. -// Moved mechanically from scripts/src/platform-infra-sub2api-codex.ts:4265-5400 for #903. - -import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { join } from "node:path"; -import type { UniDeskConfig } from "../config"; -import { rootPath } from "../config"; -import type { RenderedCliResult } from "../output"; -import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "../platform-infra-public-service"; -import { shortSha256Fingerprint } from "../platform-infra-ops-library"; -import { - codexPoolSentinelSummary, - codexPoolSentinelRuntimeImage, - readCodexPoolSentinelConfig, - renderCodexPoolSentinelManifest, - type CodexPoolSentinelConfig, - type CodexPoolSentinelProfileSecret, -} from "../platform-infra-sub2api-codex-sentinel"; -import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets"; -import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; - import type { CodexPoolConfig, CodexPoolRuntimeTarget } from "./types"; -import { desiredAccountCapacityMap, desiredAccountLoadFactorMap, desiredAccountTempUnschedulableMap, desiredAccountWebSocketsV2ModeMap, desiredDefaultAccountTempUnschedulable } from "./accounts"; -import { resolvedManualAccountProtections } from "./public-exposure"; -import { fieldManager } from "./types"; - -function pyJson(value: unknown): string { - return `json.loads(${JSON.stringify(JSON.stringify(value))})`; -} +import { remotePythonCoreScript } from "./remote-python-core"; +import { remotePythonSentinelScript } from "./remote-python-sentinel"; +import { remotePythonValidationScript } from "./remote-python-validation"; +import { remotePythonSyncValidateScript } from "./remote-python-sync-validate"; +import { remotePythonTraceScript } from "./remote-python-trace"; +import { remotePythonSentinelProbeScript } from "./remote-python-sentinel-probe"; export function remotePythonScript(mode: "sync" | "validate" | "trace" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { - const hostDockerEnvPath = target.runtimeMode === "host-docker" ? target.hostDockerEnvPath : null; - return ` -set -u -python3 - <<'PY' -import base64 -import hashlib -import json -import os -import re -import secrets -import string -import subprocess -import sys -import time -from datetime import datetime, timezone, timedelta -from urllib.parse import quote - -TARGET_ID = ${pyJson(target.id)} -RUNTIME_MODE = ${pyJson(target.runtimeMode)} -NAMESPACE = ${pyJson(target.namespace)} -SERVICE_NAME = ${pyJson(target.serviceName)} -SERVICE_DNS = ${pyJson(target.serviceDns)} -HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} -HOST_DOCKER_ENV_PATH = ${pyJson(hostDockerEnvPath)} -HOST_DOCKER_APP_CONTAINER = "sub2api-app" -FIELD_MANAGER = "${fieldManager}" -APP_SECRET_NAME = ${pyJson(target.appSecretName)} -POOL_GROUP_NAME = "${pool.groupName}" -POOL_GROUP_DESCRIPTION = ${pyJson(pool.groupDescription)} -POOL_API_KEY_NAME = "${pool.apiKeyName}" -POOL_API_KEY_SECRET_NAME = "${pool.apiKeySecretName}" -POOL_API_KEY_SECRET_KEY = "${pool.apiKeySecretKey}" -POOL_ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)} -MIN_OWNER_BALANCE_USD = ${pyJson(pool.minOwnerBalanceUsd)} -MIN_OWNER_CONCURRENCY = ${pyJson(pool.minOwnerConcurrency)} -MIN_OWNER_CONCURRENCY_SOURCE = ${pyJson(pool.minOwnerConcurrencySource)} -POOL_DEFAULT_ACCOUNT_PRIORITY = ${pyJson(pool.defaultAccountPriority)} -POOL_DEFAULT_ACCOUNT_CAPACITY = ${pyJson(pool.defaultAccountCapacity)} -POOL_DEFAULT_ACCOUNT_LOAD_FACTOR = ${pyJson(pool.defaultAccountLoadFactor)} -RESPONSES_SMOKE_MODEL = ${pyJson(pool.localCodex.responsesSmokeModel)} -EXPECTED_ACCOUNT_CAPACITIES = ${pyJson(desiredAccountCapacityMap(pool))} -EXPECTED_ACCOUNT_LOAD_FACTORS = ${pyJson(desiredAccountLoadFactorMap(pool))} -EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))}) -EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))}) -EXPECTED_DEFAULT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredDefaultAccountTempUnschedulable(pool)))}) -MANUAL_ACCOUNT_PROTECTIONS = json.loads(${JSON.stringify(JSON.stringify(resolvedManualAccountProtections(pool, target)))}) -SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))}) -TARGET_EGRESS_PROXY = json.loads(${JSON.stringify(JSON.stringify(target.egressProxy))}) -TARGET_SENTINEL_ENABLED = ${target.sentinelEnabled ? "True" : "False"} -MODE = "${mode}" -PAYLOAD_B64 = "${encodedPayload}" - -def run(cmd, input_bytes=None): - return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - -def text(data, limit=4000): - if isinstance(data, bytes): - data = data.decode("utf-8", errors="replace") - return data[-limit:] - -def read_host_env(): - if RUNTIME_MODE != "host-docker": - return {} - if not isinstance(HOST_DOCKER_ENV_PATH, str) or not HOST_DOCKER_ENV_PATH: - raise RuntimeError("host-docker env source path missing") - values = {} - lines = read_host_env_lines() - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - continue - key, value = stripped.split("=", 1) - key = key.strip() - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - if key: - values[key] = value - return values - -def read_host_env_lines(): - try: - with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle: - return handle.read().splitlines() - except FileNotFoundError: - raise RuntimeError(f"host-docker env source missing: {HOST_DOCKER_ENV_PATH}") - except PermissionError: - proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH]) - if proc.returncode != 0: - raise RuntimeError("read host-docker env source failed: " + text(proc.stderr, 1000)) - return proc.stdout.decode("utf-8", errors="replace").splitlines() - -def write_host_env_value(key, value): - if RUNTIME_MODE != "host-docker": - raise RuntimeError("write_host_env_value is only valid for host-docker") - if not isinstance(HOST_DOCKER_ENV_PATH, str) or not HOST_DOCKER_ENV_PATH: - raise RuntimeError("host-docker env source path missing") - if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): - raise RuntimeError(f"unsupported env key: {key}") - os.makedirs(os.path.dirname(HOST_DOCKER_ENV_PATH), exist_ok=True) - try: - lines = read_host_env_lines() - except RuntimeError as exc: - if "missing" not in str(exc): - raise - lines = [] - next_lines = [] - replaced = False - for line in lines: - stripped = line.strip() - if stripped.startswith("#") or "=" not in stripped: - next_lines.append(line) - continue - current_key = stripped.split("=", 1)[0].strip() - if current_key == key: - next_lines.append(f"{key}={value}") - replaced = True - else: - next_lines.append(line) - if not replaced: - next_lines.append(f"{key}={value}") - content = "\\n".join(next_lines).rstrip() + "\\n" - tmp_path = HOST_DOCKER_ENV_PATH + ".tmp" - try: - with open(tmp_path, "w", encoding="utf-8") as handle: - handle.write(content) - os.chmod(tmp_path, 0o600) - os.replace(tmp_path, HOST_DOCKER_ENV_PATH) - except PermissionError: - try: - os.unlink(tmp_path) - except Exception: - pass - script = r''' -set -eu -path="$1" -dir="$(dirname "$path")" -mkdir -p "$dir" -tmp="$path.tmp.$$" -umask 077 -cat > "$tmp" -mv "$tmp" "$path" -chmod 600 "$path" -''' - proc = run(["sudo", "-n", "sh", "-c", script, "sh", HOST_DOCKER_ENV_PATH], content.encode("utf-8")) - if proc.returncode != 0: - raise RuntimeError("write host-docker env source failed: " + text(proc.stderr, 1000)) - return "updated" if replaced else "created" - -def docker(args): - proc = run(["docker", *args]) - if proc.returncode == 0: - return proc - sudo_proc = run(["sudo", "-n", "docker", *args]) - return sudo_proc if sudo_proc.returncode == 0 else proc - -def runtime_logs(since, tail): - if RUNTIME_MODE == "host-docker": - return docker(["logs", f"--since={since}", f"--tail={tail}", HOST_DOCKER_APP_CONTAINER]) - return kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) - -def kubectl(args, input_obj=None): - if isinstance(input_obj, str): - input_bytes = input_obj.encode("utf-8") - else: - input_bytes = input_obj - return run(["kubectl", *args], input_bytes) - -def require_kubectl(args, input_obj=None, label="kubectl"): - proc = kubectl(args, input_obj) - if proc.returncode != 0: - raise RuntimeError(f"{label} failed: {text(proc.stderr, 1000)}") - return proc.stdout - -def kube_json(args, label): - raw = require_kubectl([*args, "-o", "json"], label=label) - return json.loads(raw.decode("utf-8")) - -def decode_secret_value(name, key): - if RUNTIME_MODE == "host-docker": - return read_host_env().get(key) - data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {} - if key not in data: - return None - return base64.b64decode(data[key]).decode("utf-8") - -def get_config_value(name, key): - if RUNTIME_MODE == "host-docker": - return read_host_env().get(key) - data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {} - value = data.get(key) - return value if isinstance(value, str) and value else None - -def select_app_pod(): - if RUNTIME_MODE == "host-docker": - return HOST_DOCKER_APP_CONTAINER - pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or [] - for pod in pods: - status = pod.get("status") or {} - if status.get("phase") != "Running": - continue - statuses = status.get("containerStatuses") or [] - if statuses and all(item.get("ready") is True for item in statuses): - return pod["metadata"]["name"] - if pods: - return pods[0]["metadata"]["name"] - raise RuntimeError("sub2api app pod not found") - -APP_POD = select_app_pod() - -def parse_curl_output(proc): - stdout = proc.stdout.decode("utf-8", errors="replace") - marker = "\\n__HTTP_CODE__:" - pos = stdout.rfind(marker) - if pos < 0: - return { - "ok": False, - "httpStatus": 0, - "json": None, - "body": stdout, - "stderr": text(proc.stderr, 1000), - "transportExitCode": proc.returncode, - } - body = stdout[:pos] - status_text = stdout[pos + len(marker):].strip() - try: - http_status = int(status_text[-3:]) - except ValueError: - http_status = 0 - try: - parsed = json.loads(body) if body.strip() else None - except json.JSONDecodeError: - parsed = None - return { - "ok": proc.returncode == 0 and 200 <= http_status < 300, - "httpStatus": http_status, - "json": parsed, - "body": body, - "stderr": text(proc.stderr, 1000), - "transportExitCode": proc.returncode, - } - -def utc_iso(offset_seconds=0): - return (datetime.now(timezone.utc) + timedelta(seconds=offset_seconds)).strftime("%Y-%m-%dT%H:%M:%SZ") - -def normalize_runtime_base_url(value): - if not isinstance(value, str): - return None - value = value.strip().rstrip("/") - return value or None - -def empty_to_none(value): - return value if isinstance(value, str) and value else None - -def sentinel_quality_gate_enabled(): - return TARGET_SENTINEL_ENABLED and (SENTINEL_CONFIG.get("monitor") or {}).get("enabled") is True and (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is True - -def account_notes_fingerprint(account): - notes = account.get("notes") if isinstance(account, dict) else None - if not isinstance(notes, str): - return None - match = re.search(r"fingerprint=([A-Za-z0-9_-]+)", notes) - return match.group(1) if match else None - -def runtime_account_credentials(account): - credentials = account.get("credentials") if isinstance(account, dict) and isinstance(account.get("credentials"), dict) else {} - return credentials - -def runtime_account_extra(account): - extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {} - return extra - -def sentinel_probe_change_reasons(current, profile): - if not isinstance(current, dict) or current.get("id") is None: - return ["created"] - credentials = runtime_account_credentials(current) - extra = runtime_account_extra(current) - expected_base_url = normalize_runtime_base_url(profile.get("baseUrl")) - runtime_base_url = normalize_runtime_base_url(credentials.get("base_url")) - expected_user_agent = empty_to_none(profile.get("upstreamUserAgent")) - runtime_user_agent = empty_to_none(credentials.get("user_agent")) - expected_ws_mode = empty_to_none(profile.get("openaiResponsesWebSocketsV2Mode")) - runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode")) - expected_trust_upstream = profile.get("trustUpstream") is True - runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True - expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} - runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False} - reasons = [] - if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"): - reasons.append("profile") - if runtime_base_url != expected_base_url: - reasons.append("base-url") - if account_notes_fingerprint(current) != profile.get("apiKeyFingerprint"): - reasons.append("api-key-fingerprint") - if runtime_user_agent != expected_user_agent: - reasons.append("upstream-user-agent") - if runtime_ws_mode != expected_ws_mode: - reasons.append("responses-websockets-v2-mode") - if runtime_trust_upstream != expected_trust_upstream: - reasons.append("trust-upstream") - if runtime_protect != expected_protect: - reasons.append("sentinel-protect") - return reasons - -def curl_api(method, path, bearer=None, payload=None): - body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") - script = r''' -set -eu -method="$1" -url="$2" -token="\${3:-}" -tmp="$(mktemp)" -trap 'rm -f "$tmp"' EXIT -cat > "$tmp" -if [ -n "$token" ]; then - if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then - curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url" - else - curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url" - fi -else - if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then - curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" "$url" - else - curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" - fi -fi -''' - if RUNTIME_MODE == "host-docker": - if not isinstance(HOST_DOCKER_APP_PORT, int): - raise RuntimeError("host-docker app port missing") - proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body) - else: - proc = run([ - "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, - "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or "", - ], body) - return parse_curl_output(proc) - -def envelope_data(parsed): - if isinstance(parsed, dict) and "data" in parsed: - return parsed.get("data") - return parsed - -def ensure_success(resp, label): - parsed = resp.get("json") - code = parsed.get("code") if isinstance(parsed, dict) else None - if not resp.get("ok") or (code is not None and code != 0): - message = parsed.get("message") if isinstance(parsed, dict) else text(resp.get("body", ""), 500) - raise RuntimeError(f"{label} failed: http={resp.get('httpStatus')} message={message}") - return envelope_data(parsed) - -def ensure_admin_compliance(token): - status_resp = curl_api("GET", "/api/v1/admin/compliance", bearer=token) - if status_resp.get("httpStatus") == 404: - return { - "ok": True, - "action": "not-supported-by-runtime", - "valuesPrinted": False, - } - status = ensure_success(status_resp, "get admin compliance") - if not isinstance(status, dict): - return { - "ok": True, - "action": "status-unstructured", - "valuesPrinted": False, - } - version = status.get("version") - required = status.get("required") is True - if not required: - return { - "ok": True, - "action": "already-acknowledged", - "version": version, - "requiredBefore": False, - "requiredAfter": False, - "phrasePrinted": False, - "valuesPrinted": False, - } - phrase = status.get("ack_phrase_zh") if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else status.get("ack_phrase_en") - language = "zh" if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else "en" - if not isinstance(phrase, str) or not phrase: - raise RuntimeError("admin compliance acknowledgement phrase missing") - accepted = ensure_success( - curl_api("POST", "/api/v1/admin/compliance/accept", bearer=token, payload={"phrase": phrase, "language": language}), - "accept admin compliance", - ) - required_after = accepted.get("required") if isinstance(accepted, dict) else None - return { - "ok": required_after is False, - "action": "accepted", - "version": version, - "requiredBefore": True, - "requiredAfter": required_after, - "phrasePrinted": False, - "valuesPrinted": False, - } - -def extract_items(data): - if isinstance(data, list): - return data - if isinstance(data, dict): - if isinstance(data.get("items"), list): - return data["items"] - for key in ("groups", "accounts", "api_keys", "keys"): - if isinstance(data.get(key), list): - return data[key] - return [] - -def find_access_token(data): - if isinstance(data, dict): - for key in ("access_token", "token"): - if isinstance(data.get(key), str) and data[key]: - return data[key] - for value in data.values(): - token = find_access_token(value) - if token: - return token - return None - -def login(): - admin_email = get_config_value("sub2api-config", "ADMIN_EMAIL") or POOL_ADMIN_EMAIL_DEFAULT - admin_password = decode_secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") - if not admin_password: - raise RuntimeError("ADMIN_PASSWORD missing from sub2api-secrets") - data = ensure_success(curl_api("POST", "/api/v1/auth/login", payload={"email": admin_email, "password": admin_password}), "admin login") - token = find_access_token(data) - if not token: - raise RuntimeError("admin login response did not contain access_token") - compliance = ensure_admin_compliance(token) - if compliance.get("ok") is not True: - raise RuntimeError("admin compliance acknowledgement failed") - return admin_email, token, compliance - -def group_payload(): - return { - "name": POOL_GROUP_NAME, - "description": POOL_GROUP_DESCRIPTION, - "platform": "openai", - "rate_multiplier": 1, - "is_exclusive": False, - "subscription_type": "standard", - "allow_messages_dispatch": True, - "require_oauth_only": False, - "require_privacy_set": False, - "rpm_limit": 0, - } - -def list_groups(token): - data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups") - return extract_items(data) - -def ensure_group(token): - existing = next((item for item in list_groups(token) if item.get("name") == POOL_GROUP_NAME), None) - payload = group_payload() - if existing is None: - created = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=payload), "create group") - return created, "created" - group_id = existing.get("id") - if group_id is None: - raise RuntimeError("existing group has no id") - payload["status"] = "active" - updated = ensure_success(curl_api("PUT", f"/api/v1/admin/groups/{group_id}", bearer=token, payload=payload), "update group") - return updated if isinstance(updated, dict) else existing, "updated" - -def list_accounts_for_group(token, group_id): - path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500&platform=openai" - data = ensure_success(curl_api("GET", path, bearer=token), f"list accounts for group {group_id}") - return extract_items(data) - -def account_group_ids(token, account): - if not isinstance(account, dict) or account.get("id") is None: - return [] - account_id = account.get("id") - account_name = account.get("name") - ids = [] - for group in list_groups(token): - group_id = group.get("id") if isinstance(group, dict) else None - if group_id is None: - continue - members = list_accounts_for_group(token, group_id) - if any(item.get("id") == account_id or item.get("name") == account_name for item in members if isinstance(item, dict)): - ids.append(group_id) - return sorted(set(ids)) - -def list_accounts(token): - path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-codex-") - data = ensure_success(curl_api("GET", path, bearer=token), "list accounts") - return extract_items(data) - -def find_account_by_name(token, name): - path = "/api/v1/admin/accounts?page=1&page_size=50&platform=openai&search=" + quote(str(name)) - data = ensure_success(curl_api("GET", path, bearer=token), "find account " + str(name)) - for item in extract_items(data): - if isinstance(item, dict) and item.get("name") == name: - return item - return None - -def list_proxy_candidates(token, search=""): - path = "/api/v1/admin/proxies?page=1&page_size=200" - if search: - path += "&search=" + quote(str(search)) - data = ensure_success(curl_api("GET", path, bearer=token), "list proxies") - return extract_items(data) - -def find_proxy_by_name(token, name): - for item in list_proxy_candidates(token, name): - if isinstance(item, dict) and item.get("name") == name: - return item - return None - -def manual_binding_plan(binding, expected_kind): - if not isinstance(binding, dict): - return None - plan = binding.get("sourcePlan") - if not isinstance(plan, dict): - raise RuntimeError("manual account binding is missing resolved sourcePlan") - if plan.get("enabled") is not True: - raise RuntimeError(f"manual account binding source {plan.get('id')} is disabled") - if plan.get("kind") != expected_kind: - raise RuntimeError(f"manual account binding source {plan.get('id')} kind must be {expected_kind}") - return plan - -def desired_manual_proxy_payload(protection): - binding = protection.get("proxyBinding") if isinstance(protection, dict) else None - if not isinstance(binding, dict) or binding.get("enabled") is not True: - return None - plan = manual_binding_plan(binding, "proxy") - if plan.get("provider") not in ("target-egress-proxy", "fixed-http-proxy"): - raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}") - if plan.get("provider") == "fixed-http-proxy": - fixed_proxy = plan.get("fixedProxy") - if not isinstance(fixed_proxy, dict): - raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} requires fixedProxy") - proxy_name = binding.get("proxyName") - protocol = fixed_proxy.get("protocol") - host = fixed_proxy.get("host") - port = fixed_proxy.get("port") - if not isinstance(proxy_name, str) or not proxy_name: - raise RuntimeError("manual account proxyBinding proxyName is missing") - if protocol != "http": - raise RuntimeError("fixed proxy protocol must be http") - if not isinstance(host, str) or not host: - raise RuntimeError("fixed proxy host is missing") - if not isinstance(port, int) or port <= 0: - raise RuntimeError("fixed proxy port is missing") - return { - "name": proxy_name, - "protocol": protocol, - "host": host, - "port": port, - "fallback_mode": "none", - "expiry_warn_days": 0, - } - target_proxy = plan.get("targetEgressProxy") - if not isinstance(target_proxy, dict): - raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} requires an enabled target egressProxy") - service_name = target_proxy.get("serviceName") - listen_port = target_proxy.get("listenPort") - namespace = target_proxy.get("namespace") if isinstance(target_proxy.get("namespace"), str) and target_proxy.get("namespace") else NAMESPACE - proxy_name = binding.get("proxyName") - if not isinstance(service_name, str) or not service_name: - raise RuntimeError("target egressProxy serviceName is missing") - if not isinstance(listen_port, int) or listen_port <= 0: - raise RuntimeError("target egressProxy listenPort is missing") - if not isinstance(proxy_name, str) or not proxy_name: - raise RuntimeError("manual account proxyBinding proxyName is missing") - return { - "name": proxy_name, - "protocol": "http", - "host": f"{service_name}.{namespace}.svc.cluster.local", - "port": listen_port, - "fallback_mode": "none", - "expiry_warn_days": 0, - } - -def proxy_needs_update(proxy, payload): - if not isinstance(proxy, dict): - return True - for key in ("name", "protocol", "host", "port", "fallback_mode", "expiry_warn_days"): - if proxy.get(key) != payload.get(key): - return True - return proxy.get("status") != "active" - -def ensure_manual_proxy(token, payload): - current = find_proxy_by_name(token, payload["name"]) - if current is None: - created = ensure_success(curl_api("POST", "/api/v1/admin/proxies", bearer=token, payload=payload), f"create proxy {payload['name']}") - return created, "created" - if proxy_needs_update(current, payload): - update_payload = dict(payload) - update_payload["status"] = "active" - updated = ensure_success(curl_api("PUT", f"/api/v1/admin/proxies/{current['id']}", bearer=token, payload=update_payload), f"update proxy {payload['name']}") - return updated if isinstance(updated, dict) else current, "updated" - return current, "unchanged" - -def manual_proxy_status(token, account, protection): - payload = desired_manual_proxy_payload(protection) - if payload is None: - return { - "enabled": False, - "ok": True, - "action": "not-configured", - "valuesPrinted": False, - } - binding = protection.get("proxyBinding") if isinstance(protection, dict) else None - plan = manual_binding_plan(binding, "proxy") - proxy = find_proxy_by_name(token, payload["name"]) - runtime_proxy = account.get("proxy") if isinstance(account, dict) and isinstance(account.get("proxy"), dict) else None - binding_aligned = ( - isinstance(account, dict) - and isinstance(proxy, dict) - and account.get("proxy_id") == proxy.get("id") - ) - return { - "enabled": True, - "ok": binding_aligned, - "action": "validate", - "source": plan.get("id"), - "sourceProvider": plan.get("provider"), - "expectedProxyName": payload["name"], - "expectedProtocol": payload["protocol"], - "expectedHost": payload["host"], - "expectedPort": payload["port"], - "proxyRecordExists": isinstance(proxy, dict), - "proxyId": proxy.get("id") if isinstance(proxy, dict) else None, - "runtimeProxyId": account.get("proxy_id") if isinstance(account, dict) else None, - "runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None, - "bindingAligned": binding_aligned, - "valuesPrinted": False, - } - -def ensure_manual_account_proxy_bindings(token): - items = [] - for protection in MANUAL_ACCOUNT_PROTECTIONS: - if not isinstance(protection, dict): - continue - name = protection.get("accountName") - if not isinstance(name, str) or not name: - continue - account = find_account_by_name(token, name) - payload = desired_manual_proxy_payload(protection) - if payload is None: - items.append({ - "accountName": name, - "enabled": False, - "action": "not-configured", - "ok": True, - "valuesPrinted": False, - }) - continue - if not isinstance(account, dict): - items.append({ - "accountName": name, - "enabled": True, - "action": "account-missing", - "ok": False, - "expectedProxyName": payload["name"], - "valuesPrinted": False, - }) - continue - proxy, proxy_action = ensure_manual_proxy(token, payload) - binding = protection.get("proxyBinding") if isinstance(protection, dict) else None - plan = manual_binding_plan(binding, "proxy") - proxy_id = proxy.get("id") if isinstance(proxy, dict) else None - if proxy_id is None: - raise RuntimeError(f"proxy {payload['name']} has no id") - action = "unchanged" - if account.get("proxy_id") != proxy_id: - updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"proxy_id": proxy_id}), f"bind manual account proxy {name}") - account = updated if isinstance(updated, dict) else account - action = "bound" - runtime_proxy = account.get("proxy") if isinstance(account.get("proxy"), dict) else None - items.append({ - "accountName": name, - "accountId": account.get("id"), - "enabled": True, - "action": action, - "proxyAction": proxy_action, - "source": plan.get("id"), - "sourceProvider": plan.get("provider"), - "ok": account.get("proxy_id") == proxy_id, - "expectedProxyName": payload["name"], - "expectedProtocol": payload["protocol"], - "expectedHost": payload["host"], - "expectedPort": payload["port"], - "proxyId": proxy_id, - "runtimeProxyId": account.get("proxy_id"), - "runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None, - "bindingAligned": account.get("proxy_id") == proxy_id, - "controlPolicy": "manual-protected: only proxy_id binding is YAML-controlled; credentials/status/schedulable are untouched", - "valuesPrinted": False, - }) - return { - "ok": all(item.get("ok") is True for item in items), - "itemCount": len(items), - "items": items, - "valuesPrinted": False, - } - -def manual_group_binding_plan(protection): - binding = protection.get("groupBinding") if isinstance(protection, dict) else None - if not isinstance(binding, dict) or binding.get("enabled") is not True: - return None - plan = manual_binding_plan(binding, "group") - if plan.get("provider") != "pool-group": - raise RuntimeError(f"manual account groupBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}") - return plan - -def manual_group_binding_enabled(protection): - return manual_group_binding_plan(protection) is not None - -def manual_group_status(token, account, protection, group_id): - plan = manual_group_binding_plan(protection) - if plan is None: - return { - "enabled": False, - "ok": True, - "action": "not-configured", - "valuesPrinted": False, - } - group_accounts = list_accounts_for_group(token, group_id) - account_id = account.get("id") if isinstance(account, dict) else None - account_name = account.get("name") if isinstance(account, dict) else None - binding_aligned = any( - item.get("id") == account_id or item.get("name") == account_name - for item in group_accounts - if isinstance(item, dict) - ) - return { - "enabled": True, - "ok": binding_aligned, - "action": "validate", - "source": plan.get("id"), - "sourceProvider": plan.get("provider"), - "poolGroupName": POOL_GROUP_NAME, - "poolGroupId": group_id, - "bindingAligned": binding_aligned, - "valuesPrinted": False, - } - -def ensure_manual_account_group_bindings(token, group_id): - items = [] - for protection in MANUAL_ACCOUNT_PROTECTIONS: - if not isinstance(protection, dict): - continue - name = protection.get("accountName") - if not isinstance(name, str) or not name: - continue - if not manual_group_binding_enabled(protection): - items.append({ - "accountName": name, - "enabled": False, - "action": "not-configured", - "ok": True, - "valuesPrinted": False, - }) - continue - plan = manual_group_binding_plan(protection) - account = find_account_by_name(token, name) - if not isinstance(account, dict): - items.append({ - "accountName": name, - "enabled": True, - "action": "account-missing", - "ok": False, - "poolGroupName": POOL_GROUP_NAME, - "poolGroupId": group_id, - "valuesPrinted": False, - }) - continue - existing_group_ids = account_group_ids(token, account) - desired_group_ids = sorted(set(existing_group_ids + [group_id])) - action = "unchanged" - if group_id not in existing_group_ids: - updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"group_ids": desired_group_ids}), f"bind manual account group {name}") - account = updated if isinstance(updated, dict) else account - action = "bound" - binding_aligned = any( - item.get("id") == account.get("id") or item.get("name") == name - for item in list_accounts_for_group(token, group_id) - if isinstance(item, dict) - ) - items.append({ - "accountName": name, - "accountId": account.get("id"), - "enabled": True, - "ok": binding_aligned, - "action": action, - "source": plan.get("id") if isinstance(plan, dict) else None, - "sourceProvider": plan.get("provider") if isinstance(plan, dict) else None, - "poolGroupName": POOL_GROUP_NAME, - "poolGroupId": group_id, - "previousGroupIds": existing_group_ids, - "desiredGroupIds": desired_group_ids, - "bindingAligned": binding_aligned, - "controlPolicy": "manual-protected: only pool group membership is YAML-controlled; credentials/status/schedulable are untouched and sentinel does not probe it", - "valuesPrinted": False, - }) - return { - "ok": all(item.get("ok") is True for item in items), - "itemCount": len(items), - "items": items, - "valuesPrinted": False, - } - -def manual_account_protection_status(token, group_id=None): - items = [] - desired_names = set(EXPECTED_ACCOUNT_CAPACITIES.keys()) - for protection in MANUAL_ACCOUNT_PROTECTIONS: - if not isinstance(protection, dict): - continue - name = protection.get("accountName") - if not isinstance(name, str) or not name: - continue - account = find_account_by_name(token, name) - extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {} - proxy_status = manual_proxy_status(token, account, protection) - group_status = manual_group_status(token, account, protection, group_id) if group_id is not None else {"enabled": False, "ok": True, "action": "not-checked", "valuesPrinted": False} - items.append({ - "accountName": name, - "reason": protection.get("reason") if isinstance(protection.get("reason"), str) else None, - "exists": isinstance(account, dict), - "accountId": account.get("id") if isinstance(account, dict) else None, - "status": account.get("status") if isinstance(account, dict) else None, - "schedulable": account.get("schedulable") if isinstance(account, dict) else None, - "inYamlProfiles": name in desired_names, - "runtimeMarkedUnideskManaged": extra.get("unidesk_managed") is True, - "proxyBinding": proxy_status, - "groupBinding": group_status, - "ok": proxy_status.get("ok") is True and group_status.get("ok") is True, - "controlPolicy": "manual-protected: no create/update/prune/probe/freeze; optional proxy_id and pool group membership binding only when configured", - "valuesPrinted": False, - }) - return { - "ok": all(item.get("ok") is True for item in items), - "protectedCount": len(items), - "items": items, - "valuesPrinted": False, - } - -def list_probe_accounts(token): - path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-probe-") - data = ensure_success(curl_api("GET", path, bearer=token), "list probe accounts") - return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")] - -def list_probe_groups(token): - data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups") - return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")] - -def cleanup_probe_resources(token): - api_key_results = [] - account_results = [] - group_results = [] - for item in list_user_keys(token): - name = item.get("name") - key_id = item.get("id") - if not isinstance(name, str) or not name.startswith("unidesk-probe-") or key_id is None: - continue - api_key_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{key_id}", "api-key")) - for item in list_probe_accounts(token): - account_id = item.get("id") - if account_id is None: - continue - account_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account")) - for item in list_probe_groups(token): - group_id = item.get("id") - if group_id is None: - continue - group_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group")) - return { - "ok": all(item.get("ok") is True for item in api_key_results + account_results + group_results), - "apiKeysDeleted": len(api_key_results), - "accountsDeleted": len(account_results), - "groupsDeleted": len(group_results), - "items": { - "apiKeys": api_key_results, - "accounts": account_results, - "groups": group_results, - }, - "valuesPrinted": False, - } - -def temp_unschedulable_credentials(profile): - credentials = profile.get("tempUnschedulableCredentials") - if not isinstance(credentials, dict): - credentials = {} - return normalize_temp_unschedulable_credentials(credentials) - -def account_payload(profile, group_id): - extra = { - "openai_responses_mode": "force_responses", - "unidesk_codex_profile": profile["profile"], - "unidesk_managed": True, - "unidesk_trust_upstream": profile.get("trustUpstream") is True, - "unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, - } - ws_mode = profile.get("openaiResponsesWebSocketsV2Mode") - if ws_mode: - extra["openai_apikey_responses_websockets_v2_mode"] = ws_mode - extra["openai_apikey_responses_websockets_v2_enabled"] = ws_mode != "off" - credentials = { - "api_key": profile["apiKey"], - "base_url": profile["baseUrl"], - } - upstream_user_agent = profile.get("upstreamUserAgent") - if upstream_user_agent: - credentials["user_agent"] = upstream_user_agent - temp_unschedulable = temp_unschedulable_credentials(profile) - if temp_unschedulable["enabled"]: - credentials["temp_unschedulable_enabled"] = True - credentials["temp_unschedulable_rules"] = temp_unschedulable["rules"] - return { - "name": profile["accountName"], - "notes": f"UniDesk-managed Codex profile {profile['profile']} from {profile['configFile']} and {profile['authFile']}; secret source={profile['apiKeySource']}; fingerprint={profile['apiKeyFingerprint']}.", - "platform": "openai", - "type": "apikey", - "credentials": credentials, - "extra": extra, - "concurrency": int(profile.get("capacity", 5) or 5), - "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY), - "rate_multiplier": 1, - "load_factor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR), - "group_ids": [group_id], - "confirm_mixed_channel_risk": True, - **({"proxy_id": int(profile["proxyId"])} if profile.get("proxyId") is not None else {}), - } - -def planned_sentinel_account_results(profiles, existing_accounts): - existing = {item.get("name"): item for item in existing_accounts if isinstance(item, dict)} - results = [] - for profile in profiles: - change_reasons = sentinel_probe_change_reasons(existing.get(profile["accountName"]), profile) - quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0 - results.append({ - "profile": profile["profile"], - "accountName": profile["accountName"], - "profileConfig": { - "accountName": profile["accountName"], - "trustUpstream": profile.get("trustUpstream") is True, - "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, - }, - "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), - "sentinelProbeRequired": quality_gate_required, - "sentinelChangeReasons": change_reasons if quality_gate_required else [], - "sentinelProbePending": quality_gate_required, - "sentinelDefaultFrozen": False, - "valuesPrinted": False, - }) - return results - -def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_frozen_names=None, existing_accounts=None): - if not isinstance(protected_frozen_names, set): - protected_frozen_names = set() - if not isinstance(existing_accounts, list): - existing_accounts = list_accounts(token) - existing = {item.get("name"): item for item in existing_accounts} - desired_names = {profile["accountName"] for profile in profiles} - results = [] - for profile in profiles: - payload = account_payload(profile, group_id) - current = existing.get(profile["accountName"]) - change_reasons = sentinel_probe_change_reasons(current, profile) - quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0 - keep_frozen = profile["accountName"] in protected_frozen_names - if current and current.get("id") is not None: - account_id = current["id"] - update_payload = dict(payload) - update_payload.pop("platform", None) - update_payload["status"] = "active" - data = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account_id}", bearer=token, payload=update_payload), f"update account {profile['accountName']}") - action = "updated" - else: - data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}") - action = "created" - results.append({ - "profile": profile["profile"], - "accountName": profile["accountName"], - "profileConfig": { - "accountName": profile["accountName"], - "trustUpstream": profile.get("trustUpstream") is True, - "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, - }, - "accountId": data.get("id") if isinstance(data, dict) else None, - "action": action, - "baseUrl": profile["baseUrl"], - "apiKeySource": profile["apiKeySource"], - "apiKeyFingerprint": profile["apiKeyFingerprint"], - "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), - "sentinelProbeRequired": quality_gate_required, - "sentinelChangeReasons": change_reasons if quality_gate_required else [], - "sentinelProbePending": quality_gate_required, - "sentinelDefaultFrozen": False, - "sentinelFreezeProtected": keep_frozen, - "schedulableControl": "sentinel-marker-restore", - "openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"), - "trustUpstream": profile.get("trustUpstream") is True, - "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY), - "capacity": int(profile.get("capacity", 5) or 5), - "loadFactor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR), - "runtimeConcurrency": data.get("concurrency") if isinstance(data, dict) else None, - "runtimeLoadFactor": data.get("load_factor") if isinstance(data, dict) else None, - "runtimeSchedulable": data.get("schedulable") if isinstance(data, dict) else None, - "tempUnschedulableConfigured": bool(payload["credentials"].get("temp_unschedulable_enabled")), - "tempUnschedulableRuleCount": len(payload["credentials"].get("temp_unschedulable_rules") or []), - "upstreamUserAgentConfigured": bool(profile.get("upstreamUserAgent")), - "valuesPrinted": False, - }) - prune_results = prune_removed_accounts(token, existing_accounts, desired_names) if prune_removed else [] - return results, prune_results - -def prune_removed_accounts(token, existing_accounts, desired_names): - results = [] - for account in existing_accounts: - name = account.get("name") - account_id = account.get("id") - extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} - if not isinstance(name, str) or not name.startswith("unidesk-codex-"): - continue - if extra.get("unidesk_managed") is not True: - continue - if name in desired_names: - continue - if account_id is None: - raise RuntimeError(f"removed account {name} has no id") - ensure_success(curl_api("DELETE", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"delete removed account {name}") - results.append({ - "accountName": name, - "accountId": account_id, - "profile": extra.get("unidesk_codex_profile"), - "action": "deleted", - "reason": "removed-from-yaml", - "valuesPrinted": False, - }) - return results - -def generate_api_key(): - alphabet = string.ascii_letters + string.digits - return "sk-unidesk-codex-" + "".join(secrets.choice(alphabet) for _ in range(48)) - -def sub2api_key_value(item): - if not isinstance(item, dict): - return None - key = item.get("key") - return key if isinstance(key, str) and key else None - -def find_pool_key_by_name(keys): - return next((item for item in keys if isinstance(item, dict) and item.get("name") == POOL_API_KEY_NAME), None) - -def existing_sub2api_pool_api_key(token): - try: - existing = find_pool_key_by_name(list_user_keys(token)) - except Exception: - return None - return sub2api_key_value(existing) - -def ensure_pool_api_key_for_validate(token, secret_value, source_action): - group, group_action = ensure_group(token) - group_id = group.get("id") if isinstance(group, dict) else None - if group_id is None: - raise RuntimeError("pool group id missing while ensuring validate API key") - api_key = secret_value or generate_api_key() - api_key_result = ensure_sub2api_api_key(token, api_key, group_id) - host_env_action = write_host_env_value(POOL_API_KEY_SECRET_KEY, api_key) - key_item = { - "id": api_key_result.get("id"), - "name": api_key_result.get("name") or POOL_API_KEY_NAME, - "group_id": api_key_result.get("groupId"), - "user_id": api_key_result.get("userId"), - "key": api_key, - } - return api_key, key_item, { - "ok": True, - "source": "sub2api-admin-api", - "sourceAction": source_action, - "secretPresent": bool(secret_value), - "lookup": "ensured-by-admin-api", - "groupAction": group_action, - "sub2apiAction": api_key_result.get("action"), - "sub2apiId": api_key_result.get("id"), - "hostEnvAction": host_env_action, - "hostEnvPath": HOST_DOCKER_ENV_PATH, - "hostEnvKey": POOL_API_KEY_SECRET_KEY, - "valuesPrinted": False, - } - -def resolve_pool_api_key_for_validate(token): - secret_value = None - secret_error = None - try: - secret_value = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY) - except Exception as exc: - if RUNTIME_MODE != "host-docker": - raise - secret_error = str(exc) - keys = list_user_keys(token) - matched_item = None - if secret_value: - matched_item = next((item for item in keys if isinstance(item, dict) and item.get("key") == secret_value), None) - if isinstance(matched_item, dict) and matched_item.get("name") == POOL_API_KEY_NAME: - return secret_value, matched_item, { - "ok": True, - "source": pool_api_key_secret_location(), - "sourceAction": "kept-existing", - "secretPresent": True, - "lookup": "matched-by-value", - "hostEnvAction": "none", - "valuesPrinted": False, - } - if RUNTIME_MODE == "host-docker": - named_item = find_pool_key_by_name(keys) - named_key = sub2api_key_value(named_item) - if named_key: - host_env_action = write_host_env_value(POOL_API_KEY_SECRET_KEY, named_key) - return named_key, named_item, { - "ok": True, - "source": "sub2api-admin-api", - "sourceAction": "recovered-missing" if not secret_value else "repaired-mismatch", - "secretPresent": bool(secret_value), - "lookup": "matched-by-name", - "hostEnvAction": host_env_action, - "hostEnvPath": HOST_DOCKER_ENV_PATH, - "hostEnvKey": POOL_API_KEY_SECRET_KEY, - "valuesPrinted": False, - } - if not secret_value: - return ensure_pool_api_key_for_validate(token, secret_value, "created-missing") - matched_name = matched_item.get("name") if isinstance(matched_item, dict) else None - return ensure_pool_api_key_for_validate(token, secret_value, "repaired-mismatch:" + (matched_name or "-")) - if not secret_value: - raise RuntimeError(f"{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY} missing") - return secret_value, matched_item, { - "ok": isinstance(matched_item, dict) and matched_item.get("name") == POOL_API_KEY_NAME, - "source": pool_api_key_secret_location(), - "sourceAction": "kept-existing", - "secretPresent": True, - "lookup": "matched-by-value" if matched_item else "missing-in-admin-api", - "hostEnvAction": "not-applicable", - "valuesPrinted": False, - } - -def ensure_api_key_secret(group_id, token): - existing = None - try: - existing = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY) - except Exception: - existing = None - reused = None if existing else existing_sub2api_pool_api_key(token) - api_key = existing or reused or generate_api_key() - if existing: - secret_action = "kept-existing" - elif reused: - secret_action = "reused-existing-sub2api-key" - else: - secret_action = "created" - if RUNTIME_MODE == "host-docker": - env_action = "kept-existing" if existing else write_host_env_value(POOL_API_KEY_SECRET_KEY, api_key) - return api_key, secret_action, f"host-docker-env:{env_action};source={HOST_DOCKER_ENV_PATH};key={POOL_API_KEY_SECRET_KEY};valuesPrinted=false" - manifest = { - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": POOL_API_KEY_SECRET_NAME, - "namespace": NAMESPACE, - "labels": { - "app.kubernetes.io/name": "sub2api", - "app.kubernetes.io/part-of": "platform-infra", - "app.kubernetes.io/managed-by": "unidesk", - "unidesk.ai/secret-purpose": "sub2api-codex-pool-api-key", - }, - }, - "type": "Opaque", - "stringData": { - POOL_API_KEY_SECRET_KEY: api_key, - "GROUP_ID": str(group_id), - "GROUP_NAME": POOL_GROUP_NAME, - "SERVICE_DNS": SERVICE_DNS, - }, - } - proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) - if proc.returncode != 0: - raise RuntimeError(f"apply API key secret failed: {text(proc.stderr, 1000)}") - return api_key, secret_action, text(proc.stdout, 1000) - -def pool_api_key_secret_location(): - if RUNTIME_MODE == "host-docker": - return f"{HOST_DOCKER_ENV_PATH}.{POOL_API_KEY_SECRET_KEY}" - return f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}" - -def apply_sentinel_manifest(manifest): - if not TARGET_SENTINEL_ENABLED: - return { - "ok": True, - "action": "skipped-target-disabled", - "valuesPrinted": False, - } - if not isinstance(manifest, str) or not manifest.strip(): - return { - "ok": False, - "action": "missing-manifest", - "valuesPrinted": False, - } - proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest) - if proc.returncode != 0: - return { - "ok": False, - "action": "apply-failed", - "stdoutTail": text(proc.stdout, 2000), - "stderrTail": text(proc.stderr, 4000), - "valuesPrinted": False, - } - status = sentinel_runtime_status() - return { - "ok": status.get("ok") is True, - "action": "applied", - "stdoutTail": text(proc.stdout, 2000), - "runtime": status, - "valuesPrinted": False, - } - -def safe_kube_json(args, label): - proc = kubectl([*args, "-o", "json"]) - if proc.returncode != 0: - return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)} - try: - return json.loads(proc.stdout.decode("utf-8")), None - except Exception as exc: - return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)} - -def sentinel_runtime_status(): - if not TARGET_SENTINEL_ENABLED: - return { - "ok": True, - "action": "skipped-target-disabled", - "desired": { - "monitorEnabled": SENTINEL_CONFIG.get("monitor", {}).get("enabled"), - "actionsEnabled": SENTINEL_CONFIG.get("actions", {}).get("enabled"), - }, - "valuesPrinted": False, - } - cfg = SENTINEL_CONFIG - cronjob_name = cfg.get("cronJobName") - secret_name = cfg.get("credentialsSecretName") - configmap_name = cfg.get("configMapName") - state_name = cfg.get("stateConfigMapName") - cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}") - secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}") - configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}") - state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}") - state = None - if isinstance(state_cm, dict): - raw_state = (state_cm.get("data") or {}).get("state.json") - if isinstance(raw_state, str) and raw_state: - try: - state = json.loads(raw_state) - except Exception as exc: - state = {"parseError": str(exc)} - accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {} - quarantined = [] - recent_accounts = [] - for name, account_state in accounts.items(): - if not isinstance(account_state, dict): - continue - quarantine = account_state.get("quarantine") - if isinstance(quarantine, dict) and quarantine.get("active") is True: - quarantined.append({ - "accountName": name, - "until": quarantine.get("until"), - "applied": quarantine.get("applied"), - "reason": quarantine.get("reason"), - "failureKind": quarantine.get("failureKind"), - "errorDetails": quarantine.get("errorDetails"), - "intervalMinutes": quarantine.get("intervalMinutes"), - "sentinelProtect": quarantine.get("sentinelProtect"), - }) - last_probe = account_state.get("lastProbe") - if isinstance(last_probe, dict): - recent_accounts.append({ - "accountName": name, - "lastProbeAt": account_state.get("lastProbeAt"), - "lastStatus": account_state.get("lastStatus"), - "nextProbeAfter": account_state.get("nextProbeAfter"), - "ok": last_probe.get("ok"), - "purpose": last_probe.get("purpose"), - "httpStatus": last_probe.get("httpStatus"), - "durationMs": last_probe.get("durationMs"), - "markerMatched": last_probe.get("markerMatched"), - "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), - "outputHash": last_probe.get("outputHash"), - "outputPreview": last_probe.get("outputPreview"), - "responseBodyHash": last_probe.get("responseBodyHash"), - "responseBodyPreview": last_probe.get("responseBodyPreview"), - "error": last_probe.get("error"), - "errorDetails": last_probe.get("errorDetails"), - "usage": last_probe.get("usage"), - "failureKind": last_probe.get("failureKind"), - "requestShape": last_probe.get("requestShape"), - "action": last_probe.get("action"), - }) - recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "") - last_run = state.get("lastRun") if isinstance(state, dict) else None - cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {} - secret_data = secret.get("data") if isinstance(secret, dict) else {} - configmap_data = configmap.get("data") if isinstance(configmap, dict) else {} - ok = cronjob is not None and secret is not None and configmap is not None - return { - "ok": ok, - "desired": { - "monitorEnabled": cfg.get("monitor", {}).get("enabled"), - "actionsEnabled": cfg.get("actions", {}).get("enabled"), - "schedule": cfg.get("schedule"), - "cronJobName": cronjob_name, - "configMapName": configmap_name, - "credentialsSecretName": secret_name, - "stateConfigMapName": state_name, - }, - "cronJob": { - "exists": cronjob is not None, - "schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None, - "suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None, - "lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None, - "active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None, - "error": cronjob_error, - }, - "secret": { - "exists": secret is not None, - "profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data, - "valuesPrinted": False, - "error": secret_error, - }, - "configMap": { - "exists": configmap is not None, - "configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data, - "runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data, - "error": configmap_error, - }, - "state": { - "exists": state_cm is not None, - "accountCount": len(accounts), - "quarantinedCount": len(quarantined), - "quarantined": quarantined[-10:], - "recentAccounts": recent_accounts[-12:], - "lastRun": last_run, - "error": state_error, - }, - "valuesPrinted": False, - } - -def parse_epoch_z(value): - if not isinstance(value, str) or not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() - except Exception: - return None - -def sentinel_state_object(): - if not TARGET_SENTINEL_ENABLED: - return None, None - state_name = SENTINEL_CONFIG.get("stateConfigMapName") - if not state_name: - return None, None - obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}") - if not isinstance(obj, dict): - return None, None - raw_state = (obj.get("data") or {}).get("state.json") - if not isinstance(raw_state, str) or not raw_state: - return obj, None - try: - return obj, json.loads(raw_state) - except Exception: - return obj, None - -def active_sentinel_quarantine_names(): - if not TARGET_SENTINEL_ENABLED: - return set() - _, state = sentinel_state_object() - if not isinstance(state, dict): - return set() - accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} - names = set() - for name, account_state in accounts_state.items(): - if not isinstance(name, str) or not isinstance(account_state, dict): - continue - quarantine = account_state.get("quarantine") - if isinstance(quarantine, dict) and quarantine.get("active") is True and quarantine.get("applied") is True: - names.add(name) - return names - -def default_sentinel_state(): - return {"version": 1, "accounts": {}, "ledger": {}, "history": []} - -def clamp_sentinel_freezes_for_config(state, now): - freeze_config = SENTINEL_CONFIG.get("freeze") if isinstance(SENTINEL_CONFIG.get("freeze"), dict) else {} - try: - max_interval = int(freeze_config.get("maxTtlMinutes") or 10) - except Exception: - max_interval = 10 - accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} - now_epoch = time.time() - items = [] - for name, account_state in accounts_state.items(): - if not isinstance(name, str) or not isinstance(account_state, dict): - continue - quarantine = account_state.get("quarantine") - if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True: - continue - try: - interval = int(quarantine.get("intervalMinutes") or 0) - except Exception: - interval = 0 - until_epoch = parse_epoch_z(quarantine.get("until")) - old_until = quarantine.get("until") - if interval <= max_interval and (until_epoch is None or until_epoch <= now_epoch + max_interval * 60): - continue - quarantine["previousIntervalMinutes"] = interval - quarantine["intervalMinutes"] = max_interval - quarantine["until"] = now - quarantine["clampedAt"] = now - quarantine["clampedBy"] = "sync-freeze-max-ttl" - account_state["nextProbeAfter"] = now - items.append({ - "accountName": name, - "previousIntervalMinutes": interval, - "maxIntervalMinutes": max_interval, - "previousUntil": old_until, - "nextProbeAfter": now, - }) - return items - -def parse_iso_epoch(value): - if not isinstance(value, str) or not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() - except Exception: - return None - -def profile_success_max_interval(profile): - cadence = SENTINEL_CONFIG.get("cadence") if isinstance(SENTINEL_CONFIG.get("cadence"), dict) else {} - legacy = cadence.get("successMaxIntervalMinutes") - if legacy is None: - legacy = cadence.get("trustedSuccessMaxIntervalMinutes") or cadence.get("untrustedSuccessMaxIntervalMinutes") or 1 - if profile.get("trustUpstream") is True: - value = cadence.get("trustedSuccessMaxIntervalMinutes") or legacy - else: - value = cadence.get("untrustedSuccessMaxIntervalMinutes") or legacy - try: - return int(value) - except Exception: - return int(legacy) - -def clamp_sentinel_success_cadence_for_config(state, profiles, now): - accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} - profile_map = {item.get("accountName"): item for item in profiles if isinstance(item, dict) and isinstance(item.get("accountName"), str)} - now_epoch = time.time() - items = [] - for name, profile in profile_map.items(): - account_state = accounts_state.get(name) - if not isinstance(account_state, dict): - continue - quarantine = account_state.get("quarantine") - if isinstance(quarantine, dict) and quarantine.get("active") is True: - account_state["trustUpstream"] = profile.get("trustUpstream") is True - account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} - account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile) - continue - try: - interval = int(account_state.get("successIntervalMinutes") or 0) - except Exception: - interval = 0 - next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter")) - max_interval = profile_success_max_interval(profile) - account_state["trustUpstream"] = profile.get("trustUpstream") is True - account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} - account_state["successMaxIntervalMinutes"] = max_interval - if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60): - continue - old_next = account_state.get("nextProbeAfter") - account_state["previousSuccessIntervalMinutes"] = interval - account_state["successIntervalMinutes"] = min(interval, max_interval) if interval > 0 else interval - account_state["nextProbeAfter"] = now - account_state["cadenceClampedAt"] = now - account_state["cadenceClampedBy"] = "sync-success-max-interval" - items.append({ - "accountName": name, - "trustUpstream": profile.get("trustUpstream") is True, - "previousSuccessIntervalMinutes": interval, - "maxIntervalMinutes": max_interval, - "previousNextProbeAfter": old_next, - "nextProbeAfter": now, - }) - return items - -def update_sentinel_state_configmap(obj, state): - state_name = SENTINEL_CONFIG.get("stateConfigMapName") - if not state_name: - return {"ok": False, "reason": "state-configmap-missing"} - state_json = json.dumps(state, ensure_ascii=False, indent=2) - manifest = { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": state_name, - "namespace": NAMESPACE, - "labels": { - "app.kubernetes.io/name": SERVICE_NAME, - "app.kubernetes.io/component": "account-sentinel", - "app.kubernetes.io/managed-by": "unidesk-platform-infra", - }, - }, - "data": {"state.json": state_json}, - } - proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) - action = "applied" if isinstance(obj, dict) else "created" - if proc.returncode != 0: - return {"ok": False, "reason": f"{action}-failed", "stderrTail": text(proc.stderr, 2000)} - return {"ok": True, "action": action} - -def ensure_sentinel_state_for_sync(account_results, pending_only=False): - if not sentinel_quality_gate_enabled(): - return {"ok": True, "skipped": True, "reason": "sentinel-quality-gate-disabled", "changedCount": 0, "items": [], "valuesPrinted": False} - state_obj, state = sentinel_state_object() - if not isinstance(state, dict): - state = default_sentinel_state() - state.setdefault("version", 1) - accounts_state = state.setdefault("accounts", {}) - if not isinstance(accounts_state, dict): - accounts_state = {} - state["accounts"] = accounts_state - now = utc_iso() - items = [] - clamped_items = [] if pending_only else clamp_sentinel_freezes_for_config(state, now) - cadence_clamped_items = [] if pending_only else clamp_sentinel_success_cadence_for_config(state, [item.get("profileConfig") for item in account_results if isinstance(item.get("profileConfig"), dict)], now) - changed_count = 0 - fingerprint_only_count = 0 - for item in account_results: - name = item.get("accountName") - if not isinstance(name, str) or not name: - continue - account_state = accounts_state.setdefault(name, {}) - if not isinstance(account_state, dict): - account_state = {} - accounts_state[name] = account_state - fingerprint_value = item.get("sentinelProbeConfigFingerprint") - if isinstance(fingerprint_value, str) and fingerprint_value: - account_state["probeConfigFingerprint"] = fingerprint_value - if item.get("sentinelProbeRequired") is not True: - fingerprint_only_count += 1 - continue - changed_count += 1 - reasons = item.get("sentinelChangeReasons") if isinstance(item.get("sentinelChangeReasons"), list) else [] - quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else None - if not (isinstance(quarantine, dict) and quarantine.get("active") is True): - account_state["quarantine"] = { - "active": False, - "reason": "yaml-account-change-pending-sentinel-probe", - "lastPendingAt": now, - "changeReasons": reasons, - } - account_state["nextProbeAfter"] = now - account_state["successStreak"] = 0 - account_state["successIntervalMinutes"] = 0 - profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {} - account_state["trustUpstream"] = profile_config.get("trustUpstream") is True - account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False} - account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config) - account_state["lastStatus"] = "pending-sentinel-quality-gate" - account_state["qualityGate"] = { - "pending": True, - "reason": "yaml-account-change", - "changeReasons": reasons, - "markedAt": now, - "pendingOnly": pending_only, - "defaultFrozen": False, - } - items.append({"accountName": name, "changeReasons": reasons, "nextProbeAfter": now, "defaultFrozen": False, "defaultSchedulable": True, "pendingOnly": pending_only}) - if changed_count <= 0 and len(clamped_items) <= 0 and len(cadence_clamped_items) <= 0: - return {"ok": True, "skipped": False, "reason": "no-new-or-changed-accounts", "changedCount": 0, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": 0, "cadenceClampedCount": 0, "items": [], "valuesPrinted": False} - update = update_sentinel_state_configmap(state_obj, state) - if pending_only and changed_count > 0: - reason = "new-or-changed-accounts-pending-probe-prepared-default-schedulable" - elif changed_count > 0 and (len(clamped_items) > 0 or len(cadence_clamped_items) > 0): - reason = "new-or-changed-accounts-default-schedulable-and-sentinel-cadence-clamped" - elif changed_count > 0: - reason = "new-or-changed-accounts-default-schedulable" - elif len(cadence_clamped_items) > 0: - reason = "success-cadence-clamped-to-current-config" - else: - reason = "freeze-backoff-clamped-to-current-config" - return { - "ok": update.get("ok") is True, - "skipped": False, - "reason": reason, - "changedCount": changed_count, - "fingerprintOnlyCount": fingerprint_only_count, - "clampedCount": len(clamped_items), - "cadenceClampedCount": len(cadence_clamped_items), - "pendingOnly": pending_only, - "items": items, - "clampedItems": clamped_items, - "cadenceClampedItems": cadence_clamped_items, - "update": update, - "valuesPrinted": False, - } - -def sentinel_state_summary(): - _, state = sentinel_state_object() - if not isinstance(state, dict): - return {"exists": False, "valuesPrinted": False} - accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} - quarantined = [] - recent_accounts = [] - for name, account_state in accounts.items(): - if not isinstance(account_state, dict): - continue - quarantine = account_state.get("quarantine") - if isinstance(quarantine, dict) and quarantine.get("active") is True: - quarantined.append({ - "accountName": name, - "until": quarantine.get("until"), - "applied": quarantine.get("applied"), - "reason": quarantine.get("reason"), - "failureKind": quarantine.get("failureKind"), - "errorDetails": quarantine.get("errorDetails"), - "intervalMinutes": quarantine.get("intervalMinutes"), - "sentinelProtect": quarantine.get("sentinelProtect"), - }) - last_probe = account_state.get("lastProbe") - if isinstance(last_probe, dict): - recent_accounts.append({ - "accountName": name, - "lastProbeAt": account_state.get("lastProbeAt"), - "lastStatus": account_state.get("lastStatus"), - "nextProbeAfter": account_state.get("nextProbeAfter"), - "ok": last_probe.get("ok"), - "purpose": last_probe.get("purpose"), - "httpStatus": last_probe.get("httpStatus"), - "durationMs": last_probe.get("durationMs"), - "markerMatched": last_probe.get("markerMatched"), - "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), - "usage": last_probe.get("usage"), - "responseBodyHash": last_probe.get("responseBodyHash"), - "responseBodyPreview": last_probe.get("responseBodyPreview"), - "error": last_probe.get("error"), - "errorDetails": last_probe.get("errorDetails"), - "failureKind": last_probe.get("failureKind"), - "requestShape": last_probe.get("requestShape"), - "action": last_probe.get("action"), - }) - recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "") - return { - "exists": True, - "accountCount": len(accounts), - "quarantinedCount": len(quarantined), - "quarantined": quarantined[-10:], - "recentAccounts": recent_accounts[-12:], - "lastRun": state.get("lastRun"), - "valuesPrinted": False, - } - -def reassert_sentinel_freezes_after_sync(token): - if not TARGET_SENTINEL_ENABLED: - return {"ok": True, "skipped": True, "reason": "target-disabled", "items": [], "valuesPrinted": False} - if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True: - return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False} - _, state = sentinel_state_object() - if not isinstance(state, dict): - return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False} - accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} - accounts = list_accounts(token) - by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} - items = [] - for name, account_state in accounts_state.items(): - if not isinstance(account_state, dict): - continue - quarantine = account_state.get("quarantine") - if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True: - continue - account = by_name.get(name) - if not account or account.get("id") is None: - items.append({"accountName": name, "ok": False, "reason": "account-not-found"}) - continue - try: - ensure_success( - curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}), - f"reassert sentinel freeze for {name}", - ) - items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")}) - except Exception as exc: - items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)}) - return { - "ok": all(item.get("ok") is True for item in items), - "skipped": False, - "items": items, - "valuesPrinted": False, - } - -def list_user_keys(token): - data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys") - return extract_items(data) - -def ensure_sub2api_api_key(token, api_key, group_id): - keys = list_user_keys(token) - existing = next((item for item in keys if item.get("key") == api_key), None) - action = "kept-existing" - if existing is None: - existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None) - if existing is None or existing.get("key") != api_key: - payload = { - "name": POOL_API_KEY_NAME, - "group_id": group_id, - "custom_key": api_key, - "quota": 0, - "rate_limit_5h": 0, - "rate_limit_1d": 0, - "rate_limit_7d": 0, - } - created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key") - existing = created if isinstance(created, dict) else existing - action = "created" - elif existing.get("id") is not None and (existing.get("group_id") != group_id or existing.get("name") != POOL_API_KEY_NAME): - updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group") - existing = updated if isinstance(updated, dict) else existing - action = "updated-name-group" - return { - "action": action, - "id": existing.get("id") if isinstance(existing, dict) else None, - "name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME, - "groupId": existing.get("group_id") if isinstance(existing, dict) else group_id, - "userId": existing.get("user_id") if isinstance(existing, dict) else None, - } - -def get_admin_user(token, user_id): - data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner") - if not isinstance(data, dict): - raise RuntimeError("API key owner response is not an object") - return data - -def ensure_pool_owner_balance(token, user_id): - user = get_admin_user(token, user_id) - current = float(user.get("balance") or 0) - if current >= MIN_OWNER_BALANCE_USD: - return { - "action": "kept-existing", - "userId": user_id, - "balanceBefore": current, - "balanceAfter": current, - "minimumBalanceUsd": MIN_OWNER_BALANCE_USD, - } - updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={ - "balance": MIN_OWNER_BALANCE_USD, - "operation": "set", - "notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.", - }), "set API key owner balance") - after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD - return { - "action": "set", - "userId": user_id, - "balanceBefore": current, - "balanceAfter": after, - "minimumBalanceUsd": MIN_OWNER_BALANCE_USD, - } - -def ensure_pool_owner_concurrency(token, user_id): - user = get_admin_user(token, user_id) - try: - current = int(user.get("concurrency") or 0) - except Exception: - current = 0 - if current >= MIN_OWNER_CONCURRENCY: - return { - "ok": True, - "action": "kept-existing", - "userId": user_id, - "concurrencyBefore": current, - "concurrencyAfter": current, - "minimumConcurrency": MIN_OWNER_CONCURRENCY, - "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, - } - updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={ - "concurrency": MIN_OWNER_CONCURRENCY, - }), "set API key owner concurrency") - try: - after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY - except Exception: - after = MIN_OWNER_CONCURRENCY - return { - "ok": after >= MIN_OWNER_CONCURRENCY, - "action": "set", - "userId": user_id, - "concurrencyBefore": current, - "concurrencyAfter": after, - "minimumConcurrency": MIN_OWNER_CONCURRENCY, - "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, - } - -def validate_gateway(api_key): - resp = curl_api("GET", "/v1/models", bearer=api_key) - parsed = resp.get("json") - model_count = None - if isinstance(parsed, dict) and isinstance(parsed.get("data"), list): - model_count = len(parsed["data"]) - return { - "ok": resp.get("ok"), - "httpStatus": resp.get("httpStatus"), - "transportExitCode": resp.get("transportExitCode"), - "modelCount": model_count, - "bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "", - "stderr": resp.get("stderr", ""), - "method": "GET /v1/models", - "serviceDns": SERVICE_DNS, - "valuesPrinted": False, - } - -def response_output_preview(parsed): - if not isinstance(parsed, dict): - return "" - if isinstance(parsed.get("output_text"), str): - return parsed["output_text"][:240] - output = parsed.get("output") - if not isinstance(output, list): - return "" - parts = [] - for item in output: - if not isinstance(item, dict): - continue - content = item.get("content") - if not isinstance(content, list): - continue - for block in content: - if not isinstance(block, dict): - continue - text_value = block.get("text") - if isinstance(text_value, str) and text_value: - parts.append(text_value) - return "\\n".join(parts)[:240] - -def request_log_evidence(request_id): - proc = runtime_logs("5m", 800) - stdout = proc.stdout.decode("utf-8", errors="replace") - lines = [line for line in stdout.splitlines() if request_id in line] - failovers = [] - final = None - for line in lines: - json_start = line.find("{") - if json_start < 0: - continue - try: - item = json.loads(line[json_start:]) - except Exception: - continue - if "upstream_failover_switching" in line: - failovers.append({ - "accountId": item.get("account_id"), - "upstreamStatus": item.get("upstream_status"), - "switchCount": item.get("switch_count"), - "maxSwitches": item.get("max_switches"), - }) - if "http request completed" in line: - final = { - "accountId": item.get("account_id"), - "statusCode": item.get("status_code"), - "latencyMs": item.get("latency_ms"), - "path": item.get("path"), - } - return { - "requestId": request_id, - "matchedLogLineCount": len(lines), - "failovers": failovers, - "final": final, - "logsExitCode": proc.returncode, - "logsStderr": text(proc.stderr, 1000), - } - -def recent_compact_gateway_evidence(): - proc = runtime_logs("6h", 2500) - stdout = proc.stdout.decode("utf-8", errors="replace") - failures = [] - successes = [] - failovers = [] - final_errors = [] - context_canceled = [] - for line in stdout.splitlines(): - if "/responses/compact" not in line and "remote_compact" not in line: - continue - json_start = line.find("{") - if json_start < 0: - continue - try: - item = json.loads(line[json_start:]) - except Exception: - continue - path = item.get("path") - entry = { - "requestId": item.get("request_id"), - "clientRequestId": item.get("client_request_id"), - "accountId": item.get("account_id"), - "statusCode": item.get("status_code"), - "upstreamStatus": item.get("upstream_status"), - "latencyMs": item.get("latency_ms"), - "path": path, - } - if "codex.remote_compact.failed" in line: - failures.append(entry) - elif "codex.remote_compact.succeeded" in line: - successes.append(entry) - elif "upstream_failover_switching" in line and path == "/responses/compact": - failovers.append({ - **entry, - "switchCount": item.get("switch_count"), - "maxSwitches": item.get("max_switches"), - }) - elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400: - final_errors.append(entry) - if "context canceled" in line and path == "/responses/compact": - context_canceled.append(entry) - return { - "ok": True, - "degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0, - "window": "6h", - "tailLines": 2500, - "failureCount": len(failures), - "successCount": len(successes), - "failoverCount": len(failovers), - "finalErrorCount": len(final_errors), - "contextCanceledCount": len(context_canceled), - "recentFailures": failures[-5:], - "recentSuccesses": successes[-5:], - "recentFailovers": failovers[-8:], - "recentFinalErrors": final_errors[-5:], - "recentContextCanceled": context_canceled[-5:], - "logsExitCode": proc.returncode, - "logsStderr": text(proc.stderr, 1000), - "valuesPrinted": False, - } - -def is_ignored_probe_noise(entry): - for value in (entry.get("requestId"), entry.get("clientRequestId")): - if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"): - return True - return False - -def filter_ignored_probe_noise(items): - return [item for item in items if not is_ignored_probe_noise(item)] - -def ignored_probe_noise(items, section): - result = [] - for item in items: - if is_ignored_probe_noise(item): - probe = dict(item) - probe["section"] = section - result.append(probe) - return result - -def group_by_request_id(items): - grouped = {} - for item in items: - request_id = item.get("requestId") - if not isinstance(request_id, str) or not request_id: - continue - grouped.setdefault(request_id, []).append(item) - return grouped - -def failover_budget_exhausted_evidence(failovers, final_errors): - final_by_request = {} - for item in final_errors: - request_id = item.get("requestId") - if isinstance(request_id, str) and request_id: - final_by_request[request_id] = item - exhausted = [] - for request_id, request_failovers in group_by_request_id(failovers).items(): - final = final_by_request.get(request_id) - if not final: - continue - last = request_failovers[-1] - switch_count = last.get("switchCount") - max_switches = last.get("maxSwitches") - final_status = final.get("statusCode") - if ( - isinstance(switch_count, int) - and isinstance(max_switches, int) - and max_switches > 0 - and switch_count >= max_switches - and isinstance(final_status, int) - and final_status >= 500 - ): - exhausted.append({ - "requestId": request_id, - "clientRequestId": final.get("clientRequestId") or last.get("clientRequestId"), - "path": final.get("path") or last.get("path"), - "finalAccountId": final.get("accountId"), - "finalStatusCode": final_status, - "switchCount": switch_count, - "maxSwitches": max_switches, - "lastFailoverAccountId": last.get("accountId"), - "lastUpstreamStatus": last.get("upstreamStatus"), - }) - return exhausted - -def recent_responses_gateway_evidence(): - proc = runtime_logs("6h", 2500) - stdout = proc.stdout.decode("utf-8", errors="replace") - failovers = [] - forward_failures = [] - final_errors = [] - context_canceled = [] - slow_final_errors = [] - for line in stdout.splitlines(): - if '"/responses"' not in line and '"/v1/responses"' not in line: - continue - json_start = line.find("{") - if json_start < 0: - continue - try: - item = json.loads(line[json_start:]) - except Exception: - continue - path = item.get("path") - if path not in ("/responses", "/v1/responses"): - continue - entry = { - "requestId": item.get("request_id"), - "clientRequestId": item.get("client_request_id"), - "accountId": item.get("account_id"), - "statusCode": item.get("status_code"), - "upstreamStatus": item.get("upstream_status"), - "latencyMs": item.get("latency_ms"), - "path": path, - } - if "upstream_failover_switching" in line: - failovers.append({ - **entry, - "switchCount": item.get("switch_count"), - "maxSwitches": item.get("max_switches"), - }) - elif "openai.forward_failed" in line: - forward_failures.append({ - **entry, - "errorPreview": text(str(item.get("error") or ""), 500), - "fallbackErrorResponseWritten": item.get("fallback_error_response_written"), - "upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"), - }) - elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400: - final_errors.append(entry) - latency_ms = item.get("latency_ms") - if isinstance(latency_ms, int) and latency_ms >= 30000: - slow_final_errors.append(entry) - if "context canceled" in line: - context_canceled.append(entry) - visible_failovers = filter_ignored_probe_noise(failovers) - visible_forward_failures = filter_ignored_probe_noise(forward_failures) - visible_final_errors = filter_ignored_probe_noise(final_errors) - visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors) - visible_context_canceled = filter_ignored_probe_noise(context_canceled) - failover_budget_exhausted = failover_budget_exhausted_evidence(visible_failovers, visible_final_errors) - probe_noise = ( - ignored_probe_noise(failovers, "failovers") - + ignored_probe_noise(forward_failures, "forwardFailures") - + ignored_probe_noise(final_errors, "finalErrors") - + ignored_probe_noise(slow_final_errors, "slowFinalErrors") - + ignored_probe_noise(context_canceled, "contextCanceled") - ) - return { - "ok": True, - "degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0, - "window": "6h", - "tailLines": 2500, - "failoverCount": len(visible_failovers), - "forwardFailureCount": len(visible_forward_failures), - "finalErrorCount": len(visible_final_errors), - "slowFinalErrorCount": len(visible_slow_final_errors), - "contextCanceledCount": len(visible_context_canceled), - "ignoredProbeNoiseCount": len(probe_noise), - "failoverBudgetExhausted": failover_budget_exhausted[-8:], - "rawCounts": { - "failoverCount": len(failovers), - "forwardFailureCount": len(forward_failures), - "finalErrorCount": len(final_errors), - "slowFinalErrorCount": len(slow_final_errors), - "contextCanceledCount": len(context_canceled), - }, - "recentFailovers": visible_failovers[-8:], - "recentForwardFailures": visible_forward_failures[-8:], - "recentFinalErrors": visible_final_errors[-8:], - "recentSlowFinalErrors": visible_slow_final_errors[-5:], - "recentContextCanceled": visible_context_canceled[-5:], - "recentProbeNoise": probe_noise[-5:], - "logsExitCode": proc.returncode, - "logsStderr": text(proc.stderr, 1000), - "valuesPrinted": False, - } - -def validate_gateway_responses(api_key): - request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000)) - payload = { - "model": RESPONSES_SMOKE_MODEL, - "input": "Reply exactly: unidesk-sub2api-validate-ok", - "stream": False, - "store": False, - "max_output_tokens": 32, - } - body = json.dumps(payload, separators=(",", ":")).encode("utf-8") - script = r''' -set -eu -token="$1" -request_id="$2" -url="$3" -tmp="$(mktemp)" -trap 'rm -f "$tmp"' EXIT -cat > "$tmp" -curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \ - -H "Authorization: Bearer $token" \ - -H 'Content-Type: application/json' \ - -H "X-Request-ID: $request_id" \ - -H "OpenAI-Client-Request-ID: $request_id" \ - --data-binary @"$tmp" \ - "$url" -''' - started = time.time() - if RUNTIME_MODE == "host-docker": - if not isinstance(HOST_DOCKER_APP_PORT, int): - raise RuntimeError("host-docker app port missing") - proc = run(["sh", "-c", script, "sh", api_key, request_id, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}/v1/responses"], body) - else: - proc = run([ - "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, - "--", "sh", "-c", script, "sh", api_key, request_id, "http://127.0.0.1:8080/v1/responses", - ], body) - resp = parse_curl_output(proc) - evidence = request_log_evidence(request_id) - parsed = resp.get("json") - failover_count = len(evidence.get("failovers") or []) - return { - "ok": resp.get("ok"), - "degraded": failover_count > 0, - "outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"), - "httpStatus": resp.get("httpStatus"), - "transportExitCode": resp.get("transportExitCode"), - "method": "POST /v1/responses", - "model": RESPONSES_SMOKE_MODEL, - "requestId": request_id, - "durationMs": int((time.time() - started) * 1000), - "outputTextPreview": response_output_preview(parsed), - "bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800), - "stderr": resp.get("stderr", ""), - "evidence": evidence, - "valuesPrinted": False, - } - -def bool_value(value): - if isinstance(value, bool): - return value - if isinstance(value, str): - if value.lower() == "true": - return True - if value.lower() == "false": - return False - return False - -def normalize_temp_unschedulable_credentials(credentials): - if not isinstance(credentials, dict): - credentials = {} - enabled = bool_value(credentials.get("temp_unschedulable_enabled")) - raw_rules = credentials.get("temp_unschedulable_rules") - if isinstance(raw_rules, str): - try: - raw_rules = json.loads(raw_rules) - except json.JSONDecodeError: - raw_rules = [] - rules = [] - if isinstance(raw_rules, list): - for rule in raw_rules: - if not isinstance(rule, dict): - continue - error_code = rule.get("error_code", rule.get("statusCode")) - duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes")) - keywords = rule.get("keywords") - if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list): - continue - clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()] - if not clean_keywords: - continue - description = rule.get("description") if isinstance(rule.get("description"), str) else "" - rules.append({ - "error_code": error_code, - "keywords": clean_keywords, - "duration_minutes": duration_minutes, - "description": description, - }) - return { - "enabled": enabled, - "rules": rules, - } - -def summarize_temp_unschedulable_rules(rules): - return [{ - "errorCode": rule.get("error_code"), - "durationMinutes": rule.get("duration_minutes"), - "keywordCount": len(rule.get("keywords") or []), - "keywords": rule.get("keywords") or [], - "hasDescription": bool(rule.get("description")), - } for rule in rules] - -def success_body_reclassification_requirement(): - for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): - expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) - if expected["enabled"] is not True: - continue - for rule in expected["rules"]: - error_code = rule.get("error_code") - keywords = rule.get("keywords") or [] - if isinstance(error_code, int) and 200 <= error_code < 300 and keywords: - return { - "required": True, - "sourceAccountName": name, - "statusCode": error_code, - "keywords": keywords, - "representativeKeyword": keywords[0], - "durationMinutes": rule.get("duration_minutes"), - } - return { - "required": False, - "sourceAccountName": None, - "statusCode": None, - "keywords": [], - "representativeKeyword": None, - "durationMinutes": None, - } - -def model_routing_400_failover_requirement(): - preferred = ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"] - for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): - expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) - if expected["enabled"] is not True: - continue - for rule in expected["rules"]: - error_code = rule.get("error_code") - keywords = rule.get("keywords") or [] - if error_code != 400 or not keywords: - continue - representative_keyword = next((item for item in preferred if item in keywords), keywords[0]) - return { - "required": True, - "sourceAccountName": name, - "statusCode": error_code, - "keywords": keywords, - "representativeKeyword": representative_keyword, - "durationMinutes": rule.get("duration_minutes"), - } - return { - "required": False, - "sourceAccountName": None, - "statusCode": None, - "keywords": [], - "representativeKeyword": None, - "durationMinutes": None, - } - -def delete_probe_resource(token, method, path, label): - if not path: - return {"label": label, "ok": True, "skipped": True} - resp = curl_api(method, path, bearer=token) - ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410) - return { - "label": label, - "ok": ok, - "method": method, - "path": path, - "httpStatus": resp.get("httpStatus"), - "transportExitCode": resp.get("transportExitCode"), - "bodyPreview": "" if ok else text(resp.get("body", ""), 500), - "valuesPrinted": False, - } - -def validate_runtime_capabilities(token): - success_body = success_body_reclassification_requirement() - model_routing_400 = model_routing_400_failover_requirement() - return { - "ok": success_body.get("required") is not True and model_routing_400.get("required") is True, - "runtimeImage": app_pod_runtime_image(), - "successBodyReclassification": { - "ok": success_body.get("required") is not True, - "required": success_body.get("required"), - "outcome": "not-required-by-yaml" if success_body.get("required") is not True else "requires-source-review-and-real-traffic-evidence", - "requirement": success_body, - "valuesPrinted": False, - }, - "modelRouting400Failover": { - "ok": model_routing_400.get("required") is True, - "required": model_routing_400.get("required"), - "outcome": "yaml-rules-present-runtime-observed-by-real-traffic", - "requirement": model_routing_400, - "evidence": "Use Sub2API source review plus validation.gatewayResponsesRecent and Artificer/real request ids for runtime proof; default validate does not create mock upstreams or temporary failover accounts.", - "valuesPrinted": False, - }, - "valuesPrinted": False, - } - -def app_pod_runtime_image(): - if RUNTIME_MODE == "host-docker": - proc = docker(["inspect", HOST_DOCKER_APP_CONTAINER]) - if proc.returncode != 0: - return { - "container": HOST_DOCKER_APP_CONTAINER, - "error": text(proc.stderr, 1000) or text(proc.stdout, 1000), - } - try: - data = json.loads(proc.stdout.decode("utf-8")) - item = data[0] if isinstance(data, list) and data else {} - except Exception as exc: - return {"container": HOST_DOCKER_APP_CONTAINER, "error": str(exc)} - state = item.get("State") if isinstance(item, dict) and isinstance(item.get("State"), dict) else {} - health = state.get("Health") if isinstance(state.get("Health"), dict) else {} - config = item.get("Config") if isinstance(item, dict) and isinstance(item.get("Config"), dict) else {} - return { - "container": HOST_DOCKER_APP_CONTAINER, - "id": (item.get("Id") or "")[:12] if isinstance(item.get("Id"), str) else None, - "image": config.get("Image"), - "imageID": item.get("Image"), - "ready": state.get("Running") is True and (not health or health.get("Status") in (None, "healthy")), - "restartCount": item.get("RestartCount"), - "startedAt": state.get("StartedAt"), - "health": health.get("Status"), - } - try: - pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}") - spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else [] - status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else [] - spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {}) - status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {}) - return { - "pod": APP_POD, - "image": spec.get("image"), - "imageID": status.get("imageID"), - "ready": status.get("ready"), - "restartCount": status.get("restartCount"), - } - except Exception as exc: - return {"pod": APP_POD, "error": str(exc)} - -def get_account_detail(token, account): - account_id = account.get("id") if isinstance(account, dict) else None - if account_id is None: - return account - data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}") - return data if isinstance(data, dict) else account - -def account_temp_unschedulable_status(token): - accounts = list_accounts(token) - by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} - items = [] - missing = [] - mismatched = [] - enabled_names = [] - for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): - expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) - account = by_name.get(name) - if account is None: - missing.append(name) - items.append({ - "accountName": name, - "accountId": None, - "expectedEnabled": expected["enabled"], - "runtimeEnabled": None, - "expectedRuleCount": len(expected["rules"]), - "runtimeRuleCount": None, - "ok": False, - }) - continue - detail = get_account_detail(token, account) - credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {} - runtime = normalize_temp_unschedulable_credentials(credentials) - temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil") - temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or "" - ok = runtime == expected - if expected["enabled"]: - enabled_names.append(name) - if not ok: - mismatched.append(name) - items.append({ - "accountName": name, - "accountId": account.get("id"), - "expectedEnabled": expected["enabled"], - "runtimeEnabled": runtime["enabled"], - "expectedRuleCount": len(expected["rules"]), - "runtimeRuleCount": len(runtime["rules"]), - "expectedRules": summarize_temp_unschedulable_rules(expected["rules"]), - "runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]), - "status": account.get("status"), - "schedulable": account.get("schedulable"), - "tempUnschedulableUntil": temp_until, - "tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "", - "tempUnschedulableSet": temp_until is not None or bool(temp_reason), - "ok": ok, - }) - return { - "ok": len(missing) == 0 and len(mismatched) == 0, - "desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE), - "enabled": enabled_names, - "missing": missing, - "mismatched": mismatched, - "items": items, - "valuesPrinted": False, - } - -def account_capacity_status(token): - accounts = list_accounts(token) - by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} - items = [] - missing = [] - mismatched = [] - expected_capacity_total = 0 - runtime_concurrency_total = 0 - schedulable_runtime_concurrency_total = 0 - available_runtime_concurrency_total = 0 - temp_unschedulable_runtime_concurrency_total = 0 - for name in sorted(EXPECTED_ACCOUNT_CAPACITIES): - expected = int(EXPECTED_ACCOUNT_CAPACITIES[name]) - expected_capacity_total += expected - account = by_name.get(name) - if account is None: - missing.append(name) - items.append({ - "accountName": name, - "accountId": None, - "expectedCapacity": expected, - "runtimeConcurrency": None, - "ok": False, - }) - continue - runtime = account.get("concurrency") - runtime_int = runtime if isinstance(runtime, int) else 0 - runtime_concurrency_total += runtime_int - if account.get("schedulable") is True: - runtime_status = account.get("status") - if runtime_status is None or runtime_status == "active": - runtime_status_ok = True - else: - runtime_status_ok = False - if runtime_status_ok: - schedulable_runtime_concurrency_total += runtime_int - temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil") - temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason") - if temp_until is None and not bool(temp_reason): - available_runtime_concurrency_total += runtime_int - else: - temp_unschedulable_runtime_concurrency_total += runtime_int - ok = runtime == expected - if not ok: - mismatched.append(name) - items.append({ - "accountName": name, - "accountId": account.get("id"), - "expectedCapacity": expected, - "runtimeConcurrency": runtime, - "priority": account.get("priority"), - "status": account.get("status"), - "schedulable": account.get("schedulable"), - "ok": ok, - }) - return { - "ok": len(missing) == 0 and len(mismatched) == 0, - "defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY, - "desired": len(EXPECTED_ACCOUNT_CAPACITIES), - "totals": { - "expectedCapacityTotal": expected_capacity_total, - "runtimeConcurrencyTotal": runtime_concurrency_total, - "schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total, - "availableRuntimeConcurrencyTotal": available_runtime_concurrency_total, - "tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total, - "minOwnerConcurrency": MIN_OWNER_CONCURRENCY, - "minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, - }, - "missing": missing, - "mismatched": mismatched, - "items": items, - "valuesPrinted": False, - } - -def account_load_factor_status(token): - accounts = list_accounts(token) - by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} - items = [] - missing = [] - mismatched = [] - for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS): - expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name]) - account = by_name.get(name) - if account is None: - missing.append(name) - items.append({ - "accountName": name, - "accountId": None, - "expectedLoadFactor": expected, - "runtimeLoadFactor": None, - "ok": False, - }) - continue - runtime_raw = account.get("load_factor") - if runtime_raw is None: - detail = get_account_detail(token, account) - runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None - try: - runtime = int(runtime_raw) - except Exception: - runtime = runtime_raw - ok = runtime == expected - if not ok: - mismatched.append(name) - items.append({ - "accountName": name, - "accountId": account.get("id"), - "expectedLoadFactor": expected, - "runtimeLoadFactor": runtime, - "priority": account.get("priority"), - "status": account.get("status"), - "schedulable": account.get("schedulable"), - "ok": ok, - }) - return { - "ok": len(missing) == 0 and len(mismatched) == 0, - "defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR, - "desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS), - "missing": missing, - "mismatched": mismatched, - "items": items, - "valuesPrinted": False, - } - -def account_ws_v2_status(token): - accounts = list_accounts(token) - by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} - items = [] - missing = [] - mismatched = [] - enabled_names = [] - unschedulable_enabled = [] - schedulable_enabled = [] - for name in sorted(EXPECTED_ACCOUNT_WS_MODES): - expected_mode = EXPECTED_ACCOUNT_WS_MODES[name] - expected_enabled = expected_mode not in (None, "", "off") - account = by_name.get(name) - if account is None: - missing.append(name) - items.append({ - "accountName": name, - "accountId": None, - "expectedMode": expected_mode, - "expectedEnabled": expected_enabled, - "runtimeMode": None, - "runtimeEnabled": None, - "ok": False, - }) - continue - extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} - runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode") - runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled") - schedulable = account.get("schedulable") - if expected_mode == "off": - ok = runtime_mode == "off" and runtime_enabled is False - elif expected_mode is None: - ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False) - else: - ok = runtime_mode == expected_mode and runtime_enabled is True - if not ok: - mismatched.append(name) - if expected_enabled: - enabled_names.append(name) - if schedulable is True: - schedulable_enabled.append(name) - else: - unschedulable_enabled.append(name) - items.append({ - "accountName": name, - "accountId": account.get("id"), - "expectedMode": expected_mode, - "expectedEnabled": expected_enabled, - "runtimeMode": runtime_mode, - "runtimeEnabled": runtime_enabled, - "status": account.get("status"), - "schedulable": schedulable, - "ok": ok and ((not expected_enabled) or schedulable is True), - }) - availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0 - return { - "ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok, - "desired": len(EXPECTED_ACCOUNT_WS_MODES), - "enabled": enabled_names, - "schedulableEnabled": schedulable_enabled, - "unschedulableEnabled": unschedulable_enabled, - "missing": missing, - "mismatched": mismatched, - "items": items, - "valuesPrinted": False, - } - -def api_key_preview(api_key): - if len(api_key) <= 14: - return "***" - return api_key[:10] + "..." + api_key[-4:] - -def secret_fingerprint(value): - return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest()[:16] - -def run_sync(): - global MANUAL_ACCOUNT_PROTECTIONS - payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) - manual_accounts_payload = payload.get("manualAccounts") if isinstance(payload.get("manualAccounts"), dict) else {} - resolved_manual_protections = manual_accounts_payload.get("protected") - if not isinstance(resolved_manual_protections, list): - raise RuntimeError("sync payload has no manualAccounts.protected binding plan") - MANUAL_ACCOUNT_PROTECTIONS = resolved_manual_protections - profiles = payload.get("profiles") or [] - prune_removed = bool(payload.get("pruneRemoved")) - sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {} - if not profiles and not MANUAL_ACCOUNT_PROTECTIONS: - raise RuntimeError("sync payload has no profiles and no manualAccounts.protected binding plan") - admin_email, token, admin_compliance = login() - group, group_action = ensure_group(token) - group_id = group.get("id") if isinstance(group, dict) else None - if group_id is None: - raise RuntimeError("pool group id missing after ensure") - existing_accounts = list_accounts(token) - planned_account_results = planned_sentinel_account_results(profiles, existing_accounts) - sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True) - if sentinel_quality_prepare.get("ok") is not True: - raise RuntimeError("prepare sentinel pending probe failed: " + json.dumps(sentinel_quality_prepare, ensure_ascii=False)) - protected_frozen_names = active_sentinel_quarantine_names() - account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed, protected_frozen_names, existing_accounts) - manual_account_proxy_bindings = ensure_manual_account_proxy_bindings(token) - manual_account_group_bindings = ensure_manual_account_group_bindings(token, group_id) - manual_account_protections = manual_account_protection_status(token, group_id) - capacity_status = account_capacity_status(token) - load_factor_status = account_load_factor_status(token) - ws_v2_status = account_ws_v2_status(token) - temp_unschedulable_status = account_temp_unschedulable_status(token) - api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id, token) - api_key_result = ensure_sub2api_api_key(token, api_key, group_id) - owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"]) - owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"]) - gateway = validate_gateway(api_key) - responses_smoke = validate_gateway_responses(api_key) - compact_evidence = recent_compact_gateway_evidence() - responses_evidence = recent_responses_gateway_evidence() - runtime_capabilities = validate_runtime_capabilities(token) - sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest")) - sentinel_quality = ensure_sentinel_state_for_sync(account_results) - sentinel_reassert = reassert_sentinel_freezes_after_sync(token) - return { - "ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_proxy_bindings.get("ok") is True and manual_account_group_bindings.get("ok") is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and sentinel_quality_prepare.get("ok") is True and sentinel_quality.get("ok") is True and sentinel_reassert.get("ok") is True and runtime_capabilities.get("ok") is True, - "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True, - "mode": "sync", - "namespace": NAMESPACE, - "serviceDns": SERVICE_DNS, - "appPod": APP_POD, - "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, - "pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"}, - "accounts": { - "desired": len(profiles), - "created": sum(1 for item in account_results if item["action"] == "created"), - "updated": sum(1 for item in account_results if item["action"] == "updated"), - "pruned": len(pruned_account_results), - "pruneMode": "explicit" if prune_removed else "disabled-by-default", - "items": account_results, - "prunedItems": pruned_account_results, - "processControl": {"schedulableRestore": "sentinel marker probe only; sync does not restore schedulable for existing accounts", "durableConfig": False}, - "valuesPrinted": False, - }, - "manualAccounts": {**manual_account_protections, "proxySync": manual_account_proxy_bindings, "groupSync": manual_account_group_bindings}, - "capacity": capacity_status, - "loadFactor": load_factor_status, - "webSocketsV2": ws_v2_status, - "tempUnschedulable": temp_unschedulable_status, - "apiKey": { - "ok": True, - "name": POOL_API_KEY_NAME, - "secret": pool_api_key_secret_location(), - "secretAction": secret_action, - "secretApply": secret_apply_stdout, - "sub2apiAction": api_key_result["action"], - "sub2apiId": api_key_result["id"], - "groupId": api_key_result["groupId"], - "userId": api_key_result["userId"], - "apiKeyFingerprint": secret_fingerprint(api_key), - "valuesPrinted": False, - }, - "ownerBalance": owner_balance, - "ownerConcurrency": owner_concurrency, - "sentinel": {**sentinel, "qualityGatePrepare": sentinel_quality_prepare, "qualityGate": sentinel_quality, "freezeReassert": sentinel_reassert}, - "runtimeCapabilities": runtime_capabilities, - "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, - } - -def run_validate(): - admin_email, token, admin_compliance = login() - api_key, key_item, api_key_source = resolve_pool_api_key_for_validate(token) - api_key_ok = isinstance(key_item, dict) and key_item.get("name") == POOL_API_KEY_NAME - owner_balance = None - owner_concurrency = None - if key_item is not None and key_item.get("user_id") is not None: - owner_balance = ensure_pool_owner_balance(token, key_item["user_id"]) - owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"]) - capacity_status = account_capacity_status(token) - load_factor_status = account_load_factor_status(token) - ws_v2_status = account_ws_v2_status(token) - temp_unschedulable_status = account_temp_unschedulable_status(token) - pool_group_id = key_item.get("group_id") if isinstance(key_item, dict) else None - manual_account_protections = manual_account_protection_status(token, pool_group_id) - gateway = validate_gateway(api_key) - responses_smoke = validate_gateway_responses(api_key) - compact_evidence = recent_compact_gateway_evidence() - responses_evidence = recent_responses_gateway_evidence() - runtime_capabilities = validate_runtime_capabilities(token) - sentinel = sentinel_runtime_status() - return { - "ok": api_key_ok is True and gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and runtime_capabilities.get("ok") is True, - "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True, - "mode": "validate", - "namespace": NAMESPACE, - "serviceDns": SERVICE_DNS, - "appPod": APP_POD, - "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, - "apiKey": { - "ok": api_key_ok, - "name": POOL_API_KEY_NAME, - "secret": pool_api_key_secret_location(), - "source": api_key_source, - "sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None, - "userId": key_item.get("user_id") if isinstance(key_item, dict) else None, - "groupId": key_item.get("group_id") if isinstance(key_item, dict) else None, - "apiKeyFingerprint": secret_fingerprint(api_key), - "valuesPrinted": False, - }, - "ownerBalance": owner_balance, - "ownerConcurrency": owner_concurrency, - "capacity": capacity_status, - "loadFactor": load_factor_status, - "webSocketsV2": ws_v2_status, - "tempUnschedulable": temp_unschedulable_status, - "manualAccounts": manual_account_protections, - "sentinel": sentinel, - "runtimeCapabilities": runtime_capabilities, - "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, - } - -def parse_log_line(line): - json_start = line.find("{") - if json_start < 0: - return None - prefix = line[:json_start].rstrip() - try: - item = json.loads(line[json_start:]) - except Exception: - return None - if not isinstance(item, dict): - return None - at = None - parts = prefix.split() - if parts: - at = parts[0] - message = "" - if len(parts) >= 4: - message = " ".join(parts[3:]) - elif len(parts) >= 3: - message = parts[2] - elif len(parts) >= 1: - message = parts[-1] - item["_at"] = at - item["_message"] = message - item["_line"] = line - return item - -def log_time_epoch(item): - at = item.get("_at") if isinstance(item, dict) else None - if not isinstance(at, str) or not at: - return None - try: - return datetime.strptime(at, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp() - except Exception: - try: - return datetime.fromisoformat(at.replace("Z", "+00:00")).timestamp() - except Exception: - return None - -def event_base(item, account_names_by_id): - account_id = item.get("account_id") - if isinstance(account_id, str) and account_id.isdigit(): - account_id = int(account_id) - account_name = account_names_by_id.get(account_id) - return { - "at": item.get("_at"), - "message": item.get("_message"), - "requestId": item.get("request_id"), - "clientRequestId": item.get("client_request_id"), - "path": item.get("path"), - "method": item.get("method"), - "model": item.get("model"), - "accountId": account_id, - "accountName": account_name, - "statusCode": item.get("status_code"), - "upstreamStatus": item.get("upstream_status"), - "latencyMs": item.get("latency_ms"), - } - -def classify_trace_event(item, account_names_by_id): - message = str(item.get("_message") or "") - event = event_base(item, account_names_by_id) - if "content_moderation.gateway_check_start" in message: - event.update({ - "type": "request-start", - "stream": item.get("stream"), - "bodyBytes": item.get("body_bytes"), - "groupId": item.get("group_id"), - "groupName": item.get("group_name"), - "apiKeyName": item.get("api_key_name"), - }) - elif "content_moderation.gateway_check_done" in message: - event.update({ - "type": "gateway-check", - "allowed": item.get("allowed"), - "blocked": item.get("blocked"), - "action": item.get("action"), - }) - elif "openai.upstream_failover_switching" in message: - event.update({ - "type": "failover", - "switchCount": item.get("switch_count"), - "maxSwitches": item.get("max_switches"), - }) - elif "openai.account_select_failed" in message: - event.update({ - "type": "select-failed", - "error": item.get("error"), - "excludedAccountCount": item.get("excluded_account_count"), - }) - elif "account_upstream_error" in message: - event.update({ - "type": "upstream-error", - "error": item.get("error"), - }) - elif "account_temp_unschedulable" in message: - event.update({ - "type": "temp-unschedulable", - "until": item.get("until") or item.get("temp_unschedulable_until"), - "ruleIndex": item.get("rule_index"), - "matchedKeyword": item.get("matched_keyword"), - "reason": item.get("reason") or item.get("error"), - }) - elif "http request completed" in message: - event.update({ - "type": "final", - "clientIp": item.get("client_ip"), - "protocol": item.get("protocol"), - "platform": item.get("platform"), - "completedAt": item.get("completed_at"), - }) - elif "admin account schedulable updated" in message or "account schedulable updated" in message or "/schedulable" in str(item.get("path") or ""): - event.update({ - "type": "admin-schedulable", - "schedulable": item.get("schedulable"), - }) - else: - event.update({"type": "other"}) - return event - -def with_trace_phase(event, first_epoch, last_epoch): - epoch = None - at = event.get("at") if isinstance(event, dict) else None - if isinstance(at, str) and at: - epoch = log_time_epoch({"_at": at}) - if epoch is None or first_epoch is None: - phase = "unknown" - elif epoch < first_epoch: - phase = "before-request" - elif last_epoch is not None and epoch > last_epoch: - phase = "after-request" - else: - phase = "during-request" - event["phase"] = phase - return event - -def account_snapshot_from_runtime(token): - try: - accounts = list_accounts(token) - except Exception as exc: - return [], {"error": str(exc)} - rows = [] - for item in accounts: - if not isinstance(item, dict): - continue - rows.append({ - "accountId": item.get("id"), - "accountName": item.get("name"), - "schedulable": item.get("schedulable"), - "status": item.get("status"), - "concurrency": item.get("concurrency"), - "priority": item.get("priority"), - "tempUnschedulableUntil": item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil"), - "tempUnschedulableSet": (item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil")) is not None or bool(item.get("temp_unschedulable_reason") or item.get("tempUnschedulableReason")), - }) - rows.sort(key=lambda row: (str(row.get("accountName") or ""), int(row.get("accountId") or 0))) - return rows, None - -def trace_reason(events, final_event): - failovers = [item for item in events if item.get("type") == "failover"] - select_failures = [item for item in events if item.get("type") == "select-failed"] - upstream_errors = [item for item in events if item.get("type") == "upstream-error"] - if failover_budget_exhausted(failovers, final_event): - return "failover-budget-exhausted" - if failovers and select_failures: - return "failover-attempted-no-candidate" - if failovers: - return "failover-attempted" - if select_failures: - return "account-select-failed" - if upstream_errors: - return "upstream-error" - if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") >= 400: - return "final-http-error" - if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int): - return "completed" - return "unknown" - -def failover_budget_exhausted(failovers, final_event): - if not failovers or not isinstance(final_event, dict): - return False - last = failovers[-1] - switch_count = last.get("switchCount") - max_switches = last.get("maxSwitches") - final_status = final_event.get("statusCode") - return ( - isinstance(switch_count, int) - and isinstance(max_switches, int) - and max_switches > 0 - and switch_count >= max_switches - and isinstance(final_status, int) - and final_status >= 500 - ) - -def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot): - if not failover_budget_exhausted(failovers, final_event): - return [] - tried = set() - for item in failovers: - account_id = item.get("accountId") - if isinstance(account_id, int): - tried.add(account_id) - final_account = final_event.get("accountId") if isinstance(final_event, dict) else None - if isinstance(final_account, int): - tried.add(final_account) - result = [] - for item in account_snapshot: - account_id = item.get("accountId") - if not isinstance(account_id, int) or account_id in tried: - continue - if item.get("schedulable") is True and item.get("status") == "active" and item.get("tempUnschedulableSet") is not True: - result.append({ - "accountId": account_id, - "accountName": item.get("accountName"), - "priority": item.get("priority"), - "concurrency": item.get("concurrency"), - }) - return result - -def run_trace(): - payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} - request_id = payload.get("requestId") - since = payload.get("since") or "24h" - tail = int(payload.get("tail") or 20000) - context_seconds = int(payload.get("contextSeconds") or 300) - show_lines = bool(payload.get("showLines")) - if not isinstance(request_id, str) or not request_id: - raise RuntimeError("trace payload missing requestId") - admin_email, token, admin_compliance = login() - account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token) - account_names_by_id = {} - for row in account_snapshot: - account_id = row.get("accountId") - if isinstance(account_id, str) and account_id.isdigit(): - account_id = int(account_id) - if isinstance(account_id, int) and isinstance(row.get("accountName"), str): - account_names_by_id[account_id] = row.get("accountName") - proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) - stdout = proc.stdout.decode("utf-8", errors="replace") - parsed_lines = [] - matched = [] - for line in stdout.splitlines(): - parsed = parse_log_line(line) - if parsed is None: - continue - parsed_lines.append(parsed) - if request_id in line: - matched.append(parsed) - first_epoch = None - last_epoch = None - for item in matched: - epoch = log_time_epoch(item) - if epoch is None: - continue - first_epoch = epoch if first_epoch is None else min(first_epoch, epoch) - last_epoch = epoch if last_epoch is None else max(last_epoch, epoch) - window_lines = [] - if first_epoch is not None: - start_epoch = first_epoch - context_seconds - end_epoch = (last_epoch if last_epoch is not None else first_epoch) + context_seconds - for item in parsed_lines: - epoch = log_time_epoch(item) - if epoch is not None and start_epoch <= epoch <= end_epoch: - window_lines.append(item) - else: - window_lines = matched - events = [classify_trace_event(item, account_names_by_id) for item in matched] - request_start = next((item for item in events if item.get("type") == "request-start"), None) - final_event = next((item for item in reversed(events) if item.get("type") == "final"), None) - failovers = [item for item in events if item.get("type") == "failover"] - select_failures = [item for item in events if item.get("type") == "select-failed"] - upstream_errors = [item for item in events if item.get("type") == "upstream-error"] - temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")] - admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))] - window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines] - final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400] - window_failovers = [item for item in window_events if item.get("type") == "failover"] - window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] - untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) - reason = trace_reason(events, final_event) - if not matched: - outcome = "not-found" - elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: - outcome = "succeeded" - elif isinstance(final_event, dict): - outcome = "failed" - else: - outcome = "incomplete" - return { - "ok": proc.returncode == 0 and len(matched) > 0, - "mode": "trace", - "namespace": NAMESPACE, - "serviceDns": SERVICE_DNS, - "appPod": APP_POD, - "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, - "requestId": request_id, - "summary": { - "outcome": outcome, - "reason": reason, - "eventCount": len(events), - "matchedLineCount": len(matched), - "firstAt": events[0].get("at") if events else None, - "lastAt": events[-1].get("at") if events else None, - }, - "window": { - "since": since, - "tail": tail, - "beforeSeconds": context_seconds, - "afterSeconds": context_seconds, - "lineCount": len(window_lines), - }, - "request": request_start or {}, - "final": final_event or {}, - "events": events, - "failovers": failovers, - "selectFailures": select_failures, - "upstreamErrors": upstream_errors, - "tempUnschedulable": temp_unsched, - "adminSchedulable": admin_sched[-20:], - "windowStats": { - "matchedLines": len(matched), - "eventCount": len(window_events), - "finalErrorCount": len(final_errors), - "failoverCount": len(window_failovers), - "selectFailedCount": len(window_select_failures), - "tempUnschedulableCount": len(temp_unsched), - "adminSchedulableCount": len(admin_sched), - }, - "diagnostics": { - "failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}), - "untriedSchedulableAccounts": untried_schedulable_accounts, - }, - "accountSnapshot": account_snapshot, - "accountSnapshotError": account_snapshot_error, - "rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [], - "showLines": show_lines, - "logs": { - "exitCode": proc.returncode, - "stderrTail": text(proc.stderr, 1000), - "stdoutLineCount": len(stdout.splitlines()), - }, - "valuesPrinted": False, - } - -def parse_embedded_json(stdout): - if not isinstance(stdout, str) or not stdout.strip(): - return None - start = stdout.find("{") - end = stdout.rfind("}") - if start < 0 or end <= start: - return None - try: - return json.loads(stdout[start:end + 1]) - except Exception: - return None - -def inject_env(template, name, value): - spec = template.setdefault("spec", {}) - containers = spec.setdefault("containers", []) - if not containers: - raise RuntimeError("sentinel job template has no containers") - container = containers[0] - env = container.setdefault("env", []) - env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)] - env.append({"name": name, "value": value}) - -def sentinel_probe_job_manifest(accounts): - cronjob_name = SENTINEL_CONFIG.get("cronJobName") - if not isinstance(cronjob_name, str) or not cronjob_name: - raise RuntimeError("sentinel cronJobName missing from config") - cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}") - job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {} - job_spec = copy_json(job_template.get("spec") or {}) - if not isinstance(job_spec, dict) or not job_spec: - raise RuntimeError("sentinel CronJob jobTemplate.spec missing") - template = job_spec.get("template") - if not isinstance(template, dict): - raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing") - inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts)) - job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600) - suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5)) - job_name = ("sub2api-sentinel-probe-" + suffix)[:63] - return job_name, { - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": job_name, - "namespace": NAMESPACE, - "labels": { - "app.kubernetes.io/name": cronjob_name, - "app.kubernetes.io/part-of": "platform-infra", - "app.kubernetes.io/managed-by": "unidesk", - "unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe", - }, - }, - "spec": job_spec, - } - -def copy_json(value): - return json.loads(json.dumps(value)) - -def job_condition(job, cond_type): - for item in ((job.get("status") or {}).get("conditions") or []): - if item.get("type") == cond_type and item.get("status") == "True": - return item - return None - -def wait_sentinel_probe_job(job_name, timeout_seconds): - deadline = time.time() + timeout_seconds - latest = None - while time.time() < deadline: - latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}") - if isinstance(latest, dict): - complete = job_condition(latest, "Complete") - failed = job_condition(latest, "Failed") - if complete is not None: - return "succeeded", latest, complete - if failed is not None: - return "failed", latest, failed - time.sleep(2) - return "timeout", latest, None - -def job_logs(job_name): - proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"]) - return { - "exitCode": proc.returncode, - "stdout": proc.stdout.decode("utf-8", errors="replace"), - "stderr": text(proc.stderr, 4000), - } - -def run_sentinel_probe(): - payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} - accounts = payload.get("accounts") if isinstance(payload, dict) else None - if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts): - raise RuntimeError("sentinel-probe payload requires non-empty accounts") - job_name, manifest = sentinel_probe_job_manifest(accounts) - proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) - if proc.returncode != 0: - raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}") - timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240) - status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds) - logs = job_logs(job_name) - parsed = parse_embedded_json(logs.get("stdout") or "") - results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else [] - state_summary = sentinel_state_summary() - last_run = state_summary.get("lastRun") if isinstance(state_summary, dict) and isinstance(state_summary.get("lastRun"), dict) else {} - if not results and isinstance(last_run.get("results"), list): - results = last_run.get("results") - requested = set(accounts) - measured = {item.get("accountName") for item in results if isinstance(item, dict)} - missing = sorted(name for name in requested if name not in measured) - marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested) - if not results and isinstance(last_run, dict): - selected = int(last_run.get("selected") or 0) - ok_count = int(last_run.get("okCount") or 0) - marker_mismatch_count = int(last_run.get("markerMismatchCount") or 0) - transport_failure_count = int(last_run.get("transportFailureCount") or 0) - if selected >= len(requested) and ok_count >= len(requested) and marker_mismatch_count == 0 and transport_failure_count == 0: - missing = [] - marker_ok = True - job_ok = status == "succeeded" and logs.get("exitCode") == 0 - return { - "ok": job_ok and marker_ok, - "jobExecutionOk": job_ok, - "markerOk": marker_ok, - "mode": "sentinel-probe", - "namespace": NAMESPACE, - "requestedAccounts": accounts, - "missingAccounts": missing, - "job": { - "name": job_name, - "status": status, - "condition": condition, - "timeoutSeconds": timeout_seconds, - "applyStdoutTail": text(proc.stdout, 1200), - "logsExitCode": logs.get("exitCode"), - "logsStderrTail": logs.get("stderr"), - }, - "probe": parsed, - "summary": { - "at": last_run.get("at"), - "selected": last_run.get("selected"), - "okCount": last_run.get("okCount"), - "markerMismatchCount": last_run.get("markerMismatchCount"), - "transportFailureCount": last_run.get("transportFailureCount"), - "actionsTaken": last_run.get("actionsTaken"), - "selection": last_run.get("selection"), - }, - "results": results, - "sentinelState": state_summary, - "valuesPrinted": False, - } - -def run_cleanup_probes(): - admin_email, token, admin_compliance = login() - cleanup = cleanup_probe_resources(token) - return { - "ok": cleanup.get("ok") is True, - "mode": "cleanup-probes", - "namespace": NAMESPACE, - "serviceDns": SERVICE_DNS, - "appPod": APP_POD, - "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, - "cleanup": cleanup, - "valuesPrinted": False, - } - -try: - if MODE == "sync": - result = run_sync() - elif MODE == "trace": - result = run_trace() - elif MODE == "cleanup-probes": - result = run_cleanup_probes() - elif MODE == "sentinel-probe": - result = run_sentinel_probe() - else: - result = run_validate() -except Exception as exc: - result = { - "ok": False, - "mode": MODE, - "namespace": NAMESPACE, - "serviceDns": SERVICE_DNS, - "appPod": globals().get("APP_POD"), - "error": str(exc), - "valuesPrinted": False, - } - -print(json.dumps(result, ensure_ascii=False, indent=2)) -sys.exit(0 if result.get("ok") else 1) -PY -`; + return [ + remotePythonCoreScript(mode, encodedPayload, pool, target), + remotePythonSentinelScript, + remotePythonValidationScript, + remotePythonSyncValidateScript, + remotePythonTraceScript, + remotePythonSentinelProbeScript, + ].join(""); } diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-trace.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-trace.ts new file mode 100644 index 00000000..84ef6a08 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-trace.ts @@ -0,0 +1,345 @@ +export const remotePythonTraceScript = `def parse_log_line(line): + json_start = line.find("{") + if json_start < 0: + return None + prefix = line[:json_start].rstrip() + try: + item = json.loads(line[json_start:]) + except Exception: + return None + if not isinstance(item, dict): + return None + at = None + parts = prefix.split() + if parts: + at = parts[0] + message = "" + if len(parts) >= 4: + message = " ".join(parts[3:]) + elif len(parts) >= 3: + message = parts[2] + elif len(parts) >= 1: + message = parts[-1] + item["_at"] = at + item["_message"] = message + item["_line"] = line + return item + +def log_time_epoch(item): + at = item.get("_at") if isinstance(item, dict) else None + if not isinstance(at, str) or not at: + return None + try: + return datetime.strptime(at, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp() + except Exception: + try: + return datetime.fromisoformat(at.replace("Z", "+00:00")).timestamp() + except Exception: + return None + +def event_base(item, account_names_by_id): + account_id = item.get("account_id") + if isinstance(account_id, str) and account_id.isdigit(): + account_id = int(account_id) + account_name = account_names_by_id.get(account_id) + return { + "at": item.get("_at"), + "message": item.get("_message"), + "requestId": item.get("request_id"), + "clientRequestId": item.get("client_request_id"), + "path": item.get("path"), + "method": item.get("method"), + "model": item.get("model"), + "accountId": account_id, + "accountName": account_name, + "statusCode": item.get("status_code"), + "upstreamStatus": item.get("upstream_status"), + "latencyMs": item.get("latency_ms"), + } + +def classify_trace_event(item, account_names_by_id): + message = str(item.get("_message") or "") + event = event_base(item, account_names_by_id) + if "content_moderation.gateway_check_start" in message: + event.update({ + "type": "request-start", + "stream": item.get("stream"), + "bodyBytes": item.get("body_bytes"), + "groupId": item.get("group_id"), + "groupName": item.get("group_name"), + "apiKeyName": item.get("api_key_name"), + }) + elif "content_moderation.gateway_check_done" in message: + event.update({ + "type": "gateway-check", + "allowed": item.get("allowed"), + "blocked": item.get("blocked"), + "action": item.get("action"), + }) + elif "openai.upstream_failover_switching" in message: + event.update({ + "type": "failover", + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + elif "openai.account_select_failed" in message: + event.update({ + "type": "select-failed", + "error": item.get("error"), + "excludedAccountCount": item.get("excluded_account_count"), + }) + elif "account_upstream_error" in message: + event.update({ + "type": "upstream-error", + "error": item.get("error"), + }) + elif "account_temp_unschedulable" in message: + event.update({ + "type": "temp-unschedulable", + "until": item.get("until") or item.get("temp_unschedulable_until"), + "ruleIndex": item.get("rule_index"), + "matchedKeyword": item.get("matched_keyword"), + "reason": item.get("reason") or item.get("error"), + }) + elif "http request completed" in message: + event.update({ + "type": "final", + "clientIp": item.get("client_ip"), + "protocol": item.get("protocol"), + "platform": item.get("platform"), + "completedAt": item.get("completed_at"), + }) + elif "admin account schedulable updated" in message or "account schedulable updated" in message or "/schedulable" in str(item.get("path") or ""): + event.update({ + "type": "admin-schedulable", + "schedulable": item.get("schedulable"), + }) + else: + event.update({"type": "other"}) + return event + +def with_trace_phase(event, first_epoch, last_epoch): + epoch = None + at = event.get("at") if isinstance(event, dict) else None + if isinstance(at, str) and at: + epoch = log_time_epoch({"_at": at}) + if epoch is None or first_epoch is None: + phase = "unknown" + elif epoch < first_epoch: + phase = "before-request" + elif last_epoch is not None and epoch > last_epoch: + phase = "after-request" + else: + phase = "during-request" + event["phase"] = phase + return event + +def account_snapshot_from_runtime(token): + try: + accounts = list_accounts(token) + except Exception as exc: + return [], {"error": str(exc)} + rows = [] + for item in accounts: + if not isinstance(item, dict): + continue + rows.append({ + "accountId": item.get("id"), + "accountName": item.get("name"), + "schedulable": item.get("schedulable"), + "status": item.get("status"), + "concurrency": item.get("concurrency"), + "priority": item.get("priority"), + "tempUnschedulableUntil": item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil"), + "tempUnschedulableSet": (item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil")) is not None or bool(item.get("temp_unschedulable_reason") or item.get("tempUnschedulableReason")), + }) + rows.sort(key=lambda row: (str(row.get("accountName") or ""), int(row.get("accountId") or 0))) + return rows, None + +def trace_reason(events, final_event): + failovers = [item for item in events if item.get("type") == "failover"] + select_failures = [item for item in events if item.get("type") == "select-failed"] + upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + if failover_budget_exhausted(failovers, final_event): + return "failover-budget-exhausted" + if failovers and select_failures: + return "failover-attempted-no-candidate" + if failovers: + return "failover-attempted" + if select_failures: + return "account-select-failed" + if upstream_errors: + return "upstream-error" + if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") >= 400: + return "final-http-error" + if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int): + return "completed" + return "unknown" + +def failover_budget_exhausted(failovers, final_event): + if not failovers or not isinstance(final_event, dict): + return False + last = failovers[-1] + switch_count = last.get("switchCount") + max_switches = last.get("maxSwitches") + final_status = final_event.get("statusCode") + return ( + isinstance(switch_count, int) + and isinstance(max_switches, int) + and max_switches > 0 + and switch_count >= max_switches + and isinstance(final_status, int) + and final_status >= 500 + ) + +def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot): + if not failover_budget_exhausted(failovers, final_event): + return [] + tried = set() + for item in failovers: + account_id = item.get("accountId") + if isinstance(account_id, int): + tried.add(account_id) + final_account = final_event.get("accountId") if isinstance(final_event, dict) else None + if isinstance(final_account, int): + tried.add(final_account) + result = [] + for item in account_snapshot: + account_id = item.get("accountId") + if not isinstance(account_id, int) or account_id in tried: + continue + if item.get("schedulable") is True and item.get("status") == "active" and item.get("tempUnschedulableSet") is not True: + result.append({ + "accountId": account_id, + "accountName": item.get("accountName"), + "priority": item.get("priority"), + "concurrency": item.get("concurrency"), + }) + return result + +def run_trace(): + payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} + request_id = payload.get("requestId") + since = payload.get("since") or "24h" + tail = int(payload.get("tail") or 20000) + context_seconds = int(payload.get("contextSeconds") or 300) + show_lines = bool(payload.get("showLines")) + if not isinstance(request_id, str) or not request_id: + raise RuntimeError("trace payload missing requestId") + admin_email, token, admin_compliance = login() + account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token) + account_names_by_id = {} + for row in account_snapshot: + account_id = row.get("accountId") + if isinstance(account_id, str) and account_id.isdigit(): + account_id = int(account_id) + if isinstance(account_id, int) and isinstance(row.get("accountName"), str): + account_names_by_id[account_id] = row.get("accountName") + proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) + stdout = proc.stdout.decode("utf-8", errors="replace") + parsed_lines = [] + matched = [] + for line in stdout.splitlines(): + parsed = parse_log_line(line) + if parsed is None: + continue + parsed_lines.append(parsed) + if request_id in line: + matched.append(parsed) + first_epoch = None + last_epoch = None + for item in matched: + epoch = log_time_epoch(item) + if epoch is None: + continue + first_epoch = epoch if first_epoch is None else min(first_epoch, epoch) + last_epoch = epoch if last_epoch is None else max(last_epoch, epoch) + window_lines = [] + if first_epoch is not None: + start_epoch = first_epoch - context_seconds + end_epoch = (last_epoch if last_epoch is not None else first_epoch) + context_seconds + for item in parsed_lines: + epoch = log_time_epoch(item) + if epoch is not None and start_epoch <= epoch <= end_epoch: + window_lines.append(item) + else: + window_lines = matched + events = [classify_trace_event(item, account_names_by_id) for item in matched] + request_start = next((item for item in events if item.get("type") == "request-start"), None) + final_event = next((item for item in reversed(events) if item.get("type") == "final"), None) + failovers = [item for item in events if item.get("type") == "failover"] + select_failures = [item for item in events if item.get("type") == "select-failed"] + upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")] + admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))] + window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines] + final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400] + window_failovers = [item for item in window_events if item.get("type") == "failover"] + window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] + untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) + reason = trace_reason(events, final_event) + if not matched: + outcome = "not-found" + elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: + outcome = "succeeded" + elif isinstance(final_event, dict): + outcome = "failed" + else: + outcome = "incomplete" + return { + "ok": proc.returncode == 0 and len(matched) > 0, + "mode": "trace", + "namespace": NAMESPACE, + "serviceDns": SERVICE_DNS, + "appPod": APP_POD, + "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, + "requestId": request_id, + "summary": { + "outcome": outcome, + "reason": reason, + "eventCount": len(events), + "matchedLineCount": len(matched), + "firstAt": events[0].get("at") if events else None, + "lastAt": events[-1].get("at") if events else None, + }, + "window": { + "since": since, + "tail": tail, + "beforeSeconds": context_seconds, + "afterSeconds": context_seconds, + "lineCount": len(window_lines), + }, + "request": request_start or {}, + "final": final_event or {}, + "events": events, + "failovers": failovers, + "selectFailures": select_failures, + "upstreamErrors": upstream_errors, + "tempUnschedulable": temp_unsched, + "adminSchedulable": admin_sched[-20:], + "windowStats": { + "matchedLines": len(matched), + "eventCount": len(window_events), + "finalErrorCount": len(final_errors), + "failoverCount": len(window_failovers), + "selectFailedCount": len(window_select_failures), + "tempUnschedulableCount": len(temp_unsched), + "adminSchedulableCount": len(admin_sched), + }, + "diagnostics": { + "failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}), + "untriedSchedulableAccounts": untried_schedulable_accounts, + }, + "accountSnapshot": account_snapshot, + "accountSnapshotError": account_snapshot_error, + "rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [], + "showLines": show_lines, + "logs": { + "exitCode": proc.returncode, + "stderrTail": text(proc.stderr, 1000), + "stdoutLineCount": len(stdout.splitlines()), + }, + "valuesPrinted": False, + } + +`; diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-validation.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-validation.ts new file mode 100644 index 00000000..fb5ced1d --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-validation.ts @@ -0,0 +1,809 @@ +export const remotePythonValidationScript = `def validate_gateway(api_key): + resp = curl_api("GET", "/v1/models", bearer=api_key) + parsed = resp.get("json") + model_count = None + if isinstance(parsed, dict) and isinstance(parsed.get("data"), list): + model_count = len(parsed["data"]) + return { + "ok": resp.get("ok"), + "httpStatus": resp.get("httpStatus"), + "transportExitCode": resp.get("transportExitCode"), + "modelCount": model_count, + "bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "", + "stderr": resp.get("stderr", ""), + "method": "GET /v1/models", + "serviceDns": SERVICE_DNS, + "valuesPrinted": False, + } + +def response_output_preview(parsed): + if not isinstance(parsed, dict): + return "" + if isinstance(parsed.get("output_text"), str): + return parsed["output_text"][:240] + output = parsed.get("output") + if not isinstance(output, list): + return "" + parts = [] + for item in output: + if not isinstance(item, dict): + continue + content = item.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + text_value = block.get("text") + if isinstance(text_value, str) and text_value: + parts.append(text_value) + return "\\n".join(parts)[:240] + +def request_log_evidence(request_id): + proc = runtime_logs("5m", 800) + stdout = proc.stdout.decode("utf-8", errors="replace") + lines = [line for line in stdout.splitlines() if request_id in line] + failovers = [] + final = None + for line in lines: + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except Exception: + continue + if "upstream_failover_switching" in line: + failovers.append({ + "accountId": item.get("account_id"), + "upstreamStatus": item.get("upstream_status"), + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + if "http request completed" in line: + final = { + "accountId": item.get("account_id"), + "statusCode": item.get("status_code"), + "latencyMs": item.get("latency_ms"), + "path": item.get("path"), + } + return { + "requestId": request_id, + "matchedLogLineCount": len(lines), + "failovers": failovers, + "final": final, + "logsExitCode": proc.returncode, + "logsStderr": text(proc.stderr, 1000), + } + +def recent_compact_gateway_evidence(): + proc = runtime_logs("6h", 2500) + stdout = proc.stdout.decode("utf-8", errors="replace") + failures = [] + successes = [] + failovers = [] + final_errors = [] + context_canceled = [] + for line in stdout.splitlines(): + if "/responses/compact" not in line and "remote_compact" not in line: + continue + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except Exception: + continue + path = item.get("path") + entry = { + "requestId": item.get("request_id"), + "clientRequestId": item.get("client_request_id"), + "accountId": item.get("account_id"), + "statusCode": item.get("status_code"), + "upstreamStatus": item.get("upstream_status"), + "latencyMs": item.get("latency_ms"), + "path": path, + } + if "codex.remote_compact.failed" in line: + failures.append(entry) + elif "codex.remote_compact.succeeded" in line: + successes.append(entry) + elif "upstream_failover_switching" in line and path == "/responses/compact": + failovers.append({ + **entry, + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400: + final_errors.append(entry) + if "context canceled" in line and path == "/responses/compact": + context_canceled.append(entry) + return { + "ok": True, + "degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0, + "window": "6h", + "tailLines": 2500, + "failureCount": len(failures), + "successCount": len(successes), + "failoverCount": len(failovers), + "finalErrorCount": len(final_errors), + "contextCanceledCount": len(context_canceled), + "recentFailures": failures[-5:], + "recentSuccesses": successes[-5:], + "recentFailovers": failovers[-8:], + "recentFinalErrors": final_errors[-5:], + "recentContextCanceled": context_canceled[-5:], + "logsExitCode": proc.returncode, + "logsStderr": text(proc.stderr, 1000), + "valuesPrinted": False, + } + +def is_ignored_probe_noise(entry): + for value in (entry.get("requestId"), entry.get("clientRequestId")): + if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"): + return True + return False + +def filter_ignored_probe_noise(items): + return [item for item in items if not is_ignored_probe_noise(item)] + +def ignored_probe_noise(items, section): + result = [] + for item in items: + if is_ignored_probe_noise(item): + probe = dict(item) + probe["section"] = section + result.append(probe) + return result + +def group_by_request_id(items): + grouped = {} + for item in items: + request_id = item.get("requestId") + if not isinstance(request_id, str) or not request_id: + continue + grouped.setdefault(request_id, []).append(item) + return grouped + +def failover_budget_exhausted_evidence(failovers, final_errors): + final_by_request = {} + for item in final_errors: + request_id = item.get("requestId") + if isinstance(request_id, str) and request_id: + final_by_request[request_id] = item + exhausted = [] + for request_id, request_failovers in group_by_request_id(failovers).items(): + final = final_by_request.get(request_id) + if not final: + continue + last = request_failovers[-1] + switch_count = last.get("switchCount") + max_switches = last.get("maxSwitches") + final_status = final.get("statusCode") + if ( + isinstance(switch_count, int) + and isinstance(max_switches, int) + and max_switches > 0 + and switch_count >= max_switches + and isinstance(final_status, int) + and final_status >= 500 + ): + exhausted.append({ + "requestId": request_id, + "clientRequestId": final.get("clientRequestId") or last.get("clientRequestId"), + "path": final.get("path") or last.get("path"), + "finalAccountId": final.get("accountId"), + "finalStatusCode": final_status, + "switchCount": switch_count, + "maxSwitches": max_switches, + "lastFailoverAccountId": last.get("accountId"), + "lastUpstreamStatus": last.get("upstreamStatus"), + }) + return exhausted + +def recent_responses_gateway_evidence(): + proc = runtime_logs("6h", 2500) + stdout = proc.stdout.decode("utf-8", errors="replace") + failovers = [] + forward_failures = [] + final_errors = [] + context_canceled = [] + slow_final_errors = [] + for line in stdout.splitlines(): + if '"/responses"' not in line and '"/v1/responses"' not in line: + continue + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except Exception: + continue + path = item.get("path") + if path not in ("/responses", "/v1/responses"): + continue + entry = { + "requestId": item.get("request_id"), + "clientRequestId": item.get("client_request_id"), + "accountId": item.get("account_id"), + "statusCode": item.get("status_code"), + "upstreamStatus": item.get("upstream_status"), + "latencyMs": item.get("latency_ms"), + "path": path, + } + if "upstream_failover_switching" in line: + failovers.append({ + **entry, + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + elif "openai.forward_failed" in line: + forward_failures.append({ + **entry, + "errorPreview": text(str(item.get("error") or ""), 500), + "fallbackErrorResponseWritten": item.get("fallback_error_response_written"), + "upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"), + }) + elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400: + final_errors.append(entry) + latency_ms = item.get("latency_ms") + if isinstance(latency_ms, int) and latency_ms >= 30000: + slow_final_errors.append(entry) + if "context canceled" in line: + context_canceled.append(entry) + visible_failovers = filter_ignored_probe_noise(failovers) + visible_forward_failures = filter_ignored_probe_noise(forward_failures) + visible_final_errors = filter_ignored_probe_noise(final_errors) + visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors) + visible_context_canceled = filter_ignored_probe_noise(context_canceled) + failover_budget_exhausted = failover_budget_exhausted_evidence(visible_failovers, visible_final_errors) + probe_noise = ( + ignored_probe_noise(failovers, "failovers") + + ignored_probe_noise(forward_failures, "forwardFailures") + + ignored_probe_noise(final_errors, "finalErrors") + + ignored_probe_noise(slow_final_errors, "slowFinalErrors") + + ignored_probe_noise(context_canceled, "contextCanceled") + ) + return { + "ok": True, + "degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0, + "window": "6h", + "tailLines": 2500, + "failoverCount": len(visible_failovers), + "forwardFailureCount": len(visible_forward_failures), + "finalErrorCount": len(visible_final_errors), + "slowFinalErrorCount": len(visible_slow_final_errors), + "contextCanceledCount": len(visible_context_canceled), + "ignoredProbeNoiseCount": len(probe_noise), + "failoverBudgetExhausted": failover_budget_exhausted[-8:], + "rawCounts": { + "failoverCount": len(failovers), + "forwardFailureCount": len(forward_failures), + "finalErrorCount": len(final_errors), + "slowFinalErrorCount": len(slow_final_errors), + "contextCanceledCount": len(context_canceled), + }, + "recentFailovers": visible_failovers[-8:], + "recentForwardFailures": visible_forward_failures[-8:], + "recentFinalErrors": visible_final_errors[-8:], + "recentSlowFinalErrors": visible_slow_final_errors[-5:], + "recentContextCanceled": visible_context_canceled[-5:], + "recentProbeNoise": probe_noise[-5:], + "logsExitCode": proc.returncode, + "logsStderr": text(proc.stderr, 1000), + "valuesPrinted": False, + } + +def validate_gateway_responses(api_key): + request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000)) + payload = { + "model": RESPONSES_SMOKE_MODEL, + "input": "Reply exactly: unidesk-sub2api-validate-ok", + "stream": False, + "store": False, + "max_output_tokens": 32, + } + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + script = r''' +set -eu +token="$1" +request_id="$2" +url="$3" +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +cat > "$tmp" +curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \ + -H "Authorization: Bearer $token" \ + -H 'Content-Type: application/json' \ + -H "X-Request-ID: $request_id" \ + -H "OpenAI-Client-Request-ID: $request_id" \ + --data-binary @"$tmp" \ + "$url" +''' + started = time.time() + if RUNTIME_MODE == "host-docker": + if not isinstance(HOST_DOCKER_APP_PORT, int): + raise RuntimeError("host-docker app port missing") + proc = run(["sh", "-c", script, "sh", api_key, request_id, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}/v1/responses"], body) + else: + proc = run([ + "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, + "--", "sh", "-c", script, "sh", api_key, request_id, "http://127.0.0.1:8080/v1/responses", + ], body) + resp = parse_curl_output(proc) + evidence = request_log_evidence(request_id) + parsed = resp.get("json") + failover_count = len(evidence.get("failovers") or []) + return { + "ok": resp.get("ok"), + "degraded": failover_count > 0, + "outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"), + "httpStatus": resp.get("httpStatus"), + "transportExitCode": resp.get("transportExitCode"), + "method": "POST /v1/responses", + "model": RESPONSES_SMOKE_MODEL, + "requestId": request_id, + "durationMs": int((time.time() - started) * 1000), + "outputTextPreview": response_output_preview(parsed), + "bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800), + "stderr": resp.get("stderr", ""), + "evidence": evidence, + "valuesPrinted": False, + } + +def bool_value(value): + if isinstance(value, bool): + return value + if isinstance(value, str): + if value.lower() == "true": + return True + if value.lower() == "false": + return False + return False + +def normalize_temp_unschedulable_credentials(credentials): + if not isinstance(credentials, dict): + credentials = {} + enabled = bool_value(credentials.get("temp_unschedulable_enabled")) + raw_rules = credentials.get("temp_unschedulable_rules") + if isinstance(raw_rules, str): + try: + raw_rules = json.loads(raw_rules) + except json.JSONDecodeError: + raw_rules = [] + rules = [] + if isinstance(raw_rules, list): + for rule in raw_rules: + if not isinstance(rule, dict): + continue + error_code = rule.get("error_code", rule.get("statusCode")) + duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes")) + keywords = rule.get("keywords") + if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list): + continue + clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()] + if not clean_keywords: + continue + description = rule.get("description") if isinstance(rule.get("description"), str) else "" + rules.append({ + "error_code": error_code, + "keywords": clean_keywords, + "duration_minutes": duration_minutes, + "description": description, + }) + return { + "enabled": enabled, + "rules": rules, + } + +def summarize_temp_unschedulable_rules(rules): + return [{ + "errorCode": rule.get("error_code"), + "durationMinutes": rule.get("duration_minutes"), + "keywordCount": len(rule.get("keywords") or []), + "keywords": rule.get("keywords") or [], + "hasDescription": bool(rule.get("description")), + } for rule in rules] + +def success_body_reclassification_requirement(): + for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): + expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) + if expected["enabled"] is not True: + continue + for rule in expected["rules"]: + error_code = rule.get("error_code") + keywords = rule.get("keywords") or [] + if isinstance(error_code, int) and 200 <= error_code < 300 and keywords: + return { + "required": True, + "sourceAccountName": name, + "statusCode": error_code, + "keywords": keywords, + "representativeKeyword": keywords[0], + "durationMinutes": rule.get("duration_minutes"), + } + return { + "required": False, + "sourceAccountName": None, + "statusCode": None, + "keywords": [], + "representativeKeyword": None, + "durationMinutes": None, + } + +def model_routing_400_failover_requirement(): + preferred = ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"] + for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): + expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) + if expected["enabled"] is not True: + continue + for rule in expected["rules"]: + error_code = rule.get("error_code") + keywords = rule.get("keywords") or [] + if error_code != 400 or not keywords: + continue + representative_keyword = next((item for item in preferred if item in keywords), keywords[0]) + return { + "required": True, + "sourceAccountName": name, + "statusCode": error_code, + "keywords": keywords, + "representativeKeyword": representative_keyword, + "durationMinutes": rule.get("duration_minutes"), + } + return { + "required": False, + "sourceAccountName": None, + "statusCode": None, + "keywords": [], + "representativeKeyword": None, + "durationMinutes": None, + } + +def delete_probe_resource(token, method, path, label): + if not path: + return {"label": label, "ok": True, "skipped": True} + resp = curl_api(method, path, bearer=token) + ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410) + return { + "label": label, + "ok": ok, + "method": method, + "path": path, + "httpStatus": resp.get("httpStatus"), + "transportExitCode": resp.get("transportExitCode"), + "bodyPreview": "" if ok else text(resp.get("body", ""), 500), + "valuesPrinted": False, + } + +def validate_runtime_capabilities(token): + success_body = success_body_reclassification_requirement() + model_routing_400 = model_routing_400_failover_requirement() + return { + "ok": success_body.get("required") is not True and model_routing_400.get("required") is True, + "runtimeImage": app_pod_runtime_image(), + "successBodyReclassification": { + "ok": success_body.get("required") is not True, + "required": success_body.get("required"), + "outcome": "not-required-by-yaml" if success_body.get("required") is not True else "requires-source-review-and-real-traffic-evidence", + "requirement": success_body, + "valuesPrinted": False, + }, + "modelRouting400Failover": { + "ok": model_routing_400.get("required") is True, + "required": model_routing_400.get("required"), + "outcome": "yaml-rules-present-runtime-observed-by-real-traffic", + "requirement": model_routing_400, + "evidence": "Use Sub2API source review plus validation.gatewayResponsesRecent and Artificer/real request ids for runtime proof; default validate does not create mock upstreams or temporary failover accounts.", + "valuesPrinted": False, + }, + "valuesPrinted": False, + } + +def app_pod_runtime_image(): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", HOST_DOCKER_APP_CONTAINER]) + if proc.returncode != 0: + return { + "container": HOST_DOCKER_APP_CONTAINER, + "error": text(proc.stderr, 1000) or text(proc.stdout, 1000), + } + try: + data = json.loads(proc.stdout.decode("utf-8")) + item = data[0] if isinstance(data, list) and data else {} + except Exception as exc: + return {"container": HOST_DOCKER_APP_CONTAINER, "error": str(exc)} + state = item.get("State") if isinstance(item, dict) and isinstance(item.get("State"), dict) else {} + health = state.get("Health") if isinstance(state.get("Health"), dict) else {} + config = item.get("Config") if isinstance(item, dict) and isinstance(item.get("Config"), dict) else {} + return { + "container": HOST_DOCKER_APP_CONTAINER, + "id": (item.get("Id") or "")[:12] if isinstance(item.get("Id"), str) else None, + "image": config.get("Image"), + "imageID": item.get("Image"), + "ready": state.get("Running") is True and (not health or health.get("Status") in (None, "healthy")), + "restartCount": item.get("RestartCount"), + "startedAt": state.get("StartedAt"), + "health": health.get("Status"), + } + try: + pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}") + spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else [] + status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else [] + spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {}) + status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {}) + return { + "pod": APP_POD, + "image": spec.get("image"), + "imageID": status.get("imageID"), + "ready": status.get("ready"), + "restartCount": status.get("restartCount"), + } + except Exception as exc: + return {"pod": APP_POD, "error": str(exc)} + +def get_account_detail(token, account): + account_id = account.get("id") if isinstance(account, dict) else None + if account_id is None: + return account + data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}") + return data if isinstance(data, dict) else account + +def account_temp_unschedulable_status(token): + accounts = list_accounts(token) + by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} + items = [] + missing = [] + mismatched = [] + enabled_names = [] + for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): + expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) + account = by_name.get(name) + if account is None: + missing.append(name) + items.append({ + "accountName": name, + "accountId": None, + "expectedEnabled": expected["enabled"], + "runtimeEnabled": None, + "expectedRuleCount": len(expected["rules"]), + "runtimeRuleCount": None, + "ok": False, + }) + continue + detail = get_account_detail(token, account) + credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {} + runtime = normalize_temp_unschedulable_credentials(credentials) + temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil") + temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or "" + ok = runtime == expected + if expected["enabled"]: + enabled_names.append(name) + if not ok: + mismatched.append(name) + items.append({ + "accountName": name, + "accountId": account.get("id"), + "expectedEnabled": expected["enabled"], + "runtimeEnabled": runtime["enabled"], + "expectedRuleCount": len(expected["rules"]), + "runtimeRuleCount": len(runtime["rules"]), + "expectedRules": summarize_temp_unschedulable_rules(expected["rules"]), + "runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]), + "status": account.get("status"), + "schedulable": account.get("schedulable"), + "tempUnschedulableUntil": temp_until, + "tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "", + "tempUnschedulableSet": temp_until is not None or bool(temp_reason), + "ok": ok, + }) + return { + "ok": len(missing) == 0 and len(mismatched) == 0, + "desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE), + "enabled": enabled_names, + "missing": missing, + "mismatched": mismatched, + "items": items, + "valuesPrinted": False, + } + +def account_capacity_status(token): + accounts = list_accounts(token) + by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} + items = [] + missing = [] + mismatched = [] + expected_capacity_total = 0 + runtime_concurrency_total = 0 + schedulable_runtime_concurrency_total = 0 + available_runtime_concurrency_total = 0 + temp_unschedulable_runtime_concurrency_total = 0 + for name in sorted(EXPECTED_ACCOUNT_CAPACITIES): + expected = int(EXPECTED_ACCOUNT_CAPACITIES[name]) + expected_capacity_total += expected + account = by_name.get(name) + if account is None: + missing.append(name) + items.append({ + "accountName": name, + "accountId": None, + "expectedCapacity": expected, + "runtimeConcurrency": None, + "ok": False, + }) + continue + runtime = account.get("concurrency") + runtime_int = runtime if isinstance(runtime, int) else 0 + runtime_concurrency_total += runtime_int + if account.get("schedulable") is True: + runtime_status = account.get("status") + if runtime_status is None or runtime_status == "active": + runtime_status_ok = True + else: + runtime_status_ok = False + if runtime_status_ok: + schedulable_runtime_concurrency_total += runtime_int + temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil") + temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason") + if temp_until is None and not bool(temp_reason): + available_runtime_concurrency_total += runtime_int + else: + temp_unschedulable_runtime_concurrency_total += runtime_int + ok = runtime == expected + if not ok: + mismatched.append(name) + items.append({ + "accountName": name, + "accountId": account.get("id"), + "expectedCapacity": expected, + "runtimeConcurrency": runtime, + "priority": account.get("priority"), + "status": account.get("status"), + "schedulable": account.get("schedulable"), + "ok": ok, + }) + return { + "ok": len(missing) == 0 and len(mismatched) == 0, + "defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY, + "desired": len(EXPECTED_ACCOUNT_CAPACITIES), + "totals": { + "expectedCapacityTotal": expected_capacity_total, + "runtimeConcurrencyTotal": runtime_concurrency_total, + "schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total, + "availableRuntimeConcurrencyTotal": available_runtime_concurrency_total, + "tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total, + "minOwnerConcurrency": MIN_OWNER_CONCURRENCY, + "minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, + }, + "missing": missing, + "mismatched": mismatched, + "items": items, + "valuesPrinted": False, + } + +def account_load_factor_status(token): + accounts = list_accounts(token) + by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} + items = [] + missing = [] + mismatched = [] + for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS): + expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name]) + account = by_name.get(name) + if account is None: + missing.append(name) + items.append({ + "accountName": name, + "accountId": None, + "expectedLoadFactor": expected, + "runtimeLoadFactor": None, + "ok": False, + }) + continue + runtime_raw = account.get("load_factor") + if runtime_raw is None: + detail = get_account_detail(token, account) + runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None + try: + runtime = int(runtime_raw) + except Exception: + runtime = runtime_raw + ok = runtime == expected + if not ok: + mismatched.append(name) + items.append({ + "accountName": name, + "accountId": account.get("id"), + "expectedLoadFactor": expected, + "runtimeLoadFactor": runtime, + "priority": account.get("priority"), + "status": account.get("status"), + "schedulable": account.get("schedulable"), + "ok": ok, + }) + return { + "ok": len(missing) == 0 and len(mismatched) == 0, + "defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR, + "desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS), + "missing": missing, + "mismatched": mismatched, + "items": items, + "valuesPrinted": False, + } + +def account_ws_v2_status(token): + accounts = list_accounts(token) + by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} + items = [] + missing = [] + mismatched = [] + enabled_names = [] + unschedulable_enabled = [] + schedulable_enabled = [] + for name in sorted(EXPECTED_ACCOUNT_WS_MODES): + expected_mode = EXPECTED_ACCOUNT_WS_MODES[name] + expected_enabled = expected_mode not in (None, "", "off") + account = by_name.get(name) + if account is None: + missing.append(name) + items.append({ + "accountName": name, + "accountId": None, + "expectedMode": expected_mode, + "expectedEnabled": expected_enabled, + "runtimeMode": None, + "runtimeEnabled": None, + "ok": False, + }) + continue + extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} + runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode") + runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled") + schedulable = account.get("schedulable") + if expected_mode == "off": + ok = runtime_mode == "off" and runtime_enabled is False + elif expected_mode is None: + ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False) + else: + ok = runtime_mode == expected_mode and runtime_enabled is True + if not ok: + mismatched.append(name) + if expected_enabled: + enabled_names.append(name) + if schedulable is True: + schedulable_enabled.append(name) + else: + unschedulable_enabled.append(name) + items.append({ + "accountName": name, + "accountId": account.get("id"), + "expectedMode": expected_mode, + "expectedEnabled": expected_enabled, + "runtimeMode": runtime_mode, + "runtimeEnabled": runtime_enabled, + "status": account.get("status"), + "schedulable": schedulable, + "ok": ok and ((not expected_enabled) or schedulable is True), + }) + availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0 + return { + "ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok, + "desired": len(EXPECTED_ACCOUNT_WS_MODES), + "enabled": enabled_names, + "schedulableEnabled": schedulable_enabled, + "unschedulableEnabled": unschedulable_enabled, + "missing": missing, + "mismatched": mismatched, + "items": items, + "valuesPrinted": False, + } + +def api_key_preview(api_key): + if len(api_key) <= 14: + return "***" + return api_key[:10] + "..." + api_key[-4:] + +def secret_fingerprint(value): + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest()[:16] + +`;