import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import type { RenderedCliResult } from "./output"; import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "./platform-infra-public-service"; import { shortSha256Fingerprint } from "./platform-infra-ops-library"; import { codexPoolSentinelSummary, codexPoolSentinelRuntimeImage, readCodexPoolSentinelConfig, renderCodexPoolSentinelManifest, type CodexPoolSentinelConfig, type CodexPoolSentinelProfileSecret, } from "./platform-infra-sub2api-codex-sentinel"; import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "./secrets"; import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; const serviceName = "sub2api"; const fieldManager = "unidesk-platform-infra"; const sub2apiConfigPath = rootPath("config", "platform-infra", "sub2api.yaml"); const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml"); const sentinelImageDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile"); const remoteJobDir = "/tmp/unidesk-platform-infra-sub2api-codex-pool"; const remoteJobTimeoutMs = 15 * 60_000; const remoteJobPollMs = 5_000; interface DisclosureOptions { full: boolean; raw: boolean; targetId: string; } interface SyncOptions extends DisclosureOptions { confirm: boolean; pruneRemoved: boolean; } interface ConfirmOptions extends DisclosureOptions { confirm: boolean; } interface SentinelProbeOptions extends ConfirmOptions { accounts: string[]; } interface SentinelReportOptions extends DisclosureOptions { events: number; } interface TraceOptions extends DisclosureOptions { requestId: string | null; since: string; tail: number; contextSeconds: number; showLines: boolean; } interface SentinelImageOptions extends DisclosureOptions { action: "status" | "build"; confirm: boolean; dryRun: boolean; } interface CodexPoolRuntimeTarget { id: string; route: string; namespace: string; serviceName: string; serviceDns: string; publicBaseUrl: string | null; publicExposure: PublicServiceExposure | null; appSecretName: string; secretsRoot: string; sentinelImageBuild: { baseImageCachePolicy: "pull" | "local-if-present"; noProxy: string; }; egressProxy: { enabled: boolean; applyToSentinel: boolean; serviceName: string; listenPort: number; httpProxy: string; noProxy: string; } | null; } interface CodexProfile { profile: string; accountName: string; configFile: string; authFile: string; provider: string; baseUrl: string; wireApi: string | null; model: string | null; envKey: string | null; apiKey: string | null; apiKeySource: "auth-json" | "env" | null; openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null; upstreamUserAgent: string | null; trustUpstream: boolean; sentinelProtect: CodexSentinelProtectPolicy; priority: number; capacity: number; loadFactor: number; tempUnschedulable: CodexTempUnschedulablePolicy; authOpenAIKeyShape: string; ok: boolean; error: string | null; } type OpenAIResponsesWebSocketsV2Mode = "off" | "ctx_pool" | "passthrough"; export interface CodexTempUnschedulableRule { statusCode: number; keywords: string[]; durationMinutes: number; description: string | null; } export interface CodexTempUnschedulablePolicy { enabled: boolean; rules: CodexTempUnschedulableRule[]; } interface CodexPoolConfig { version: number; kind: string; metadata: { id: string; owner: string; relatedIssues: number[]; }; groupName: string; groupDescription: string; apiKeyName: string; apiKeySecretName: string; apiKeySecretKey: string; adminEmailDefault: string; minOwnerBalanceUsd: number; minOwnerConcurrency: number; minOwnerConcurrencySource: "auto" | "yaml"; defaultAccountPriority: number; defaultAccountCapacity: number; defaultAccountLoadFactor: number; defaultTempUnschedulable: CodexTempUnschedulablePolicy; defaultSentinelProtect: CodexSentinelProtectPolicy; profiles: CodexPoolProfileConfig[]; manualAccounts: CodexPoolManualAccountsConfig; publicExposure: CodexPoolPublicExposureConfig; localCodex: CodexPoolLocalCodexConfig; sentinel: CodexPoolSentinelConfig; } interface CodexPoolManualAccountsConfig { bindingSources: CodexPoolManualBindingSourcesConfig; protected: CodexPoolManualAccountProtection[]; } type CodexPoolManualBindingKind = "proxy" | "group"; type CodexPoolManualBindingProvider = "target-egress-proxy" | "pool-group"; interface CodexPoolManualBindingSource { id: string; enabled: boolean; kind: CodexPoolManualBindingKind; provider: CodexPoolManualBindingProvider; description: string | null; } interface CodexPoolManualBindingSourcesConfig { items: CodexPoolManualBindingSource[]; byId: Record; } interface CodexPoolManualAccountProxyBinding { enabled: boolean; source: string; proxyName: string; } interface CodexPoolManualAccountGroupBinding { enabled: boolean; source: string; } interface CodexPoolManualAccountProtection { accountName: string; reason: string | null; proxyBinding: CodexPoolManualAccountProxyBinding | null; groupBinding: CodexPoolManualAccountGroupBinding | null; } interface CodexPoolProfileConfig { profile: string; accountName: string | null; configFile: string; authFile: string; fallbackConfigFile: string | null; fallbackAuthFile: string | null; openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null; upstreamUserAgent: string | null; trustUpstream: boolean; sentinelProtect: CodexSentinelProtectPolicy; priority: number; capacity: number | null; loadFactor: number | null; tempUnschedulable: CodexTempUnschedulablePolicy; } export interface CodexSentinelProtectPolicy { enabled: boolean; consecutiveFailures: number; initialRetryDelaySeconds: number; maxRetryDelaySeconds: number; backoffMultiplier: number; } interface CodexPoolPublicExposureConfig { enabled: boolean; proxyName: string; configMapName: string; deploymentName: string; frpcImage: string; serverAddr: string; serverPort: number; remotePort: number; localIP: string; localPort: number; publicBaseUrl: string; masterBaseUrl: string; masterFrps: { configPath: string; containerName: string; }; masterCaddy: { enabled: boolean; domain: string; configPath: string; serviceName: string; upstreamBaseUrl: string; responseHeaderTimeoutSeconds: number; edgeRetry: CodexPoolCaddyEdgeRetryConfig; }; } interface CodexPoolCaddyEdgeRetryConfig { enabled: boolean; tryDurationSeconds: number; tryIntervalMilliseconds: number; retryMatch: { methods: string[]; paths: string[]; }; } interface CodexPoolLocalCodexConfig { backupSuffix: string; providerName: string; wireApi: string; modelContextWindow: number; modelAutoCompactTokenLimit: number; supportsWebSockets: boolean; responsesWebSocketsV2: boolean; responsesSmokeModel: string; } interface CodexLocalConsumerTomlOptions { providerName: string; baseUrl: string; wireApi: string; modelContextWindow: number; modelAutoCompactTokenLimit: number; supportsWebSockets: boolean; responsesWebSocketsV2: boolean; } export function codexPoolHelp(): unknown { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(); return { command: "platform-infra sub2api codex-pool plan|sync|validate|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", output: "json, except trace and sentinel-report default to low-noise text tables", usage: [ "bun scripts/cli.ts platform-infra sub2api codex-pool plan", "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", "bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]", "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report [--target D601] [--events 20|--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes [--target D601] --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm", ], description: "Import YAML-selected ~/.codex API-key profiles into one Sub2API OpenAI pool, expose one unified API_KEY, and optionally configure this master server's ~/.codex consumer endpoint.", target: { default: runtimeTarget.id, route: runtimeTarget.route, namespace: runtimeTarget.namespace, serviceDns: runtimeTarget.serviceDns, configPath: codexPoolConfigPath, poolGroupName: pool.groupName, poolApiKeySecretName: pool.apiKeySecretName, poolApiKeySecretKey: pool.apiKeySecretKey, publicBaseUrl: runtimeTarget.publicBaseUrl, publicExposureSource: `${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure`, sentinelMonitorEnabled: pool.sentinel.monitor.enabled, sentinelActionsEnabled: pool.sentinel.actions.enabled, secretValuesPrinted: false, }, }; } export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const [action = "plan"] = args; if (action === "plan") return codexPoolPlan(parseDisclosureOptions(args.slice(1))); if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1))); if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1))); if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1))); if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1))); if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1))); if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1))); if (action === "cleanup-probes") return await codexPoolCleanupProbes(config, parseConfirmOptions(args.slice(1))); if (action === "expose") return await codexPoolExpose(config, parseConfirmOptions(args.slice(1))); if (action === "configure-local") return await codexPoolConfigureLocal(config, parseConfirmOptions(args.slice(1))); return { ok: false, error: "unsupported-platform-infra-sub2api-codex-pool-command", args, help: codexPoolHelp(), }; } function parseSyncOptions(args: string[]): SyncOptions { validateOptions(args, new Set(["--confirm", "--prune-removed", "--full", "--raw", "--target"])); const disclosure = parseDisclosureOptions(stripBooleanOptions(args, new Set(["--confirm", "--prune-removed"]))); return { ...disclosure, confirm: args.includes("--confirm"), pruneRemoved: args.includes("--prune-removed") }; } function parseConfirmOptions(args: string[]): ConfirmOptions { validateOptions(args, new Set(["--confirm", "--full", "--raw", "--target"])); const disclosure = parseDisclosureOptions(stripBooleanOptions(args, new Set(["--confirm"]))); return { ...disclosure, confirm: args.includes("--confirm") }; } function parseSentinelImageOptions(args: string[]): SentinelImageOptions { const [actionRaw = "status", ...rest] = args; if (actionRaw !== "status" && actionRaw !== "build") throw new Error("sentinel-image usage: status|build [--dry-run|--confirm] [--full|--raw]"); let confirm = false; let explicitDryRun = false; const disclosureArgs: string[] = []; for (let index = 0; index < rest.length; index += 1) { const arg = rest[index]!; if (arg === "--confirm") { confirm = true; continue; } if (arg === "--dry-run") { explicitDryRun = true; continue; } if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); if (arg === "--target") { const value = rest[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); disclosureArgs.push(value); index += 1; } continue; } throw new Error(`unsupported option: ${arg}`); } if (confirm && explicitDryRun) throw new Error("sentinel-image accepts only one of --confirm or --dry-run"); const disclosure = parseDisclosureOptions(disclosureArgs); return { ...disclosure, action: actionRaw, confirm, dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, }; } function parseSentinelProbeOptions(args: string[]): SentinelProbeOptions { const accounts: string[] = []; const disclosureArgs: string[] = []; let confirm = false; for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; if (arg === "--confirm") { confirm = true; continue; } if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); if (arg === "--target") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); disclosureArgs.push(value); index += 1; } continue; } if (arg === "--account") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--account requires an account name"); accounts.push(...splitAccountNames(value)); index += 1; continue; } if (arg.startsWith("--account=")) { accounts.push(...splitAccountNames(arg.slice("--account=".length))); continue; } throw new Error(`unsupported option: ${arg}`); } const uniqueAccounts = [...new Set(accounts)]; if (uniqueAccounts.length === 0) throw new Error("sentinel-probe requires --account "); for (const account of uniqueAccounts) validateSub2ApiAccountSelector(account, "--account"); const disclosure = parseDisclosureOptions(disclosureArgs); return { ...disclosure, confirm, accounts: uniqueAccounts }; } function parseSentinelReportOptions(args: string[]): SentinelReportOptions { let events = 20; let explicitEvents = false; const disclosureArgs: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); if (arg === "--target") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); disclosureArgs.push(value); index += 1; } continue; } if (arg === "--events") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--events requires a positive integer"); events = readReportEventLimit(value, "--events"); explicitEvents = true; index += 1; continue; } if (arg.startsWith("--events=")) { events = readReportEventLimit(arg.slice("--events=".length), "--events"); explicitEvents = true; continue; } throw new Error(`unsupported option: ${arg}`); } const disclosure = parseDisclosureOptions(disclosureArgs); if (disclosure.full && !explicitEvents) events = 80; return { ...disclosure, events }; } function parseTraceOptions(args: string[]): TraceOptions { let requestId: string | null = null; let since = "24h"; let tail = 20_000; let contextSeconds = 300; let showLines = false; const disclosureArgs: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); if (arg === "--target") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); disclosureArgs.push(value); index += 1; } continue; } if (arg === "--show-lines") { showLines = true; continue; } if (arg === "--request-id" || arg === "--id") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a request id`); requestId = value.trim(); index += 1; continue; } if (arg.startsWith("--request-id=")) { requestId = arg.slice("--request-id=".length).trim(); continue; } if (arg.startsWith("--id=")) { requestId = arg.slice("--id=".length).trim(); continue; } if (arg === "--since") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--since requires a kubectl duration such as 24h or 90m"); since = parseKubectlDuration(value); index += 1; continue; } if (arg.startsWith("--since=")) { since = parseKubectlDuration(arg.slice("--since=".length)); continue; } if (arg === "--tail") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--tail requires an integer"); tail = parseTraceLimit(value, "--tail", 100, 200_000); index += 1; continue; } if (arg.startsWith("--tail=")) { tail = parseTraceLimit(arg.slice("--tail=".length), "--tail", 100, 200_000); continue; } if (arg === "--context-seconds") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--context-seconds requires an integer"); contextSeconds = parseTraceLimit(value, "--context-seconds", 0, 3600); index += 1; continue; } if (arg.startsWith("--context-seconds=")) { contextSeconds = parseTraceLimit(arg.slice("--context-seconds=".length), "--context-seconds", 0, 3600); continue; } throw new Error(`unsupported option: ${arg}`); } if (requestId === null || requestId.length === 0) throw new Error("trace requires --request-id "); if (!/^[A-Za-z0-9_.:-]{8,128}$/u.test(requestId)) throw new Error("--request-id has an unsupported format"); const disclosure = parseDisclosureOptions(disclosureArgs); return { ...disclosure, requestId, since, tail, contextSeconds, showLines }; } function parseKubectlDuration(raw: string): string { const value = raw.trim(); if (!/^[1-9][0-9]*(?:s|m|h)$/u.test(value)) throw new Error("--since must be a kubectl duration such as 24h, 90m, or 300s"); return value; } function parseTraceLimit(raw: string, option: string, min: number, max: number): number { const value = Number(raw); if (!Number.isInteger(value) || value < min || value > max) throw new Error(`${option} must be an integer from ${min} to ${max}`); return value; } function readReportEventLimit(raw: string, option: string): number { const value = Number(raw); if (!Number.isInteger(value) || value < 1 || value > 200) throw new Error(`${option} must be an integer from 1 to 200`); return value; } function parseDisclosureOptions(args: string[]): DisclosureOptions { validateOptions(args, new Set(["--full", "--raw", "--target"])); const raw = args.includes("--raw"); return { full: raw || args.includes("--full"), raw, targetId: parseTargetId(args) }; } function parseTargetId(args: string[]): string { let targetId: string | null = null; for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; if (arg === "--target") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); targetId = value; index += 1; continue; } if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length); } const resolvedTargetId = targetId ?? defaultCodexPoolRuntimeTargetId(); if (!/^[A-Za-z0-9._-]+$/u.test(resolvedTargetId)) throw new Error("--target must be a simple target id"); return resolvedTargetId; } function splitAccountNames(value: string): string[] { return value.split(",").map((item) => item.trim()).filter(Boolean); } function validateSub2ApiAccountSelector(value: string, option: string): void { if (value.length === 0 || value.length > 256) throw new Error(`${option} must be a non-empty account name up to 256 characters`); if (/[\r\n]/u.test(value)) throw new Error(`${option} must not contain newlines`); if (!/^[^<>"'`\\]+$/u.test(value)) throw new Error(`${option} contains unsupported characters`); } function validateOptions(args: string[], booleanOptions: Set): void { for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; if (arg === "--target") { index += 1; continue; } if (arg.startsWith("--target=") && booleanOptions.has("--target")) continue; if (booleanOptions.has(arg)) continue; throw new Error(`unsupported option: ${arg}`); } } function stripBooleanOptions(args: string[], stripped: Set): string[] { return args.filter((arg) => !stripped.has(arg)); } interface Sub2ApiRuntimeConfig { defaultTargetId: string; appSecretName: string; secretsRoot: string; targets: Record[]; } function readSub2ApiRuntimeConfig(): Sub2ApiRuntimeConfig { const parsed = Bun.YAML.parse(readFileSync(sub2apiConfigPath, "utf8")) as unknown; if (!isRecord(parsed)) throw new Error(`${sub2apiConfigPath} must contain a YAML object`); const defaults = isRecord(parsed.defaults) ? parsed.defaults : null; const defaultTargetId = defaults === null ? null : stringValue(defaults.targetId); if (defaultTargetId === null || !/^[A-Za-z0-9._-]+$/u.test(defaultTargetId)) throw new Error(`${sub2apiConfigPath}.defaults.targetId must be a simple target id`); const runtime = isRecord(parsed.runtime) ? parsed.runtime : null; const database = runtime !== null && isRecord(runtime.database) ? runtime.database : null; const appSecretName = database === null ? null : stringValue(database.secretName); if (appSecretName === null) throw new Error(`${sub2apiConfigPath}.runtime.database.secretName is required`); validateKubernetesName(appSecretName, `${sub2apiConfigPath}.runtime.database.secretName`, true); const secrets = runtime !== null && isRecord(runtime.secrets) ? runtime.secrets : null; const secretsRoot = secrets === null ? null : stringValue(secrets.root); if (secretsRoot === null || !secretsRoot.startsWith("/")) throw new Error(`${sub2apiConfigPath}.runtime.secrets.root must be an absolute path`); if (!Array.isArray(parsed.targets) || !parsed.targets.every(isRecord)) throw new Error(`${sub2apiConfigPath}.targets must be a list`); return { defaultTargetId, appSecretName, secretsRoot, targets: parsed.targets, }; } function defaultCodexPoolRuntimeTargetId(): string { return readSub2ApiRuntimeConfig().defaultTargetId; } function codexPoolRuntimeTarget(targetId?: string): CodexPoolRuntimeTarget { const runtimeConfig = readSub2ApiRuntimeConfig(); const resolvedTargetId = targetId ?? runtimeConfig.defaultTargetId; const raw = runtimeConfig.targets.find((item) => String(item.id ?? "").toLowerCase() === resolvedTargetId.toLowerCase()); if (!isRecord(raw)) throw new Error(`${sub2apiConfigPath}.targets does not contain target ${targetId}`); const id = stringValue(raw.id) ?? resolvedTargetId; const route = stringValue(raw.route) ?? ""; const targetNamespace = stringValue(raw.namespace); if (route.length === 0) throw new Error(`${sub2apiConfigPath}.targets[${id}].route is required`); if (targetNamespace === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].namespace is required`); validateKubernetesName(targetNamespace, `${sub2apiConfigPath}.targets[${id}].namespace`, true); const sentinelImageBuild = readTargetSentinelImageBuild(raw, id); let egressProxy: CodexPoolRuntimeTarget["egressProxy"] = null; if (isRecord(raw.egressProxy) && raw.egressProxy.enabled === true) { const proxyServiceName = stringValue(raw.egressProxy.serviceName); const listenPort = numberValue(raw.egressProxy.listenPort); if (proxyServiceName === null || listenPort === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].egressProxy.serviceName/listenPort are required`); validateKubernetesName(proxyServiceName, `${sub2apiConfigPath}.targets[${id}].egressProxy.serviceName`, true); if (!Number.isInteger(listenPort) || listenPort < 1 || listenPort > 65535) throw new Error(`${sub2apiConfigPath}.targets[${id}].egressProxy.listenPort must be a TCP port`); const noProxyRaw = Array.isArray(raw.egressProxy.noProxy) ? raw.egressProxy.noProxy : []; const noProxy = noProxyRaw.map((entry) => stringValue(entry)).filter((entry): entry is string => entry !== null && entry.length > 0).join(","); egressProxy = { enabled: true, applyToSentinel: raw.egressProxy.applyToSentinel === undefined ? true : raw.egressProxy.applyToSentinel === true, serviceName: proxyServiceName, listenPort, httpProxy: `http://${proxyServiceName}.${targetNamespace}.svc.cluster.local:${listenPort}`, noProxy, }; } let publicBaseUrl: string | null = null; const publicExposure = readTargetPublicExposure(raw, id); if (publicExposure !== null && publicExposure.enabled) publicBaseUrl = publicExposure.publicBaseUrl; return { id, route, namespace: targetNamespace, serviceName, serviceDns: `${serviceName}.${targetNamespace}.svc.cluster.local:8080`, publicBaseUrl, publicExposure, appSecretName: runtimeConfig.appSecretName, secretsRoot: runtimeConfig.secretsRoot, sentinelImageBuild, egressProxy, }; } function readTargetSentinelImageBuild(raw: Record, targetId: string): CodexPoolRuntimeTarget["sentinelImageBuild"] { const codexPool = isRecord(raw.codexPool) ? raw.codexPool : null; const imageBuild = codexPool !== null && isRecord(codexPool.sentinelImageBuild) ? codexPool.sentinelImageBuild : null; if (imageBuild === null) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild must be a YAML object`); const policy = stringValue(imageBuild.baseImageCachePolicy); if (policy !== "pull" && policy !== "local-if-present") { throw new Error(`${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild.baseImageCachePolicy must be pull or local-if-present`); } return { baseImageCachePolicy: policy, noProxy: readTargetNoProxy(imageBuild.noProxy, `${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild.noProxy`), }; } function readTargetPublicExposure(raw: Record, targetId: string): PublicServiceExposure | null { if (raw.publicExposure === undefined || raw.publicExposure === null) return null; if (!isRecord(raw.publicExposure)) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].publicExposure must be a YAML object`); const exposure = raw.publicExposure; const enabled = readRequiredTargetBoolean(exposure.enabled, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.enabled`); const dns = readRequiredTargetRecord(exposure.dns, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns`); const frpc = readRequiredTargetRecord(exposure.frpc, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc`); const pk01 = readRequiredTargetRecord(exposure.pk01, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01`); const publicBaseUrl = readTargetBaseUrl(exposure.publicBaseUrl, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.publicBaseUrl`, "https:"); const hostname = readRequiredTargetString(dns.hostname, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.hostname`); if (new URL(publicBaseUrl).hostname !== hostname) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].publicExposure publicBaseUrl hostname must match dns.hostname`); const config: PublicServiceExposure = { enabled, publicBaseUrl, dns: { hostname, expectedA: readRequiredTargetString(dns.expectedA, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.expectedA`), resolvers: readTargetStringArray(dns.resolvers, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.resolvers`), }, frpc: { deploymentName: readRequiredTargetString(frpc.deploymentName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.deploymentName`), secretName: readRequiredTargetString(frpc.secretName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretName`), secretKey: readRequiredTargetString(frpc.secretKey, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretKey`), image: readRequiredTargetString(frpc.image, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.image`), serverAddr: readRequiredTargetString(frpc.serverAddr, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverAddr`), serverPort: readTargetPort(frpc.serverPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverPort`), proxyName: readRequiredTargetString(frpc.proxyName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.proxyName`), remotePort: readTargetPort(frpc.remotePort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.remotePort`), localIP: readRequiredTargetString(frpc.localIP, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localIP`), localPort: readTargetPort(frpc.localPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localPort`), tokenSourceRef: readRequiredTargetString(frpc.tokenSourceRef, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.tokenSourceRef`), tokenSourceKey: readRequiredTargetString(frpc.tokenSourceKey, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.tokenSourceKey`), }, pk01: { route: readRequiredTargetString(pk01.route, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.route`), caddyConfigPath: readAbsoluteTargetPath(pk01.caddyConfigPath, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.caddyConfigPath`), caddyServiceName: readRequiredTargetString(pk01.caddyServiceName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.caddyServiceName`), responseHeaderTimeoutSeconds: readPositiveTargetInteger(pk01.responseHeaderTimeoutSeconds, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.responseHeaderTimeoutSeconds`), }, }; validateKubernetesName(config.frpc.deploymentName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.deploymentName`, true); validateKubernetesName(config.frpc.secretName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretName`, true); validateProxyName(config.frpc.proxyName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.proxyName`); validatePort(config.frpc.serverPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverPort`); validatePort(config.frpc.remotePort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.remotePort`); validatePort(config.frpc.localPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localPort`); return config; } function readRequiredTargetRecord(value: unknown, path: string): Record { if (!isRecord(value)) throw new Error(`${path} must be a YAML object`); return value; } function readRequiredTargetString(value: unknown, path: string): string { const text = stringValue(value); if (text === null) throw new Error(`${path} must be a non-empty string`); if (/[\r\n]/u.test(text)) throw new Error(`${path} must not contain newlines`); return text; } function readRequiredTargetBoolean(value: unknown, path: string): boolean { const parsed = booleanValue(value); if (parsed === null) throw new Error(`${path} must be a boolean`); return parsed; } function readTargetBaseUrl(value: unknown, path: string, protocol: "http:" | "https:" | null = null): string { const baseUrl = normalizeBaseUrl(stringValue(value)); if (baseUrl === null) throw new Error(`${path} must be a valid http(s) URL`); if (protocol !== null && new URL(baseUrl).protocol !== protocol) throw new Error(`${path} must use ${protocol}//`); return baseUrl; } function readTargetPort(value: unknown, path: string): number { const port = numberValue(value); if (port === null || !Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path} must be an integer TCP port`); return port; } function readPositiveTargetInteger(value: unknown, path: string): number { const number = numberValue(value); if (number === null || !Number.isInteger(number) || number < 1) throw new Error(`${path} must be a positive integer`); return number; } function readAbsoluteTargetPath(value: unknown, path: string): string { const text = readRequiredTargetString(value, path); if (!text.startsWith("/")) throw new Error(`${path} must be an absolute path`); return text; } function readTargetStringArray(value: unknown, path: string): string[] { if (!Array.isArray(value)) throw new Error(`${path} must be a YAML array`); const result = value.map((item, index) => readRequiredTargetString(item, `${path}[${index}]`)); if (result.length === 0) throw new Error(`${path} must not be empty`); return result; } function readTargetNoProxy(value: unknown, path: string): string { return readTargetStringArray(value, path).join(","); } function targetFlag(target: CodexPoolRuntimeTarget): string { return target.id === defaultCodexPoolRuntimeTargetId() ? "" : ` --target ${target.id}`; } function codexPoolPlan(options?: DisclosureOptions): Record { const resolvedOptions = options ?? { full: false, raw: false, targetId: defaultCodexPoolRuntimeTargetId() }; const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(resolvedOptions.targetId); const profiles = collectCodexProfiles(); const ok = profiles.length > 0 && profiles.every((profile) => profile.ok); const consumerBaseUrl = runtimeTarget.publicBaseUrl === null ? null : codexConsumerBaseUrl(pool, runtimeTarget); return { ok, action: "platform-infra-sub2api-codex-pool-plan", source: { directory: join(homedir(), ".codex"), configPattern: "YAML-selected config files under ~/.codex", authPattern: "YAML-selected auth files under ~/.codex", valuesPrinted: false, }, target: poolTarget(pool, runtimeTarget), config: { path: codexPoolConfigPath, pool: options.full ? pool : codexPoolConfigSummary(pool, runtimeTarget), }, profiles: options.full ? profiles.map(redactProfile) : profiles.map(compactProfile), decision: { accountType: "openai/apikey", grouping: `All discovered Codex profiles are bound to one Sub2API group named ${pool.groupName}.`, unifiedApiKey: `The client-facing API_KEY is controlled by k3s Secret ${runtimeTarget.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}.`, sentinel: pool.sentinel.monitor.enabled ? `Account sentinel is enabled as k8s CronJob ${runtimeTarget.namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.` : "Account sentinel monitoring is disabled by YAML.", publicExposure: runtimeTarget.publicBaseUrl === null ? `Target-level public exposure is disabled or absent in ${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.` : `Codex consumers for target ${runtimeTarget.id} use target-level public exposure ${consumerBaseUrl}.`, idempotency: "sync reuses the group, account names, and k3s Secret when they already exist; credentials are updated from the current local Codex files for YAML-managed profiles only; managed accounts missing from YAML are preserved unless --prune-removed is explicitly provided.", configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.", manualAccountProtection: pool.manualAccounts.protected.length === 0 ? "No manual Sub2API accounts are protected by YAML." : `${pool.manualAccounts.protected.length} manual Sub2API account(s) are protected from UniDesk-managed credentials, prune, sentinel probe, and sentinel freeze paths; only explicitly declared proxy/group bindings are reconciled.`, }, next: ok ? { sync: `bun scripts/cli.ts platform-infra sub2api codex-pool sync${targetFlag(runtimeTarget)} --confirm` } : { fix: "Ensure every discovered config.toml profile has a base_url and either auth.json OPENAI_API_KEY or the configured env_key present in this shell." }, }; } async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const profiles = collectCodexProfiles(); const planOk = profiles.length > 0 && profiles.every((profile) => profile.ok); if (!options.confirm || !planOk) { return { ...codexPoolPlan(options), ok: !options.confirm ? planOk : false, mode: options.confirm ? "blocked-invalid-local-profile" : "dry-run", next: options.confirm ? { fix: "Repair invalid local Codex profiles, then rerun sync --confirm." } : { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" }, }; } const sentinelImage = pool.sentinel.monitor.enabled ? await runCodexPoolSentinelImage(config, pool, { action: "build", confirm: true, dryRun: false, full: options.full, raw: false, targetId: options.targetId }) : { ok: true, mode: "skipped-monitor-disabled" }; if (sentinelImage.ok !== true) { return { ok: false, action: "platform-infra-sub2api-codex-pool-sync", mode: "blocked-sentinel-image", sentinelImage, next: { image: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build${targetFlag(runtimeTarget)} --confirm`, }, }; } const payload = { pruneRemoved: options.pruneRemoved, sentinel: { manifest: renderCodexPoolSentinelManifest(pool.sentinel, sentinelProfileSecrets(profiles), { namespace: runtimeTarget.namespace, serviceName: runtimeTarget.serviceName, serviceDns: runtimeTarget.serviceDns, appSecretName: runtimeTarget.appSecretName, adminEmailDefault: pool.adminEmailDefault, proxy: runtimeTarget.egressProxy?.applyToSentinel ? { httpProxy: runtimeTarget.egressProxy.httpProxy, noProxy: runtimeTarget.egressProxy.noProxy, } : null, }), summary: codexPoolSentinelSummary(pool.sentinel), }, pool: { groupName: pool.groupName, groupDescription: pool.groupDescription, apiKeyName: pool.apiKeyName, apiKeySecretName: pool.apiKeySecretName, apiKeySecretKey: pool.apiKeySecretKey, adminEmailDefault: pool.adminEmailDefault, minOwnerBalanceUsd: pool.minOwnerBalanceUsd, minOwnerConcurrency: pool.minOwnerConcurrency, defaultAccountPriority: pool.defaultAccountPriority, defaultAccountCapacity: pool.defaultAccountCapacity, defaultAccountLoadFactor: pool.defaultAccountLoadFactor, defaultSentinelProtect: pool.defaultSentinelProtect, }, manualAccounts: { bindingSources: pool.manualAccounts.bindingSources.items.map(manualBindingSourcePlan), protected: resolvedManualAccountProtections(pool, runtimeTarget), }, profiles: profiles.map((profile) => ({ profile: profile.profile, accountName: profile.accountName, configFile: profile.configFile, authFile: profile.authFile, provider: profile.provider, baseUrl: profile.baseUrl, wireApi: profile.wireApi, model: profile.model, apiKey: profile.apiKey, apiKeySource: profile.apiKeySource, apiKeyFingerprint: fingerprint(profile.apiKey ?? ""), sentinelProbeConfigFingerprint: codexPoolSentinelProbeConfigFingerprint({ accountName: profile.accountName, profile: profile.profile, baseUrl: profile.baseUrl, apiKeyFingerprint: fingerprint(profile.apiKey ?? ""), upstreamUserAgent: profile.upstreamUserAgent, openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, trustUpstream: profile.trustUpstream, sentinelProtect: profile.sentinelProtect, }), openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, sentinelProtect: profile.sentinelProtect, priority: profile.priority, capacity: profile.capacity, loadFactor: profile.loadFactor, tempUnschedulable: profile.tempUnschedulable, tempUnschedulableCredentials: renderSub2ApiTempUnschedulableCredentials(profile.tempUnschedulable), })), }; const result = await runRemoteCodexPoolScript(config, "sync", syncScript(payload, pool, runtimeTarget), runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-sync", remote: compactCapture(result, { full: true }), parsed, }; } return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-sync", local: { profileCount: profiles.length, pruneRemoved: options.pruneRemoved, invalidProfiles: profiles.filter((profile) => !profile.ok).map(compactProfile), profiles: options.full ? profiles.map(redactProfile) : undefined, valuesPrinted: false, }, sentinelImage, remote: parsed === null ? compactCapture(result, { full: options.full || result.exitCode !== 0 }) : options.full ? parsed : codexPoolSyncSummary(parsed), next: { validate: `bun scripts/cli.ts platform-infra sub2api codex-pool validate${targetFlag(runtimeTarget)}`, }, }; } async function codexPoolSentinelImage(config: UniDeskConfig, options: SentinelImageOptions): Promise> { const pool = readCodexPoolConfig(); return await runCodexPoolSentinelImage(config, pool, options); } async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolConfig, options: SentinelImageOptions): Promise> { const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const target = codexPoolSentinelRuntimeImage(pool.sentinel); if (options.action === "build" && options.dryRun) { return { ok: true, action: "platform-infra-sub2api-codex-pool-sentinel-image", mode: "dry-run", image: target, dockerfile: sentinelImageDockerfilePath, mutation: false, next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build${targetFlag(runtimeTarget)} --confirm`, }, }; } const mode: RemoteCodexPoolMode = options.action === "status" ? "sentinel-image-status" : "sentinel-image-build"; const script = options.action === "status" ? sentinelImageStatusScript(pool, runtimeTarget) : sentinelImageBuildScript(pool, runtimeTarget); const result = await runRemoteCodexPoolScript(config, mode, script, runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-sentinel-image", mode: options.action, image: target, remote: compactCapture(result, { full: true }), parsed, }; } return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-sentinel-image", mode: options.action, image: target, summary: parsed, remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }), }; } async function codexPoolValidate(config: UniDeskConfig, options: DisclosureOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const result = await runRemoteCodexPoolScript(config, "validate", validateScript(pool, runtimeTarget), runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-validate", remote: compactCapture(result, { full: true }), parsed, }; } return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-validate", summary: options.full ? parsed : codexPoolValidationSummary(parsed), remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }), }; } async function codexPoolTrace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const result = await capture(config, runtimeTarget.route, ["sh"], traceScript(pool, options, runtimeTarget)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && boolField(parsed, "ok", false); if (options.raw) { return { ok, action: "platform-infra-sub2api-codex-pool-trace", remote: compactCapture(result, { full: true }), trace: parsed, valuesPrinted: false, }; } const text = renderTraceReport(parsed, { requestId: options.requestId ?? "", showLines: options.showLines, remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), }); return renderedCliResult(ok, "platform-infra sub2api codex-pool trace", text); } async function codexPoolSentinelReport(config: UniDeskConfig, options: SentinelReportOptions): Promise | RenderedCliResult> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const result = await capture(config, runtimeTarget.route, ["sh"], sentinelReportScript(pool, options.events, runtimeTarget)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && boolField(parsed, "ok", false); if (options.raw) { return { ok, action: "platform-infra-sub2api-codex-pool-sentinel-report", remote: compactCapture(result, { full: true }), report: parsed, valuesPrinted: false, }; } const text = renderSentinelReport(parsed, { events: options.events, full: options.full, remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), }); return renderedCliResult(ok, "platform-infra sub2api codex-pool sentinel-report", text); } async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelProbeOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const protectedNames = new Set(pool.manualAccounts.protected.map((account) => account.accountName.toLowerCase())); const protectedRequested = options.accounts.filter((account) => protectedNames.has(account.toLowerCase())); if (protectedRequested.length > 0) { return { ok: false, action: "platform-infra-sub2api-codex-pool-sentinel-probe", error: "account-protected-manual", protected: protectedRequested, message: "Protected manual Sub2API accounts are not YAML-managed and must not be probed by the UniDesk sentinel.", valuesPrinted: false, }; } const configuredAccounts = desiredAccountNames(pool); const missing = options.accounts.filter((account) => !configuredAccounts.includes(account)); if (missing.length > 0) { return { ok: false, action: "platform-infra-sub2api-codex-pool-sentinel-probe", error: "account-not-in-yaml", missing, configuredAccounts, valuesPrinted: false, }; } if (!options.confirm) { return { ok: true, action: "platform-infra-sub2api-codex-pool-sentinel-probe", mode: "dry-run", target: poolTarget(pool, runtimeTarget), accounts: options.accounts, effect: "Would create one Kubernetes Job from the managed sentinel CronJob and force an immediate marker probe for the requested account(s).", next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe${targetFlag(runtimeTarget)} --account ${options.accounts.join(",")} --confirm`, }, valuesPrinted: false, }; } const payload = { accounts: options.accounts, }; const result = await runRemoteCodexPoolScript(config, "sentinel-probe", sentinelProbeScript(payload, pool, runtimeTarget), runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-sentinel-probe", remote: compactCapture(result, { full: true }), parsed, }; } return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-sentinel-probe", summary: options.full ? parsed : compactSentinelProbeResult(parsed), remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }), }; } async function codexPoolCleanupProbes(config: UniDeskConfig, options: ConfirmOptions): Promise> { const runtimeTarget = codexPoolRuntimeTarget(options.targetId); if (!options.confirm) { return { ok: true, action: "platform-infra-sub2api-codex-pool-cleanup-probes", mode: "dry-run", target: poolTarget(readCodexPoolConfig(), runtimeTarget), scope: "Only deletes temporary resources whose names start with unidesk-probe-.", next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes${targetFlag(runtimeTarget)} --confirm` }, valuesPrinted: false, }; } const pool = readCodexPoolConfig(); const result = await capture(config, runtimeTarget.route, ["sh"], cleanupProbesScript(pool, runtimeTarget)); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-cleanup-probes", remote: compactCapture(result, { full: true }), parsed, }; } return { ok: result.exitCode === 0 && boolField(parsed, "ok", false), action: "platform-infra-sub2api-codex-pool-cleanup-probes", summary: parsed, remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }), }; } async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); if (runtimeTarget.publicExposure === null || !runtimeTarget.publicExposure.enabled) { return { ok: true, action: "platform-infra-sub2api-codex-pool-expose", mode: "disabled-by-yaml", target: poolTarget(pool, runtimeTarget), source: `${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure`, }; } if (!options.confirm) { return { ok: true, action: "platform-infra-sub2api-codex-pool-expose", mode: "dry-run", target: poolTarget(pool, runtimeTarget), publicExposure: targetPublicExposureSummary(runtimeTarget), next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool expose${targetFlag(runtimeTarget)} --confirm`, }, }; } const secretMaterial = prepareTargetPublicExposureSecret(runtimeTarget); const caddyResult = await applyPk01CaddyBlock(config, serviceName, runtimeTarget.publicExposure); const remoteResult = await capture(config, runtimeTarget.route, ["sh"], targetPublicExposureApplyScript(runtimeTarget, secretMaterial)); const parsed = parseJsonOutput(remoteResult.stdout); const publicProbe = await probePublicModels(pool, "without-api-key", undefined, runtimeTarget); const ok = caddyResult.ok === true && remoteResult.exitCode === 0 && boolField(parsed, "ok", false) && publicProbe.httpStatus === 401; if (options.raw) { return { ok, action: "platform-infra-sub2api-codex-pool-expose", frpcSecret: secretMaterialSummary(secretMaterial), pk01Caddy: caddyResult, remote: compactCapture(remoteResult, { full: true }), parsed, publicProbe, }; } return { ok, action: "platform-infra-sub2api-codex-pool-expose", mode: "confirmed", publicExposure: targetPublicExposureSummary(runtimeTarget), frpcSecret: secretMaterialSummary(secretMaterial), pk01Caddy: caddyResult, remote: parsed ?? compactCapture(remoteResult, { full: options.full || remoteResult.exitCode !== 0 }), publicProbe, next: { configureLocal: `bun scripts/cli.ts platform-infra sub2api codex-pool configure-local${targetFlag(runtimeTarget)} --confirm`, validate: `bun scripts/cli.ts platform-infra sub2api codex-pool validate${targetFlag(runtimeTarget)}`, }, }; } async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const consumerExposureAvailable = runtimeTarget.publicBaseUrl !== null; const codexDir = join(homedir(), ".codex"); const configPath = join(codexDir, "config.toml"); const authPath = join(codexDir, "auth.json"); const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`); const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`); if (!options.confirm) { return { ok: true, action: "platform-infra-sub2api-codex-pool-configure-local", mode: "dry-run", target: { codexDir, configPath, authPath, backupConfigPath, backupAuthPath, baseUrl: consumerExposureAvailable ? codexConsumerBaseUrl(pool, runtimeTarget) : null, providerName: pool.localCodex.providerName, wireApi: pool.localCodex.wireApi, modelContextWindow: pool.localCodex.modelContextWindow, modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit, supportsWebSockets: pool.localCodex.supportsWebSockets, responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2, responsesSmokeModel: pool.localCodex.responsesSmokeModel, valuesPrinted: false, }, next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool configure-local${targetFlag(runtimeTarget)} --confirm`, }, }; } if (!consumerExposureAvailable) throw new Error(`${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.enabled must be true; configure-local needs one consumer URL`); const keyResult = await fetchPoolApiKey(config, pool, runtimeTarget); if (keyResult.apiKey === null) { return { ok: false, action: "platform-infra-sub2api-codex-pool-configure-local", error: keyResult.error ?? "pool API key missing", valuesPrinted: false, }; } const writeResult = writeLocalCodexConfig(pool, keyResult.apiKey, runtimeTarget); const validateResult = await validatePublicGatewayWithKey(pool, keyResult.apiKey, runtimeTarget); return { ok: writeResult.ok && validateResult.ok, action: "platform-infra-sub2api-codex-pool-configure-local", mode: "confirmed", local: writeResult, validation: validateResult, apiKey: { secret: `${runtimeTarget.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, keyPreview: apiKeyPreview(keyResult.apiKey), apiKeyFingerprint: fingerprint(keyResult.apiKey), valuesPrinted: false, }, }; } function collectCodexProfiles(): CodexProfile[] { const codexDir = join(homedir(), ".codex"); const pool = readCodexPoolConfig(); if (!existsSync(codexDir)) return []; const seenAccountNames = new Set(); return pool.profiles.map((entry) => { const resolved = resolveProfileFiles(codexDir, entry); const profile = entry.profile; const accountName = entry.accountName ?? uniqueAccountName(profile, seenAccountNames); seenAccountNames.add(accountName); const configFile = resolved.configFile; const authFile = resolved.authFile; const configPath = join(codexDir, configFile); const authPath = join(codexDir, authFile); const base: CodexProfile = { profile, accountName, configFile, authFile, provider: "", baseUrl: "", wireApi: null, model: null, envKey: null, apiKey: null, apiKeySource: null, openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: entry.upstreamUserAgent, trustUpstream: entry.trustUpstream, sentinelProtect: entry.sentinelProtect, priority: entry.priority, capacity: entry.capacity ?? pool.defaultAccountCapacity, loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor, tempUnschedulable: entry.tempUnschedulable, authOpenAIKeyShape: existsSync(authPath) ? "unknown" : "missing", ok: false, error: null, }; try { if (!existsSync(configPath)) throw new Error(`config file ${configFile} is missing`); const parsed = Bun.TOML.parse(readFileSync(configPath, "utf8")) as unknown; if (!isRecord(parsed)) throw new Error("config is not a TOML object"); const providers = parsed.model_providers; if (!isRecord(providers)) throw new Error("model_providers is missing"); const provider = stringValue(parsed.model_provider) ?? Object.keys(providers)[0] ?? ""; if (provider === "") throw new Error("model_provider is missing"); const providerConfig = providers[provider]; if (!isRecord(providerConfig)) throw new Error(`model provider ${provider} is missing`); const baseUrl = normalizeBaseUrl(stringValue(providerConfig.base_url)); if (baseUrl === null) throw new Error(`model provider ${provider} base_url is missing or invalid`); base.provider = provider; base.baseUrl = baseUrl; base.envKey = stringValue(providerConfig.env_key); base.wireApi = stringValue(providerConfig.wire_api); base.model = stringValue(parsed.model); const auth = readAuthAPIKey(authPath); base.authOpenAIKeyShape = auth.shape; if (auth.apiKey !== null) { base.apiKey = auth.apiKey; base.apiKeySource = "auth-json"; } else if (base.envKey !== null && typeof process.env[base.envKey] === "string" && process.env[base.envKey]!.length > 0) { base.apiKey = process.env[base.envKey]!; base.apiKeySource = "env"; } if (base.apiKey === null || base.apiKey.length === 0) { throw new Error(base.envKey === null ? "auth OPENAI_API_KEY is missing or empty" : `auth OPENAI_API_KEY is missing and env ${base.envKey} is not present`); } base.ok = true; return base; } catch (error) { base.error = error instanceof Error ? error.message : String(error); return base; } }); } function resolveProfileFiles(codexDir: string, profile: CodexPoolProfileConfig): { configFile: string; authFile: string } { const configFile = existsSync(join(codexDir, profile.configFile)) || profile.fallbackConfigFile === null ? profile.configFile : profile.fallbackConfigFile; const authFile = existsSync(join(codexDir, profile.authFile)) || profile.fallbackAuthFile === null ? profile.authFile : profile.fallbackAuthFile; return { configFile, authFile }; } function readCodexPoolConfig(): CodexPoolConfig { if (!existsSync(codexPoolConfigPath)) throw new Error(`${codexPoolConfigPath} is required`); const parsed = Bun.YAML.parse(readFileSync(codexPoolConfigPath, "utf8")) as unknown; if (!isRecord(parsed)) throw new Error(`${codexPoolConfigPath} must contain a YAML object`); const version = integerConfigField(parsed, "version", ""); if (version !== 1) throw new Error(`${codexPoolConfigPath}.version must be 1`); const kind = requiredStringConfigField(parsed, "kind", ""); if (kind !== "platform-infra-sub2api-codex-pool") throw new Error(`${codexPoolConfigPath}.kind must be platform-infra-sub2api-codex-pool`); const metadata = requiredRecordConfigField(parsed, "metadata", ""); const pool = parsed.pool; if (!isRecord(pool)) throw new Error(`${codexPoolConfigPath}.pool must be a YAML object`); if (!isRecord(parsed.profiles)) throw new Error(`${codexPoolConfigPath}.profiles must be a YAML object`); rejectSchedulableYamlField(pool, "pool"); const defaultTempUnschedulable = readTempUnschedulablePolicy(pool.defaultTempUnschedulable, "pool.defaultTempUnschedulable"); const defaultAccountPriorityValue = readAccountPriority(pool.defaultAccountPriority, "pool.defaultAccountPriority"); const defaultAccountCapacityValue = readAccountCapacity(pool.defaultAccountCapacity, "pool.defaultAccountCapacity"); const defaultAccountLoadFactorValue = readAccountLoadFactor(pool.defaultAccountLoadFactor, "pool.defaultAccountLoadFactor"); const defaultSentinelProtect = readSentinelProtectPolicy(pool.defaultSentinelProtect, "pool.defaultSentinelProtect"); const profiles = readProfileConfig( parsed.profiles, defaultTempUnschedulable, defaultSentinelProtect, defaultAccountPriorityValue, ); const manualAccounts = readManualAccountsConfig(parsed.manualAccounts); assertProtectedManualAccountsNotManaged(profiles, manualAccounts); const declaredAccountCapacity = desiredProfileCapacityTotal(profiles, defaultAccountCapacityValue); const minOwnerConcurrencySource = pool.minOwnerConcurrency === undefined || pool.minOwnerConcurrency === null ? "auto" : "yaml"; const minOwnerConcurrency = minOwnerConcurrencySource === "auto" ? Math.max(1, declaredAccountCapacity) : readOwnerConcurrency(pool.minOwnerConcurrency, "pool.minOwnerConcurrency"); const config: CodexPoolConfig = { version, kind, metadata: { id: requiredStringConfigField(metadata, "id", "metadata"), owner: requiredStringConfigField(metadata, "owner", "metadata"), relatedIssues: integerArrayConfigField(metadata, "relatedIssues", "metadata"), }, groupName: requiredStringConfigField(pool, "groupName", "pool"), groupDescription: readRequiredDescription(pool.groupDescription, "pool.groupDescription", 300), apiKeyName: requiredStringConfigField(pool, "apiKeyName", "pool"), apiKeySecretName: requiredStringConfigField(pool, "apiKeySecretName", "pool"), apiKeySecretKey: requiredStringConfigField(pool, "apiKeySecretKey", "pool"), adminEmailDefault: readRequiredEmail(pool.adminEmailDefault, "pool.adminEmailDefault"), minOwnerBalanceUsd: readRequiredPositiveNumber(pool.minOwnerBalanceUsd, "pool.minOwnerBalanceUsd"), minOwnerConcurrency, minOwnerConcurrencySource, defaultAccountPriority: defaultAccountPriorityValue, defaultAccountCapacity: defaultAccountCapacityValue, defaultAccountLoadFactor: defaultAccountLoadFactorValue, defaultTempUnschedulable, defaultSentinelProtect, profiles, manualAccounts, publicExposure: readPublicExposureConfig(parsed.publicExposure), localCodex: readLocalCodexConfig(parsed.localCodex), sentinel: { ...readCodexPoolSentinelConfig(parsed.sentinel, codexPoolConfigPath), protectedManualAccounts: manualAccounts.protected.map((account) => account.accountName), }, }; validateKubernetesName(config.groupName, "pool.groupName", false); validateKubernetesName(config.apiKeySecretName, "pool.apiKeySecretName", true); if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(config.apiKeySecretKey)) { throw new Error(`${codexPoolConfigPath}.pool.apiKeySecretKey must be a valid secret key name`); } if (config.minOwnerBalanceUsd <= 0) throw new Error(`${codexPoolConfigPath}.pool.minOwnerBalanceUsd must be > 0`); if (declaredAccountCapacity > 0 && config.minOwnerConcurrency < declaredAccountCapacity) { throw new Error(`${codexPoolConfigPath}.pool.minOwnerConcurrency must be >= the sum of declared account capacities (${declaredAccountCapacity})`); } return config; } function readProfileConfig( value: unknown, defaultTempUnschedulable: CodexTempUnschedulablePolicy, defaultSentinelProtect: CodexSentinelProtectPolicy, defaultPriority: number, ): CodexPoolProfileConfig[] { if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.profiles must be a YAML object`); const entries = value.entries; if (!Array.isArray(entries)) throw new Error(`${codexPoolConfigPath}.profiles.entries must be a YAML array`); return entries.map((entry, index) => { if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] must be an object`); rejectSchedulableYamlField(entry, `profiles.entries[${index}]`); const profile = stringValue(entry.profile); const configFile = stringValue(entry.configFile); const authFile = stringValue(entry.authFile); if (profile === null || configFile === null || authFile === null) { throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] requires profile, configFile, and authFile`); } const accountName = stringValue(entry.accountName); if (accountName !== null) validateKubernetesName(accountName, `profiles.entries[${index}].accountName`, false); validateCodexFileName(configFile, `profiles.entries[${index}].configFile`); validateCodexFileName(authFile, `profiles.entries[${index}].authFile`); const fallbackConfigFile = stringValue(entry.fallbackConfigFile); const fallbackAuthFile = stringValue(entry.fallbackAuthFile); if (fallbackConfigFile !== null) validateCodexFileName(fallbackConfigFile, `profiles.entries[${index}].fallbackConfigFile`); if (fallbackAuthFile !== null) validateCodexFileName(fallbackAuthFile, `profiles.entries[${index}].fallbackAuthFile`); const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`); const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`); const trustUpstream = readTrustUpstream(entry.trustUpstream, `profiles.entries[${index}].trustUpstream`); const sentinelProtect = readSentinelProtectPolicy(entry.sentinelProtect, `profiles.entries[${index}].sentinelProtect`, defaultSentinelProtect); const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority); const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`); const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`); const tempUnschedulable = readTempUnschedulablePolicy(entry.tempUnschedulable, `profiles.entries[${index}].tempUnschedulable`, defaultTempUnschedulable); return { profile, accountName, configFile, authFile, fallbackConfigFile, fallbackAuthFile, openaiResponsesWebSocketsV2Mode, upstreamUserAgent, trustUpstream, sentinelProtect, priority, capacity, loadFactor, tempUnschedulable, }; }); } function desiredProfileCapacityTotal(profiles: CodexPoolProfileConfig[], defaultCapacity: number): number { return profiles.reduce((total, profile) => total + (profile.capacity ?? defaultCapacity), 0); } function readManualAccountsConfig(value: unknown): CodexPoolManualAccountsConfig { if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.manualAccounts must be a YAML object`); const bindingSources = readManualBindingSources(value.bindingSources, "manualAccounts.bindingSources"); const protectedRaw = value.protected; if (!Array.isArray(protectedRaw)) throw new Error(`${codexPoolConfigPath}.manualAccounts.protected must be a YAML array`); const seen = new Set(); const protectedAccounts = protectedRaw.map((entry, index): CodexPoolManualAccountProtection => { const key = `manualAccounts.protected[${index}]`; const accountName = typeof entry === "string" ? readManualAccountName(entry, key) : isRecord(entry) ? readManualAccountName(entry.accountName, `${key}.accountName`) : null; if (accountName === null) throw new Error(`${codexPoolConfigPath}.${key} must be an account name string or object with accountName`); const normalized = accountName.toLowerCase(); if (seen.has(normalized)) throw new Error(`${codexPoolConfigPath}.${key}.accountName is duplicated in manualAccounts.protected`); seen.add(normalized); const reason = isRecord(entry) ? readManualAccountReason(entry.reason, `${key}.reason`) : null; const proxyBinding = isRecord(entry) ? readManualAccountProxyBinding(entry.proxyBinding, `${key}.proxyBinding`, bindingSources) : null; const groupBinding = isRecord(entry) ? readManualAccountGroupBinding(entry.groupBinding, `${key}.groupBinding`, bindingSources) : null; return { accountName, reason, proxyBinding, groupBinding }; }); return { bindingSources, protected: protectedAccounts }; } function readManualBindingSources(value: unknown, key: string): CodexPoolManualBindingSourcesConfig { if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); const items: CodexPoolManualBindingSource[] = []; const byId: Record = {}; for (const [id, rawSource] of Object.entries(value)) { const path = `${key}.${id}`; validateManualBindingSourceId(id, path); if (!isRecord(rawSource)) throw new Error(`${codexPoolConfigPath}.${path} must be a YAML object`); const source: CodexPoolManualBindingSource = { id, enabled: readBooleanConfig(rawSource.enabled, `${path}.enabled`), kind: readManualBindingKind(rawSource.kind, `${path}.kind`), provider: readManualBindingProvider(rawSource.provider, `${path}.provider`), description: readManualAccountReason(rawSource.description, `${path}.description`), }; if (source.kind === "proxy" && source.provider !== "target-egress-proxy") { throw new Error(`${codexPoolConfigPath}.${path}.provider must be target-egress-proxy for kind=proxy`); } if (source.kind === "group" && source.provider !== "pool-group") { throw new Error(`${codexPoolConfigPath}.${path}.provider must be pool-group for kind=group`); } byId[id] = source; items.push(source); } if (items.length === 0) throw new Error(`${codexPoolConfigPath}.${key} must declare at least one source`); return { items, byId }; } function readManualAccountProxyBinding(value: unknown, key: string, bindingSources: CodexPoolManualBindingSourcesConfig): CodexPoolManualAccountProxyBinding | null { if (value === undefined || value === null) return null; if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); const enabled = readBooleanConfig(value.enabled, `${key}.enabled`); const source = readManualBindingSourceRef(value.source, `${key}.source`, bindingSources, "proxy", enabled); const proxyName = stringValue(value.proxyName); if (proxyName === null || proxyName.trim().length === 0) throw new Error(`${codexPoolConfigPath}.${key}.proxyName is required`); validateProxyName(proxyName, `${key}.proxyName`); return { enabled, source, proxyName, }; } function readManualAccountGroupBinding(value: unknown, key: string, bindingSources: CodexPoolManualBindingSourcesConfig): CodexPoolManualAccountGroupBinding | null { if (value === undefined || value === null) return null; if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); const enabled = readBooleanConfig(value.enabled, `${key}.enabled`); const source = readManualBindingSourceRef(value.source, `${key}.source`, bindingSources, "group", enabled); return { enabled, source, }; } function readManualBindingSourceRef(value: unknown, key: string, bindingSources: CodexPoolManualBindingSourcesConfig, expectedKind: CodexPoolManualBindingKind, bindingEnabled: boolean): string { const sourceId = stringValue(value); if (sourceId === null) throw new Error(`${codexPoolConfigPath}.${key} is required`); validateManualBindingSourceId(sourceId, key); const source = bindingSources.byId[sourceId]; if (source === undefined) throw new Error(`${codexPoolConfigPath}.${key} references unknown manualAccounts.bindingSources id ${sourceId}`); if (source.kind !== expectedKind) throw new Error(`${codexPoolConfigPath}.${key} references ${sourceId} with kind=${source.kind}; expected ${expectedKind}`); if (bindingEnabled && !source.enabled) throw new Error(`${codexPoolConfigPath}.${key} references disabled manualAccounts.bindingSources.${sourceId}`); return sourceId; } function readManualBindingKind(value: unknown, key: string): CodexPoolManualBindingKind { const text = stringValue(value); if (text !== "proxy" && text !== "group") throw new Error(`${codexPoolConfigPath}.${key} must be proxy or group`); return text; } function readManualBindingProvider(value: unknown, key: string): CodexPoolManualBindingProvider { const text = stringValue(value); if (text !== "target-egress-proxy" && text !== "pool-group") throw new Error(`${codexPoolConfigPath}.${key} must be target-egress-proxy or pool-group`); return text; } function validateManualBindingSourceId(value: string, key: string): void { if (!/^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported source id format`); } function readManualAccountName(value: unknown, key: string): string | null { const text = stringValue(value)?.trim() ?? null; if (text === null || text.length === 0) return null; if (text.length > 256) throw new Error(`${codexPoolConfigPath}.${key} must be at most 256 characters`); if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`); if (!/^[^<>"'`\\]+$/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} contains unsupported characters`); return text; } function readManualAccountReason(value: unknown, key: string): string | null { if (value === undefined || value === null) return null; const text = stringValue(value)?.trim() ?? null; if (text === null || text.length === 0) return null; if (text.length > 240) throw new Error(`${codexPoolConfigPath}.${key} must be at most 240 characters`); if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`); return text; } function assertProtectedManualAccountsNotManaged(profiles: CodexPoolProfileConfig[], manualAccounts: CodexPoolManualAccountsConfig): void { const protectedNames = new Set(manualAccounts.protected.map((account) => account.accountName.toLowerCase())); for (const [index, profile] of profiles.entries()) { const accountName = profile.accountName; if (accountName !== null && protectedNames.has(accountName.toLowerCase())) { throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}].accountName is listed in manualAccounts.protected; protected manual accounts must not be YAML-managed`); } } } function rejectSchedulableYamlField(value: Record, key: string): void { if (Object.prototype.hasOwnProperty.call(value, "schedulable")) { throw new Error(`${codexPoolConfigPath}.${key}.schedulable is process control, not durable YAML config; use codex-pool sync actions instead`); } } function readOpenAIResponsesWebSocketsV2Mode(value: unknown, key: string): OpenAIResponsesWebSocketsV2Mode | null { if (value === undefined || value === null) return null; const text = stringValue(value); if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`); if (text === "off" || text === "ctx_pool" || text === "passthrough") return text; throw new Error(`${codexPoolConfigPath}.${key} must be one of off, ctx_pool, passthrough`); } function readUpstreamUserAgent(value: unknown, key: string): string | null { if (value === undefined || value === null) return null; const text = stringValue(value); if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`); if (text.length > 512) throw new Error(`${codexPoolConfigPath}.${key} must be at most 512 characters`); if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`); return text; } function readTrustUpstream(value: unknown, key: string): boolean { if (value === undefined || value === null) return false; const parsed = booleanValue(value); if (parsed === null) throw new Error(`${codexPoolConfigPath}.${key} must be a boolean`); return parsed; } function readSentinelProtectPolicy(value: unknown, key: string, fallback?: CodexSentinelProtectPolicy): CodexSentinelProtectPolicy { if (value === undefined || value === null) { if (fallback !== undefined) return fallback; throw new Error(`${codexPoolConfigPath}.${key} is required`); } if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback?.enabled); const required = ["consecutiveFailures", "initialRetryDelaySeconds", "maxRetryDelaySeconds", "backoffMultiplier"]; for (const field of required) { if (fallback === undefined && (value[field] === undefined || value[field] === null)) { throw new Error(`${codexPoolConfigPath}.${key}.${field} is required`); } if (enabled && value[field] === undefined && fallback === undefined) { throw new Error(`${codexPoolConfigPath}.${key}.${field} is required when enabled=true`); } } const consecutiveFailures = value.consecutiveFailures === undefined || value.consecutiveFailures === null ? fallback?.consecutiveFailures : readBoundedInteger(value.consecutiveFailures, `${key}.consecutiveFailures`, 1, 20); const initialRetryDelaySeconds = value.initialRetryDelaySeconds === undefined || value.initialRetryDelaySeconds === null ? fallback?.initialRetryDelaySeconds : readBoundedInteger(value.initialRetryDelaySeconds, `${key}.initialRetryDelaySeconds`, 1, 3600); const maxRetryDelaySeconds = value.maxRetryDelaySeconds === undefined || value.maxRetryDelaySeconds === null ? fallback?.maxRetryDelaySeconds : readBoundedInteger(value.maxRetryDelaySeconds, `${key}.maxRetryDelaySeconds`, 1, 3600); const backoffMultiplier = value.backoffMultiplier === undefined || value.backoffMultiplier === null ? fallback?.backoffMultiplier : readBoundedInteger(value.backoffMultiplier, `${key}.backoffMultiplier`, 1, 10); if (consecutiveFailures === undefined || initialRetryDelaySeconds === undefined || maxRetryDelaySeconds === undefined || backoffMultiplier === undefined) { throw new Error(`${codexPoolConfigPath}.${key} is missing required sentinel protection fields`); } if (maxRetryDelaySeconds < initialRetryDelaySeconds) { throw new Error(`${codexPoolConfigPath}.${key}.maxRetryDelaySeconds must be >= initialRetryDelaySeconds`); } return { enabled, consecutiveFailures, initialRetryDelaySeconds, maxRetryDelaySeconds, backoffMultiplier, }; } function readBoundedInteger(value: unknown, key: string, min: number, max: number): number { const parsed = numberValue(value); if (parsed === null || !Number.isInteger(parsed) || parsed < min || parsed > max) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer from ${min} to ${max}`); } return parsed; } function readAccountPriority(value: unknown, key: string, fallback?: number): number { if (value === undefined || value === null) { if (fallback !== undefined) return fallback; throw new Error(`${codexPoolConfigPath}.${key} is required`); } const priority = numberValue(value); if (priority === null || !Number.isInteger(priority) || priority < 0 || priority > 1000) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 0 to 1000`); } return priority; } function readAccountCapacity(value: unknown, key: string): number { const capacity = numberValue(value); if (capacity === null || !Number.isInteger(capacity) || capacity < 1 || capacity > 1000) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`); } return capacity; } function readOwnerConcurrency(value: unknown, key: string): number { const concurrency = numberValue(value); if (concurrency === null || !Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100000) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 100000`); } return concurrency; } function readAccountLoadFactor(value: unknown, key: string): number { const loadFactor = numberValue(value); if (loadFactor === null || !Number.isInteger(loadFactor) || loadFactor < 1 || loadFactor > 1000) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`); } return loadFactor; } function readCaddyTimeoutSeconds(value: unknown, key: string): number { if (value === undefined || value === null) throw new Error(`${codexPoolConfigPath}.${key} is required`); const seconds = numberValue(value); if (seconds === null || !Number.isInteger(seconds) || seconds < 30 || seconds > 900) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 30 to 900`); } return seconds; } function readCaddyEdgeRetryConfig(value: unknown, key: string): CodexPoolCaddyEdgeRetryConfig { if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); const enabled = readBooleanConfig(value.enabled, `${key}.enabled`); const tryDurationSeconds = readPositiveInteger(value.tryDurationSeconds, `${key}.tryDurationSeconds`); const tryIntervalMilliseconds = readPositiveInteger(value.tryIntervalMilliseconds, `${key}.tryIntervalMilliseconds`); const retryMatchValue = isRecord(value.retryMatch) ? value.retryMatch : null; if (retryMatchValue === null) throw new Error(`${codexPoolConfigPath}.${key}.retryMatch must be a YAML object`); const retryMatch = { methods: readCaddyRetryMethods(retryMatchValue.methods, `${key}.retryMatch.methods`), paths: readCaddyRetryPaths(retryMatchValue.paths, `${key}.retryMatch.paths`), }; if (retryMatch.methods.length === 0 && retryMatch.paths.length === 0) { throw new Error(`${codexPoolConfigPath}.${key}.retryMatch must include at least one method or path matcher when enabled=true`); } return { enabled, tryDurationSeconds, tryIntervalMilliseconds, retryMatch }; } function readPositiveInteger(value: unknown, key: string): number { const parsed = numberValue(value); if (parsed === null || !Number.isInteger(parsed) || parsed < 1) { throw new Error(`${codexPoolConfigPath}.${key} must be a positive integer`); } return parsed; } function readPositiveIntegerConfig(value: unknown, key: string): number { if (value === undefined || value === null) throw new Error(`${codexPoolConfigPath}.${key} is required`); return readPositiveInteger(value, key); } function readCaddyRetryMethods(value: unknown, key: string): string[] { if (value === undefined || value === null) return []; if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`); const seen = new Set(); const methods: string[] = []; for (const item of value) { const method = stringValue(item)?.toUpperCase() ?? null; if (method === null || !/^[A-Z][A-Z0-9_-]*$/u.test(method)) throw new Error(`${codexPoolConfigPath}.${key} entries must be HTTP method tokens`); if (seen.has(method)) continue; seen.add(method); methods.push(method); } return methods; } function readCaddyRetryPaths(value: unknown, key: string): string[] { if (value === undefined || value === null) return []; if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`); const seen = new Set(); const paths: string[] = []; for (const item of value) { const path = stringValue(item); if (path === null || !path.startsWith("/") || /[\r\n]/u.test(path)) { throw new Error(`${codexPoolConfigPath}.${key} entries must be Caddy path matchers starting with / and without newlines`); } if (seen.has(path)) continue; seen.add(path); paths.push(path); } return paths; } function readTempUnschedulablePolicy(value: unknown, key: string, fallback?: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy { if (value === undefined || value === null) { if (fallback !== undefined) return cloneTempUnschedulablePolicy(fallback); throw new Error(`${codexPoolConfigPath}.${key} is required`); } if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback?.enabled); let rules: CodexTempUnschedulableRule[]; if (value.rules === undefined || value.rules === null) { if (fallback === undefined) throw new Error(`${codexPoolConfigPath}.${key}.rules is required`); rules = cloneTempUnschedulablePolicy(fallback).rules; } else { rules = readTempUnschedulableRules(value.rules, `${key}.rules`); } if (enabled && rules.length === 0) throw new Error(`${codexPoolConfigPath}.${key}.rules must not be empty when enabled=true`); return { enabled, rules }; } function readTempUnschedulableRules(value: unknown, key: string): CodexTempUnschedulableRule[] { if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`); return value.map((entry, index) => { if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.${key}[${index}] must be an object`); const statusCode = numberValue(entry.statusCode ?? entry.errorCode); if (statusCode === null || !Number.isInteger(statusCode) || statusCode < 100 || statusCode > 599) { throw new Error(`${codexPoolConfigPath}.${key}[${index}].statusCode must be an HTTP status code from 100 to 599`); } const durationMinutes = numberValue(entry.durationMinutes); if (durationMinutes === null || !Number.isInteger(durationMinutes) || durationMinutes < 1 || durationMinutes > 1440) { throw new Error(`${codexPoolConfigPath}.${key}[${index}].durationMinutes must be an integer from 1 to 1440`); } const keywords = readTempUnschedulableKeywords(entry.keywords, `${key}[${index}].keywords`); const description = stringValue(entry.description); if (description !== null && description.length > 240) throw new Error(`${codexPoolConfigPath}.${key}[${index}].description must be at most 240 characters`); return { statusCode, keywords, durationMinutes, description }; }); } function readTempUnschedulableKeywords(value: unknown, key: string): string[] { if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`); const seen = new Set(); const keywords: string[] = []; for (const item of value) { const keyword = stringValue(item); if (keyword === null) throw new Error(`${codexPoolConfigPath}.${key} entries must be non-empty strings`); if (keyword.length > 120) throw new Error(`${codexPoolConfigPath}.${key} entries must be at most 120 characters`); if (/[\r\n]/u.test(keyword)) throw new Error(`${codexPoolConfigPath}.${key} entries must not contain newlines`); const normalized = keyword.toLowerCase(); if (seen.has(normalized)) continue; seen.add(normalized); keywords.push(keyword); } if (keywords.length === 0) throw new Error(`${codexPoolConfigPath}.${key} must not be empty`); return keywords; } function cloneTempUnschedulablePolicy(policy: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy { return { enabled: policy.enabled, rules: policy.rules.map((rule) => ({ statusCode: rule.statusCode, keywords: [...rule.keywords], durationMinutes: rule.durationMinutes, description: rule.description, })), }; } function readBooleanConfig(value: unknown, key: string, fallback?: boolean): boolean { if (value === undefined || value === null) { if (fallback !== undefined) return fallback; throw new Error(`${codexPoolConfigPath}.${key} is required`); } const parsed = booleanValue(value); if (parsed === null) throw new Error(`${codexPoolConfigPath}.${key} must be a boolean`); return parsed; } function readPublicExposureConfig(value: unknown): CodexPoolPublicExposureConfig { if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.publicExposure must be a YAML object`); const masterFrpsValue = requiredRecordConfigField(value, "masterFrps", "publicExposure"); const masterCaddyValue = requiredRecordConfigField(value, "masterCaddy", "publicExposure"); const config: CodexPoolPublicExposureConfig = { enabled: readBooleanConfig(value.enabled, "publicExposure.enabled"), proxyName: requiredStringConfigField(value, "proxyName", "publicExposure"), configMapName: requiredStringConfigField(value, "configMapName", "publicExposure"), deploymentName: requiredStringConfigField(value, "deploymentName", "publicExposure"), frpcImage: requiredStringConfigField(value, "frpcImage", "publicExposure"), serverAddr: requiredStringConfigField(value, "serverAddr", "publicExposure"), serverPort: readRequiredPort(value.serverPort, "publicExposure.serverPort"), remotePort: readRequiredPort(value.remotePort, "publicExposure.remotePort"), localIP: requiredStringConfigField(value, "localIP", "publicExposure"), localPort: readRequiredPort(value.localPort, "publicExposure.localPort"), publicBaseUrl: readRequiredBaseUrl(value.publicBaseUrl, "publicExposure.publicBaseUrl"), masterBaseUrl: readRequiredBaseUrl(value.masterBaseUrl, "publicExposure.masterBaseUrl"), masterFrps: { configPath: requiredStringConfigField(masterFrpsValue, "configPath", "publicExposure.masterFrps"), containerName: requiredStringConfigField(masterFrpsValue, "containerName", "publicExposure.masterFrps"), }, masterCaddy: { enabled: readBooleanConfig(masterCaddyValue.enabled, "publicExposure.masterCaddy.enabled"), domain: requiredStringConfigField(masterCaddyValue, "domain", "publicExposure.masterCaddy"), configPath: requiredStringConfigField(masterCaddyValue, "configPath", "publicExposure.masterCaddy"), serviceName: requiredStringConfigField(masterCaddyValue, "serviceName", "publicExposure.masterCaddy"), upstreamBaseUrl: readRequiredBaseUrl(masterCaddyValue.upstreamBaseUrl, "publicExposure.masterCaddy.upstreamBaseUrl"), responseHeaderTimeoutSeconds: readCaddyTimeoutSeconds( masterCaddyValue.responseHeaderTimeoutSeconds, "publicExposure.masterCaddy.responseHeaderTimeoutSeconds", ), edgeRetry: readCaddyEdgeRetryConfig( masterCaddyValue.edgeRetry, "publicExposure.masterCaddy.edgeRetry", ), }, }; validateKubernetesName(config.configMapName, "publicExposure.configMapName", true); validateKubernetesName(config.deploymentName, "publicExposure.deploymentName", true); validateProxyName(config.proxyName, "publicExposure.proxyName"); validatePort(config.serverPort, "publicExposure.serverPort"); validatePort(config.remotePort, "publicExposure.remotePort"); validatePort(config.localPort, "publicExposure.localPort"); if (!/^[A-Za-z0-9._:/-]+$/u.test(config.frpcImage)) throw new Error(`${codexPoolConfigPath}.publicExposure.frpcImage has an unsupported format`); if (!/^[A-Za-z0-9._:-]+$/u.test(config.serverAddr)) throw new Error(`${codexPoolConfigPath}.publicExposure.serverAddr has an unsupported format`); if (!/^[A-Za-z0-9._:-]+$/u.test(config.localIP)) throw new Error(`${codexPoolConfigPath}.publicExposure.localIP has an unsupported format`); if (!config.masterFrps.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterFrps.configPath must be absolute`); validateProxyName(config.masterFrps.containerName, "publicExposure.masterFrps.containerName"); validatePublicHostname(config.masterCaddy.domain, "publicExposure.masterCaddy.domain"); if (!config.masterCaddy.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.configPath must be absolute`); validateProxyName(config.masterCaddy.serviceName, "publicExposure.masterCaddy.serviceName"); const upstream = new URL(config.masterCaddy.upstreamBaseUrl); if (upstream.protocol !== "http:") throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.upstreamBaseUrl must use http:// for the local upstream`); return config; } function readLocalCodexConfig(value: unknown): CodexPoolLocalCodexConfig { if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.localCodex must be a YAML object`); const config = { backupSuffix: requiredStringConfigField(value, "backupSuffix", "localCodex"), providerName: requiredStringConfigField(value, "providerName", "localCodex"), wireApi: requiredStringConfigField(value, "wireApi", "localCodex"), modelContextWindow: readPositiveIntegerConfig(value.modelContextWindow, "localCodex.modelContextWindow"), modelAutoCompactTokenLimit: readPositiveIntegerConfig(value.modelAutoCompactTokenLimit, "localCodex.modelAutoCompactTokenLimit"), supportsWebSockets: readBooleanConfig(value.supportsWebSockets, "localCodex.supportsWebSockets"), responsesWebSocketsV2: readBooleanConfig(value.responsesWebSocketsV2, "localCodex.responsesWebSocketsV2"), responsesSmokeModel: requiredStringConfigField(value, "responsesSmokeModel", "localCodex"), }; if (!/^[A-Za-z0-9._-]+$/u.test(config.backupSuffix)) throw new Error(`${codexPoolConfigPath}.localCodex.backupSuffix has an unsupported format`); validateProxyName(config.providerName, "localCodex.providerName"); validateProxyName(config.wireApi, "localCodex.wireApi"); validateModelName(config.responsesSmokeModel, "localCodex.responsesSmokeModel"); return config; } function validateCodexFileName(value: string, key: string): void { if (!/^(config\.toml|config\.toml\.[A-Za-z0-9._-]+|auth\.json|auth\.json\.[A-Za-z0-9._-]+)$/u.test(value)) { throw new Error(`${codexPoolConfigPath}.${key} has an unsupported file name`); } } function validateProxyName(value: string, key: string): void { if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`); } function validateModelName(value: string, key: string): void { if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported model name`); } function validatePublicHostname(value: string, key: string): void { if (value.length > 253 || !/^[A-Za-z0-9.-]+$/u.test(value) || value.startsWith(".") || value.endsWith(".")) { throw new Error(`${codexPoolConfigPath}.${key} has an unsupported hostname format`); } } function validatePort(value: number, key: string): void { if (!Number.isInteger(value) || value < 1 || value > 65535) throw new Error(`${codexPoolConfigPath}.${key} must be an integer port`); } function readAuthAPIKey(authPath: string): { apiKey: string | null; shape: string } { if (!existsSync(authPath)) return { apiKey: null, shape: "missing" }; const parsed = JSON.parse(readFileSync(authPath, "utf8")) as unknown; if (!isRecord(parsed)) return { apiKey: null, shape: "non-object" }; const value = parsed.OPENAI_API_KEY; const shape = value === null ? "null" : Array.isArray(value) ? "array" : typeof value; if (typeof value === "string" && value.length > 0) return { apiKey: value, shape }; return { apiKey: null, shape }; } function uniqueAccountName(profile: string, seen: Set): string { const normalized = profile .toLowerCase() .replace(/[^a-z0-9._-]+/gu, "-") .replace(/^-+|-+$/gu, "") || "default"; let candidate = `unidesk-codex-${normalized}`; let counter = 2; while (seen.has(candidate)) { candidate = `unidesk-codex-${normalized}-${counter}`; counter += 1; } seen.add(candidate); return candidate; } function redactProfile(profile: CodexProfile): Record { return { profile: profile.profile, accountName: profile.accountName, configFile: profile.configFile, authFile: profile.authFile, provider: profile.provider || null, baseUrl: profile.baseUrl || null, wireApi: profile.wireApi, model: profile.model, envKey: profile.envKey, apiKeySource: profile.apiKeySource, openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, sentinelProtect: profile.sentinelProtect, priority: profile.priority, capacity: profile.capacity, loadFactor: profile.loadFactor, tempUnschedulable: tempUnschedulableSummary(profile.tempUnschedulable), apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0, apiKeyBytes: profile.apiKey === null ? 0 : Buffer.byteLength(profile.apiKey, "utf8"), apiKeyFingerprint: profile.apiKey === null ? null : fingerprint(profile.apiKey), authOpenAIKeyShape: profile.authOpenAIKeyShape, ok: profile.ok, error: profile.error, valuesPrinted: false, }; } function compactProfile(profile: CodexProfile): Record { return { profile: profile.profile, accountName: profile.accountName, provider: profile.provider || null, model: profile.model, priority: profile.priority, trustUpstream: profile.trustUpstream, sentinelProtectEnabled: profile.sentinelProtect.enabled, sentinelProtectConsecutiveFailures: profile.sentinelProtect.enabled ? profile.sentinelProtect.consecutiveFailures : undefined, capacity: profile.capacity, loadFactor: profile.loadFactor, tempUnschedulableEnabled: profile.tempUnschedulable.enabled, tempUnschedulableRuleCount: profile.tempUnschedulable.rules.length, apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0, ok: profile.ok, error: profile.error, valuesPrinted: false, }; } export function renderSub2ApiTempUnschedulableCredentials(policy: CodexTempUnschedulablePolicy): Record { if (!policy.enabled) return {}; return { temp_unschedulable_enabled: policy.enabled, temp_unschedulable_rules: policy.rules.map((rule) => ({ error_code: rule.statusCode, keywords: [...rule.keywords], duration_minutes: rule.durationMinutes, description: rule.description ?? "", })), }; } function tempUnschedulableSummary(policy: CodexTempUnschedulablePolicy): Record { return { enabled: policy.enabled, ruleCount: policy.rules.length, statusCodes: policy.rules.map((rule) => rule.statusCode), }; } function codexPoolConfigSummary(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record { const accountCapacityTotal = desiredAccountCapacityTotal(pool); return { version: pool.version, kind: pool.kind, metadata: pool.metadata, groupName: pool.groupName, groupDescription: pool.groupDescription, apiKeyName: pool.apiKeyName, apiKeySecretName: pool.apiKeySecretName, apiKeySecretKey: pool.apiKeySecretKey, adminEmailDefault: pool.adminEmailDefault, minOwnerBalanceUsd: pool.minOwnerBalanceUsd, minOwnerConcurrency: pool.minOwnerConcurrency, minOwnerConcurrencySource: pool.minOwnerConcurrencySource, accountCapacityTotal, defaultAccountPriority: pool.defaultAccountPriority, defaultAccountCapacity: pool.defaultAccountCapacity, defaultAccountLoadFactor: pool.defaultAccountLoadFactor, defaultTempUnschedulable: tempUnschedulableSummary(pool.defaultTempUnschedulable), defaultSentinelProtect: pool.defaultSentinelProtect, profileCount: pool.profiles.length, manualAccounts: { bindingSources: pool.manualAccounts.bindingSources.items.map(manualBindingSourcePlan), protectedCount: pool.manualAccounts.protected.length, protected: pool.manualAccounts.protected, controlPolicy: "manual accounts are not created, credential-updated, pruned, probed, or frozen by UniDesk codex-pool sync/sentinel; optional proxy_id and pool group membership bindings are narrow YAML-controlled exceptions", }, publicExposure: targetPublicExposureSummary(target), localCodex: pool.localCodex, sentinel: codexPoolSentinelSummary(pool.sentinel), disclosure: { full: "bun scripts/cli.ts platform-infra sub2api codex-pool plan --full", }, valuesPrinted: false, }; } function compactText(value: unknown, limit = 260): string { if (typeof value !== "string") return ""; return value.length > limit ? `${value.slice(0, limit)}…` : value; } function recordArray(value: unknown): Record[] { return Array.isArray(value) ? value.filter(isRecord) : []; } function pickSummaryFields(item: Record, keys: string[]): Record { const result: Record = {}; for (const key of keys) { if (item[key] !== undefined) result[key] = item[key]; } return result; } function compactStatusBlock(block: unknown, keys: string[]): Record | null { if (!isRecord(block)) return null; const items = recordArray(block.items); const attentionItems = items .filter((item) => item.ok === false) .map((item) => pickSummaryFields(item, keys)); return { ok: block.ok, desired: block.desired, missing: block.missing, mismatched: block.mismatched, totals: block.totals, itemCount: items.length, attentionItems, valuesPrinted: false, }; } function compactTempUnschedulableStatus(block: unknown): Record | null { if (!isRecord(block)) return null; const items = recordArray(block.items); const compactItems = items.map((item) => { const result = pickSummaryFields(item, [ "accountName", "accountId", "status", "schedulable", "tempUnschedulableUntil", "tempUnschedulableSet", "ok", ]); if (item.tempUnschedulableReasonPreview !== undefined) { result.tempUnschedulableReason = tempUnschedulableReasonSummary(item.tempUnschedulableReasonPreview); } return result; }); const frozenItems = compactItems.filter((item) => item.tempUnschedulableSet === true); const focusedFrozenItems = uniqueByAccountName([ ...frozenItems.filter((item) => isRecord(item.tempUnschedulableReason) && item.tempUnschedulableReason.statusCode === 400), ...frozenItems.filter((item) => item.schedulable === false), ...frozenItems, ]).slice(0, 6); return { ok: block.ok, desired: block.desired, enabledCount: Array.isArray(block.enabled) ? block.enabled.length : undefined, missing: block.missing, mismatched: block.mismatched, itemCount: compactItems.length, frozenCount: frozenItems.length, frozenShown: focusedFrozenItems.length, frozen: focusedFrozenItems.map((item) => pickSummaryFields(item, [ "accountName", "accountId", "schedulable", "tempUnschedulableUntil", "tempUnschedulableReason", ])), manuallyUnschedulable: compactItems .filter((item) => item.schedulable === false && item.tempUnschedulableSet !== true) .map((item) => pickSummaryFields(item, ["accountName", "accountId", "status", "schedulable"])), valuesPrinted: false, }; } function compactManualAccounts(block: unknown): Record | null { if (!isRecord(block)) return null; const items = recordArray(block.items).map((item) => pickSummaryFields(item, [ "accountName", "reason", "ok", "exists", "accountId", "status", "schedulable", "inYamlProfiles", "runtimeMarkedUnideskManaged", "proxyBinding", "groupBinding", "controlPolicy", ])); const proxySync = isRecord(block.proxySync) ? { ok: block.proxySync.ok, itemCount: block.proxySync.itemCount, items: recordArray(block.proxySync.items).map((item) => pickSummaryFields(item, [ "accountName", "accountId", "enabled", "ok", "action", "proxyAction", "expectedProxyName", "proxyId", "runtimeProxyId", "runtimeProxyName", "bindingAligned", "controlPolicy", ])), valuesPrinted: false, } : undefined; const groupSync = isRecord(block.groupSync) ? { ok: block.groupSync.ok, itemCount: block.groupSync.itemCount, items: recordArray(block.groupSync.items).map((item) => pickSummaryFields(item, [ "accountName", "accountId", "enabled", "ok", "action", "source", "poolGroupName", "poolGroupId", "bindingAligned", "controlPolicy", ])), valuesPrinted: false, } : undefined; return { ok: block.ok, protectedCount: block.protectedCount, items, proxySync, groupSync, valuesPrinted: false, }; } function tempUnschedulableReasonSummary(value: unknown): Record { const reason = compactText(value, 180); const statusCodeMatch = reason.match(/"status_code":(\d{3})/u) ?? reason.match(/OpenAI\s+(\d{3})/u) ?? reason.match(/\((\d{3})\)/u); const keywordMatch = reason.match(/"matched_keyword":"([^"]+)"/u); const summary: Record = { statusCode: statusCodeMatch ? Number(statusCodeMatch[1]) : undefined, matchedKeyword: keywordMatch ? keywordMatch[1] : undefined, }; if (!statusCodeMatch || !keywordMatch) summary.preview = compactText(reason, 120); return summary; } function uniqueByAccountName(items: Record[]): Record[] { const seen = new Set(); const result: Record[] = []; for (const item of items) { const key = item.accountName ?? item.accountId; if (seen.has(key)) continue; seen.add(key); result.push(item); } return result; } function compactRuntimeCapability(block: unknown): unknown { if (!isRecord(block)) return block; const probe = isRecord(block.probe) ? block.probe : {}; const logEvidence = isRecord(probe.logEvidence) ? probe.logEvidence : {}; const accountState = isRecord(probe.accountState) ? probe.accountState : (isRecord(probe.badAccountState) ? probe.badAccountState : {}); const resources = isRecord(block.resources) ? block.resources : {}; const requirement = isRecord(block.requirement) ? block.requirement : {}; return { ok: block.ok, required: block.required, capability: block.capability, outcome: block.outcome, requirement: Object.keys(requirement).length === 0 ? undefined : { statusCode: requirement.statusCode, representativeKeyword: requirement.representativeKeyword, durationMinutes: requirement.durationMinutes, sourceAccountName: requirement.sourceAccountName, }, requestEvidence: Object.keys(probe).length === 0 ? undefined : { requestId: probe.requestId, durationMs: probe.durationMs, httpStatus: probe.httpStatus, responseOk: probe.responseOk, accountState: Object.keys(accountState).length === 0 ? undefined : { accountId: accountState.accountId, status: accountState.status, schedulable: accountState.schedulable, tempUnschedulableUntil: accountState.tempUnschedulableUntil, tempUnschedulableSet: accountState.tempUnschedulableSet, tempUnschedulableReasonPreview: compactText(accountState.tempUnschedulableReasonPreview, 180), }, logEvidence: Object.keys(logEvidence).length === 0 ? undefined : { matchedLogLineCount: logEvidence.matchedLogLineCount, failovers: logEvidence.failovers, final: logEvidence.final, }, bodyPreview: probe.responseOk === false ? compactText(probe.bodyPreview, 240) : undefined, }, resources: Object.keys(resources).length === 0 ? undefined : { groupId: resources.groupId, accountId: resources.accountId, badAccountId: resources.badAccountId, goodAccountId: resources.goodAccountId, apiKeyId: resources.apiKeyId, valuesPrinted: false, }, message: block.message, valuesPrinted: false, }; } function compactGatewayModels(block: unknown): unknown { if (!isRecord(block)) return block; return pickSummaryFields(block, [ "ok", "httpStatus", "modelCount", "method", "serviceDns", "valuesPrinted", ]); } function compactGatewayResponses(block: unknown): unknown { if (!isRecord(block)) return block; const evidence = isRecord(block.evidence) ? block.evidence : {}; return { ok: block.ok, degraded: block.degraded, outcome: block.outcome, httpStatus: block.httpStatus, method: block.method, model: block.model, requestId: block.requestId, durationMs: block.durationMs, outputTextPreview: block.outputTextPreview, evidence: Object.keys(evidence).length === 0 ? undefined : { matchedLogLineCount: evidence.matchedLogLineCount, failoverCount: Array.isArray(evidence.failovers) ? evidence.failovers.length : undefined, failovers: Array.isArray(evidence.failovers) ? evidence.failovers.slice(-5) : undefined, final: evidence.final, }, valuesPrinted: false, }; } function compactGatewayResponsesRecent(block: unknown): unknown { if (!isRecord(block)) return block; return { ok: block.ok, degraded: block.degraded, window: block.window, tailLines: block.tailLines, failoverCount: block.failoverCount, forwardFailureCount: block.forwardFailureCount, finalErrorCount: block.finalErrorCount, slowFinalErrorCount: block.slowFinalErrorCount, contextCanceledCount: block.contextCanceledCount, ignoredProbeNoiseCount: block.ignoredProbeNoiseCount, failoverBudgetExhaustedCount: Array.isArray(block.failoverBudgetExhausted) ? block.failoverBudgetExhausted.length : undefined, failoverBudgetExhausted: Array.isArray(block.failoverBudgetExhausted) ? block.failoverBudgetExhausted.slice(-3).reverse() : undefined, recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined, recentForwardFailures: Array.isArray(block.recentForwardFailures) ? block.recentForwardFailures.slice(-4).reverse().map((item) => ({ ...item, errorPreview: compactText(item.errorPreview, 220), })) : undefined, recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-3).reverse() : undefined, recentSlowFinalErrors: Array.isArray(block.recentSlowFinalErrors) ? block.recentSlowFinalErrors.slice(-3).reverse() : undefined, recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-3).reverse() : undefined, logsExitCode: block.logsExitCode, valuesPrinted: false, }; } function compactGatewayCompactRecent(block: unknown): unknown { if (!isRecord(block)) return block; return { ok: block.ok, degraded: block.degraded, window: block.window, tailLines: block.tailLines, failureCount: block.failureCount, successCount: block.successCount, failoverCount: block.failoverCount, finalErrorCount: block.finalErrorCount, contextCanceledCount: block.contextCanceledCount, recentFailures: Array.isArray(block.recentFailures) ? block.recentFailures.slice(-2).reverse() : undefined, recentSuccesses: Array.isArray(block.recentSuccesses) ? block.recentSuccesses.slice(-2).reverse() : undefined, recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined, recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-2).reverse() : undefined, recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-2).reverse() : undefined, logsExitCode: block.logsExitCode, valuesPrinted: false, }; } function compactSentinelErrorDetails(value: unknown): Record | undefined { if (!isRecord(value)) return undefined; const body = isRecord(value.body) ? value.body : {}; const openaiError = isRecord(value.openaiError) ? value.openaiError : {}; return { kind: value.kind, statusCode: value.statusCode, code: value.code ?? body.code ?? openaiError.code, type: value.type ?? body.type ?? openaiError.type, bodyHash: value.bodyHash, valuesPrinted: false, }; } function compactSentinelQuarantine(item: unknown): Record { if (!isRecord(item)) return {}; const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {}; return { accountName: item.accountName, until: item.until, applied: item.applied, reason: item.reason, failureKind: item.failureKind, intervalMinutes: item.intervalMinutes, sentinelProtect: Object.keys(protect).length > 0 ? { enabled: protect.enabled, decision: protect.decision, failureCount: protect.failureCount, threshold: protect.threshold, } : undefined, error: compactSentinelErrorDetails(item.errorDetails), }; } function compactSentinelRecentAccount(item: unknown): Record { if (!isRecord(item)) return {}; const action = isRecord(item.action) ? item.action : {}; const error = compactSentinelErrorDetails(item.errorDetails); const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {}; return { accountName: item.accountName, lastProbeAt: item.lastProbeAt, lastStatus: item.lastStatus, nextProbeAfter: item.nextProbeAfter, ok: item.ok, purpose: item.purpose, httpStatus: item.httpStatus, durationMs: item.durationMs, markerMatched: item.markerMatched, sentinelProtect: Object.keys(protect).length > 0 ? { enabled: protect.enabled, decision: protect.decision, protected: protect.protected, failureCount: protect.failureCount, threshold: protect.threshold, } : undefined, failureKind: item.failureKind, requestShape: item.requestShape, action: Object.keys(action).length > 0 ? { taken: action.taken, type: action.type, } : undefined, errorStatusCode: error?.statusCode, errorCode: error?.code, errorBodyHash: error?.bodyHash, }; } function compactSentinelLastRun(value: unknown): Record | undefined { if (!isRecord(value)) return undefined; return { at: value.at, monitorEnabled: value.monitorEnabled, actionsEnabled: value.actionsEnabled, profileCount: value.profileCount, selected: value.selected, okCount: value.okCount, mismatchCount: value.mismatchCount, markerMismatchCount: value.markerMismatchCount, transportFailureCount: value.transportFailureCount, actionsTaken: value.actionsTaken, gatewayFailureMonitor: value.gatewayFailureMonitor, selection: value.selection, reconcileCount: Array.isArray(value.reconcile) ? value.reconcile.length : undefined, }; } function compactSentinelStatus(block: unknown): unknown { if (!isRecord(block)) return block; const runtime = isRecord(block.runtime) ? block.runtime : block; const desired = isRecord(runtime.desired) ? runtime.desired : {}; const cronJob = isRecord(runtime.cronJob) ? runtime.cronJob : {}; const secret = isRecord(runtime.secret) ? runtime.secret : {}; const configMap = isRecord(runtime.configMap) ? runtime.configMap : {}; const state = isRecord(runtime.state) ? runtime.state : {}; const freezeReassert = isRecord(block.freezeReassert) ? block.freezeReassert : {}; const qualityGatePrepare = isRecord(block.qualityGatePrepare) ? block.qualityGatePrepare : {}; const qualityGate = isRecord(block.qualityGate) ? block.qualityGate : {}; const quarantined = recordArray(state.quarantined).map(compactSentinelQuarantine); const recentAccounts = recordArray(state.recentAccounts).map(compactSentinelRecentAccount); const recentAttention = recentAccounts.filter((item) => item.ok === false || isRecord(item.action) && item.action.taken === true); const recentHealthy = recentAccounts .filter((item) => item.ok === true) .slice(-3) .map((item) => pickSummaryFields(item, ["accountName", "lastProbeAt", "lastStatus", "nextProbeAfter", "ok", "httpStatus", "markerMatched"])); return { ok: block.ok, action: block.action, desired: { monitorEnabled: desired.monitorEnabled, actionsEnabled: desired.actionsEnabled, schedule: desired.schedule, cronJobName: desired.cronJobName, configMapName: desired.configMapName, credentialsSecretName: desired.credentialsSecretName, stateConfigMapName: desired.stateConfigMapName, }, cronJob: { exists: cronJob.exists, schedule: cronJob.schedule, suspend: cronJob.suspend, lastScheduleTime: cronJob.lastScheduleTime, active: cronJob.active, error: cronJob.error, }, secret: { exists: secret.exists, profileSecretPresent: secret.profileSecretPresent, valuesPrinted: false, error: secret.error, }, configMap: { exists: configMap.exists, configPresent: configMap.configPresent, runnerPresent: configMap.runnerPresent, error: configMap.error, }, state: { exists: state.exists, accountCount: state.accountCount, quarantinedCount: state.quarantinedCount, quarantinedShown: Math.min(quarantined.length, 3), quarantined: quarantined.slice(-3), recentAccountCount: recentAccounts.length, recentAttention: uniqueByAccountName(recentAttention.slice(-3)), recentHealthy, lastRun: compactSentinelLastRun(state.lastRun), error: state.error, }, freezeReassert: Object.keys(freezeReassert).length > 0 ? { ok: freezeReassert.ok, skipped: freezeReassert.skipped, reason: freezeReassert.reason, itemCount: freezeReassert.itemCount, attentionItems: freezeReassert.attentionItems, } : undefined, pendingProbePrepare: Object.keys(qualityGatePrepare).length > 0 ? { ok: qualityGatePrepare.ok, skipped: qualityGatePrepare.skipped, reason: qualityGatePrepare.reason, changedCount: qualityGatePrepare.changedCount, fingerprintOnlyCount: qualityGatePrepare.fingerprintOnlyCount, pendingOnly: qualityGatePrepare.pendingOnly, items: qualityGatePrepare.items, } : undefined, pendingProbe: Object.keys(qualityGate).length > 0 ? { ok: qualityGate.ok, skipped: qualityGate.skipped, reason: qualityGate.reason, changedCount: qualityGate.changedCount, fingerprintOnlyCount: qualityGate.fingerprintOnlyCount, clampedCount: qualityGate.clampedCount, items: qualityGate.items, clampedItems: qualityGate.clampedItems, } : undefined, valuesPrinted: false, }; } function compactSentinelProbeResult(parsed: Record | null): Record | null { if (parsed === null) return null; const probe = isRecord(parsed.probe) ? parsed.probe : {}; const summary = isRecord(probe.summary) ? probe.summary : {}; const state = isRecord(parsed.sentinelState) ? parsed.sentinelState : {}; return { ok: parsed.ok, mode: parsed.mode, namespace: parsed.namespace, job: parsed.job, requestedAccounts: parsed.requestedAccounts, summary: { at: summary.at, monitorEnabled: summary.monitorEnabled, actionsEnabled: summary.actionsEnabled, selected: summary.selected, okCount: summary.okCount, mismatchCount: summary.mismatchCount, markerMismatchCount: summary.markerMismatchCount, transportFailureCount: summary.transportFailureCount, actionsTaken: summary.actionsTaken, gatewayFailureMonitor: summary.gatewayFailureMonitor, selection: summary.selection, }, results: recordArray(probe.results).map((item) => pickSummaryFields(item, [ "accountName", "purpose", "ok", "markerMatched", "sentinelProtect", "httpStatus", "durationMs", "usage", "outputHash", "outputPreview", "responseBodyHash", "responseBodyPreview", "error", "errorDetails", "failureKind", "sdk", "requestShape", ])), actions: recordArray(probe.actions).map((item) => pickSummaryFields(item, [ "accountName", "taken", "type", "error", ])), sentinelState: { quarantined: state.quarantined, recentAccounts: state.recentAccounts, lastRun: state.lastRun, }, valuesPrinted: false, }; } function renderedCliResult(ok: boolean, command: string, renderedText: string): RenderedCliResult { return { ok, command, renderedText, contentType: "text/plain" }; } function renderSentinelReport( parsed: Record | null, context: { events: number; full: boolean; remote: Record }, ): string { if (parsed === null) { return [ "SUB2API SENTINEL REPORT unavailable", `remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`, stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "", ].filter(Boolean).join("\n"); } const metadata = isRecord(parsed.metadata) ? parsed.metadata : {}; const cronJob = isRecord(parsed.cronJob) ? parsed.cronJob : {}; const summary = isRecord(parsed.summary) ? parsed.summary : {}; const accounts = recordArray(parsed.accounts); const runs = recordArray(parsed.runs); const globalLedger = isRecord(parsed.globalLedger) ? parsed.globalLedger : {}; const lines: string[] = []; lines.push([ "SUB2API SENTINEL", `ok=${parsed.ok === true ? "true" : "false"}`, `accounts=${summary.accountCount ?? accounts.length}`, `quarantined=${summary.quarantinedCount ?? "?"}`, `schedulable=${summary.runtimeSchedulableCount ?? "?"}`, `unschedulable=${summary.runtimeUnschedulableCount ?? "?"}`, `history=${summary.historyCount ?? runs.length}`, `window=${formatWindow(summary.historyFrom, summary.historyTo)}`, ].join(" ")); lines.push([ "CRON", `schedule=${cronJob.schedule ?? "-"}`, `last=${shortIso(cronJob.lastScheduleTime)}`, `active=${cronJob.active ?? "-"}`, `state=${metadata.namespace ?? "-"}/${metadata.stateConfigMapName ?? "-"}`, `ledger=req:${globalLedger.requestCount ?? 0} tok:${formatNumber(globalLedger.totalTokens)} cost:$${formatCost(globalLedger.estimatedCostUsd)}`, ].join(" ")); lines.push(""); lines.push("ACCOUNTS"); lines.push(renderTable([ ["ACCOUNT", "STATE", "Q", "SCH", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"], ...accounts.map((account) => [ stringValue(account.account) ?? "-", stringValue(account.status) ?? "-", account.quarantineActive === true ? "Y" : "-", account.runtimeSchedulable === true ? "Y" : account.runtimeSchedulable === false ? "N" : "-", account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-", account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-", account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-", textValue(account.freezeIntervalMin), textValue(account.successIntervalMin), textValue(account.successMaxIntervalMin), textValue(account.probeCount), shortIso(account.lastEventAt ?? account.lastProbeAt), textValue(account.lastHttp), account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-", shorten(stringValue(account.lastFailureKind) ?? "-", 20), shorten(stringValue(account.lastAction) ?? "-", 16), shortIso(account.nextProbeAfter), textValue(account.observedLastToNextMin), ]), ])); if (runs.length > 0 || context.full) { lines.push(""); lines.push(`RUNS last=${Math.min(context.events, runs.length)}`); lines.push(renderTable([ ["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "GF", "GACT", "REASSERT"], ...runs.slice(-context.events).map((run) => [ shortIso(run.at), textValue(run.selected), textValue(run.due), textValue(run.ok), textValue(run.mismatch), textValue(run.transportFailures), textValue(run.actionsTaken), textValue(run.gatewayFailures), textValue(run.gatewayActions), textValue(run.reasserts), ]), ])); } lines.push(""); lines.push("LEGEND Q=quarantined SCH=Sub2API runtime schedulable T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions"); lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw"); return lines.join("\n"); } function renderTraceReport( parsed: Record | null, context: { requestId: string; showLines: boolean; remote: Record }, ): string { if (parsed === null) { return [ `SUB2API TRACE ${context.requestId} unavailable`, `remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`, stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "", ].filter(Boolean).join("\n"); } const summary = isRecord(parsed.summary) ? parsed.summary : {}; const request = isRecord(parsed.request) ? parsed.request : {}; const final = isRecord(parsed.final) ? parsed.final : {}; const window = isRecord(parsed.window) ? parsed.window : {}; const events = recordArray(parsed.events); const failovers = recordArray(parsed.failovers); const selectFailures = recordArray(parsed.selectFailures); const upstreamErrors = recordArray(parsed.upstreamErrors); const tempUnschedulable = recordArray(parsed.tempUnschedulable); const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {}; const accountSnapshot = recordArray(parsed.accountSnapshot); const lines: string[] = []; lines.push([ "SUB2API TRACE", stringValue(parsed.requestId) ?? context.requestId, `ok=${parsed.ok === true ? "true" : "false"}`, `outcome=${stringValue(summary.outcome) ?? "-"}`, `reason=${stringValue(summary.reason) ?? "-"}`, ].join(" ")); lines.push([ "REQUEST", `path=${request.path ?? final.path ?? "-"}`, `model=${request.model ?? final.model ?? "-"}`, `stream=${textValue(request.stream)}`, `body=${textValue(request.bodyBytes)}`, `first=${shortIso(summary.firstAt)}`, `last=${shortIso(summary.lastAt)}`, ].join(" ")); lines.push([ "FINAL", `status=${textValue(final.statusCode)}`, `account=${formatAccountRef(final)}`, `latency_ms=${textValue(final.latencyMs)}`, `events=${textValue(summary.eventCount)}`, `window=${window.beforeSeconds ?? "?"}s/${window.afterSeconds ?? "?"}s`, ].join(" ")); if (failovers.length > 0) { lines.push(""); lines.push("FAILOVER"); lines.push(renderTable([ ["AT", "ACCOUNT", "UPSTREAM", "SWITCH", "MAX"], ...failovers.map((item) => [ shortIso(item.at), formatAccountRef(item), textValue(item.upstreamStatus), textValue(item.switchCount), textValue(item.maxSwitches), ]), ])); } if (selectFailures.length > 0) { lines.push(""); lines.push("SELECT-FAILED"); lines.push(renderTable([ ["AT", "ERROR", "EXCLUDED"], ...selectFailures.map((item) => [ shortIso(item.at), shorten(stringValue(item.error) ?? "-", 56), textValue(item.excludedAccountCount), ]), ])); } if (upstreamErrors.length > 0 || tempUnschedulable.length > 0) { lines.push(""); lines.push("ACCOUNT SIGNALS"); lines.push(renderTable([ ["TYPE", "PHASE", "AT", "ACCOUNT", "STATUS", "RULE", "UNTIL", "DETAIL"], ...upstreamErrors.map((item) => [ "upstream-error", textValue(item.phase), shortIso(item.at), formatAccountRef(item), textValue(item.statusCode), "-", "-", shorten(stringValue(item.error) ?? "-", 36), ]), ...tempUnschedulable.map((item) => [ "temp-unsched", textValue(item.phase), shortIso(item.at), formatAccountRef(item), textValue(item.statusCode), textValue(item.ruleIndex), shortIso(item.until), shorten(stringValue(item.reason) ?? stringValue(item.matchedKeyword) ?? "-", 36), ]), ])); } lines.push(""); lines.push("WINDOW STATS"); lines.push(renderTable([ ["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"], [ textValue(windowStats.matchedLines), textValue(windowStats.eventCount), textValue(windowStats.finalErrorCount), textValue(windowStats.failoverCount), textValue(windowStats.selectFailedCount), textValue(windowStats.tempUnschedulableCount), textValue(windowStats.adminSchedulableCount), ], ])); if (accountSnapshot.length > 0) { lines.push(""); lines.push("CURRENT ACCOUNTS"); lines.push(renderTable([ ["ID", "ACCOUNT", "SCHED", "STATUS", "CONC", "TEMP_UNTIL"], ...accountSnapshot.slice(0, 20).map((item) => [ textValue(item.accountId), shorten(stringValue(item.accountName) ?? "-", 32), textValue(item.schedulable), textValue(item.status), textValue(item.concurrency), shortIso(item.tempUnschedulableUntil), ]), ])); } if (context.showLines || parsed.showLines === true) { const rawLines = recordArray(parsed.rawLines); if (rawLines.length > 0) { lines.push(""); lines.push("RAW LINES"); for (const item of rawLines) { lines.push(shorten(stringValue(item.line) ?? "", 1000)); } } } lines.push(""); lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id --raw"); lines.push("Lines: add --show-lines for bounded matched log lines."); return lines.join("\n"); } function formatAccountRef(item: Record): string { const name = stringValue(item.accountName); const id = textValue(item.accountId); if (name !== null && id !== "-") return `${name}#${id}`; if (name !== null) return name; return id; } function renderTable(rows: string[][]): string { if (rows.length === 0) return ""; const widths: number[] = []; for (const row of rows) { row.forEach((cell, index) => { widths[index] = Math.max(widths[index] ?? 0, displayWidth(cell)); }); } return rows.map((row) => row.map((cell, index) => padRight(cell, widths[index] ?? 0)).join(" ").trimEnd()).join("\n"); } function padRight(value: string, width: number): string { const pad = width - displayWidth(value); return pad <= 0 ? value : `${value}${" ".repeat(pad)}`; } function displayWidth(value: string): number { return [...value].reduce((width, char) => width + (char.charCodeAt(0) > 0x7f ? 2 : 1), 0); } function formatWindow(from: unknown, to: unknown): string { const left = shortIso(from); const right = shortIso(to); return left === "-" && right === "-" ? "-" : `${left}..${right}`; } function shortIso(value: unknown): string { const text = stringValue(value); if (text === null) return "-"; return text.replace(/^\d{4}-/u, "").replace(/:00Z$/u, "Z").replace("T", " "); } function textValue(value: unknown): string { if (value === null || value === undefined || value === "") return "-"; if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Math.round(value * 10) / 10); if (typeof value === "boolean") return value ? "true" : "false"; return String(value); } function shorten(value: string, maxChars: number): string { return value.length <= maxChars ? value : `${value.slice(0, Math.max(0, maxChars - 1))}…`; } function formatNumber(value: unknown): string { const num = numberValue(value); if (num === null) return "0"; if (Math.abs(num) >= 1_000_000) return `${(num / 1_000_000).toFixed(1)}M`; if (Math.abs(num) >= 1_000) return `${(num / 1_000).toFixed(1)}K`; return String(Math.round(num)); } function formatCost(value: unknown): string { const num = numberValue(value); if (num === null) return "0.0000"; return num.toFixed(4); } function codexPoolValidationSummary(parsed: Record | null): Record | null { if (parsed === null) return null; const validation = isRecord(parsed.validation) ? parsed.validation : {}; const runtimeCapabilities = isRecord(parsed.runtimeCapabilities) ? parsed.runtimeCapabilities : {}; return { ok: parsed.ok, degraded: parsed.degraded, mode: parsed.mode, namespace: parsed.namespace, serviceDns: parsed.serviceDns, appPod: parsed.appPod, admin: parsed.admin, apiKey: parsed.apiKey, ownerBalance: parsed.ownerBalance, ownerConcurrency: parsed.ownerConcurrency, capacity: compactStatusBlock(parsed.capacity, ["accountName", "accountId", "expectedCapacity", "runtimeConcurrency", "priority", "status", "schedulable", "ok"]), loadFactor: compactStatusBlock(parsed.loadFactor, ["accountName", "accountId", "expectedLoadFactor", "runtimeLoadFactor", "priority", "status", "schedulable", "ok"]), webSocketsV2: compactStatusBlock(parsed.webSocketsV2, ["accountName", "accountId", "expectedMode", "runtimeMode", "runtimeEnabled", "status", "schedulable", "ok"]), tempUnschedulable: compactTempUnschedulableStatus(parsed.tempUnschedulable), manualAccounts: compactManualAccounts(parsed.manualAccounts), sentinel: compactSentinelStatus(parsed.sentinel), runtimeCapabilities: { ok: runtimeCapabilities.ok, runtimeImage: runtimeCapabilities.runtimeImage, successBodyReclassification: compactRuntimeCapability(runtimeCapabilities.successBodyReclassification), modelRouting400Failover: compactRuntimeCapability(runtimeCapabilities.modelRouting400Failover), valuesPrinted: false, }, validation: { gatewayModels: compactGatewayModels(validation.gatewayModels), gatewayResponses: compactGatewayResponses(validation.gatewayResponses), gatewayResponsesRecent: compactGatewayResponsesRecent(validation.gatewayResponsesRecent), gatewayCompactRecent: compactGatewayCompactRecent(validation.gatewayCompactRecent), }, disclosure: { full: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --full", raw: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw", }, valuesPrinted: false, }; } function codexPoolSyncSummary(parsed: Record | null): Record | null { const base = codexPoolValidationSummary(parsed); if (parsed === null || base === null) return base; const accounts = isRecord(parsed.accounts) ? parsed.accounts : {}; const compactAccounts = recordArray(accounts.items).map((item) => pickSummaryFields(item, [ "accountName", "accountId", "action", "status", "runtimeSchedulable", "priority", "capacity", "loadFactor", "tempUnschedulableConfigured", "tempUnschedulableRuleCount", "ok", ])); return { ...base, pool: parsed.pool, accounts: { desired: accounts.desired, created: accounts.created, updated: accounts.updated, pruned: accounts.pruned, pruneMode: accounts.pruneMode, itemCount: compactAccounts.length, attentionItems: compactAccounts.filter((item) => item.ok === false || item.runtimeSchedulable === false), prunedItems: accounts.prunedItems, processControl: accounts.processControl, valuesPrinted: false, }, disclosure: { full: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full", raw: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw", pruneRemoved: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --prune-removed", }, }; } function poolTarget(pool = readCodexPoolConfig(), target = codexPoolRuntimeTarget()): Record { return { id: target.id, route: target.route, namespace: target.namespace, service: serviceName, serviceDns: target.serviceDns, publicBaseUrl: target.publicBaseUrl, consumerBaseUrl: target.publicBaseUrl === null ? null : codexConsumerBaseUrl(pool, target), configPath: codexPoolConfigPath, groupName: pool.groupName, apiKeyName: pool.apiKeyName, apiKeySecret: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, publicExposure: targetPublicExposureSummary(target), sentinelImageBuild: { source: `${sub2apiConfigPath}.targets[${target.id}].codexPool.sentinelImageBuild`, baseImageCachePolicy: target.sentinelImageBuild.baseImageCachePolicy, noProxy: target.sentinelImageBuild.noProxy, }, minOwnerConcurrency: pool.minOwnerConcurrency, minOwnerConcurrencySource: pool.minOwnerConcurrencySource, accountCapacityTotal: desiredAccountCapacityTotal(pool), defaultAccountPriority: pool.defaultAccountPriority, defaultAccountCapacity: pool.defaultAccountCapacity, defaultAccountLoadFactor: pool.defaultAccountLoadFactor, defaultSentinelProtect: pool.defaultSentinelProtect, sentinel: { monitorEnabled: pool.sentinel.monitor.enabled, actionsEnabled: pool.sentinel.actions.enabled, cronJobName: pool.sentinel.cronJobName, stateConfigMapName: pool.sentinel.stateConfigMapName, }, egressProxy: target.egressProxy === null ? null : { enabled: target.egressProxy.enabled, applyToSentinel: target.egressProxy.applyToSentinel, serviceName: target.egressProxy.serviceName, listenPort: target.egressProxy.listenPort, httpProxy: target.egressProxy.httpProxy, noProxy: target.egressProxy.noProxy, }, valuesPrinted: false, }; } function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProfileSecret[] { return profiles .filter((profile) => profile.ok && profile.apiKey !== null && profile.apiKey.length > 0) .map((profile) => ({ accountName: profile.accountName, profile: profile.profile, baseUrl: profile.baseUrl, apiKey: profile.apiKey ?? "", upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, sentinelProtect: profile.sentinelProtect, })); } function resolvedManualAccountProtections(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record[] { return pool.manualAccounts.protected.map((account) => ({ accountName: account.accountName, reason: account.reason, proxyBinding: resolveManualProxyBinding(account.proxyBinding, pool, target), groupBinding: resolveManualGroupBinding(account.groupBinding, pool), })); } function resolveManualProxyBinding(binding: CodexPoolManualAccountProxyBinding | null, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record | null { if (binding === null) return null; const source = manualBindingSource(pool, binding.source); if (source.kind !== "proxy") throw new Error(`${codexPoolConfigPath}.manualAccounts binding source ${source.id} must have kind=proxy`); if (binding.enabled && source.provider === "target-egress-proxy" && (target.egressProxy === null || target.egressProxy.enabled !== true)) { throw new Error(`${codexPoolConfigPath}.manualAccounts.bindingSources.${source.id} requires ${sub2apiConfigPath}.targets[${target.id}].egressProxy.enabled=true`); } return { enabled: binding.enabled, source: source.id, proxyName: binding.proxyName, sourcePlan: { ...manualBindingSourcePlan(source), targetEgressProxy: source.provider === "target-egress-proxy" && target.egressProxy !== null ? { targetId: target.id, namespace: target.namespace, serviceName: target.egressProxy.serviceName, listenPort: target.egressProxy.listenPort, httpProxy: target.egressProxy.httpProxy, noProxy: target.egressProxy.noProxy, } : null, }, }; } function resolveManualGroupBinding(binding: CodexPoolManualAccountGroupBinding | null, pool: CodexPoolConfig): Record | null { if (binding === null) return null; const source = manualBindingSource(pool, binding.source); if (source.kind !== "group") throw new Error(`${codexPoolConfigPath}.manualAccounts binding source ${source.id} must have kind=group`); return { enabled: binding.enabled, source: source.id, sourcePlan: { ...manualBindingSourcePlan(source), poolGroupName: source.provider === "pool-group" ? pool.groupName : null, }, }; } function manualBindingSource(pool: CodexPoolConfig, sourceId: string): CodexPoolManualBindingSource { const source = pool.manualAccounts.bindingSources.byId[sourceId]; if (source === undefined) throw new Error(`${codexPoolConfigPath}.manualAccounts binding source ${sourceId} is not declared`); return source; } function manualBindingSourcePlan(source: CodexPoolManualBindingSource): Record { return { id: source.id, enabled: source.enabled, kind: source.kind, provider: source.provider, description: source.description, }; } function targetPublicExposureSummary(target: CodexPoolRuntimeTarget): Record | null { const exposure = target.publicExposure; if (exposure === null) return null; return { source: `${sub2apiConfigPath}.targets[${target.id}].publicExposure`, enabled: exposure.enabled, target: { id: target.id, route: target.route, namespace: target.namespace, serviceDns: target.serviceDns, }, urls: { publicBaseUrl: exposure.publicBaseUrl, consumerBaseUrl: target.publicBaseUrl === null ? null : `${target.publicBaseUrl.replace(/\/+$/u, "")}/`, }, dns: exposure.dns, frpc: { deploymentName: exposure.frpc.deploymentName, secretName: exposure.frpc.secretName, secretKey: exposure.frpc.secretKey, image: exposure.frpc.image, serverAddr: exposure.frpc.serverAddr, serverPort: exposure.frpc.serverPort, proxyName: exposure.frpc.proxyName, remotePort: exposure.frpc.remotePort, localIP: exposure.frpc.localIP, localPort: exposure.frpc.localPort, tokenSourceRef: exposure.frpc.tokenSourceRef, tokenSourceKey: exposure.frpc.tokenSourceKey, }, pk01: exposure.pk01, valuesPrinted: false, }; } function prepareTargetPublicExposureSecret(target: CodexPoolRuntimeTarget): ReturnType { if (target.publicExposure === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure is required`); return prepareFrpcSecret({ secretRoot: target.secretsRoot, exposure: target.publicExposure, sourcePathRedactor: redactRepoPath, parseEnvFile, requiredEnvValue, readTextFile: (sourcePath) => { if (!existsSync(sourcePath)) throw new Error(`publicExposure requires ${redactRepoPath(sourcePath)} with ${target.publicExposure?.frpc.tokenSourceKey}; materialize the PK01 frps token source first`); return readTextFile(sourcePath); }, }); } function secretMaterialSummary(secret: ReturnType): Record { return { sourceRef: secret.sourceRef, sourcePath: secret.sourcePath, secretName: secret.secretName, secretKey: secret.secretKey, fingerprint: secret.fingerprint, valuesPrinted: false, }; } function targetPublicExposureApplyScript(target: CodexPoolRuntimeTarget, secret: ReturnType): string { if (target.publicExposure === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure is required`); const manifest = renderFrpcManifest({ id: target.id, route: target.route, namespace: target.namespace, replicas: 1, publicExposure: target.publicExposure, } satisfies PublicServiceTarget); const manifestB64 = Buffer.from(manifest, "utf8").toString("base64"); const frpcTomlB64 = Buffer.from(secret.frpcToml, "utf8").toString("base64"); return ` set -u tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT manifest="$tmp/sub2api-public-exposure.yaml" frpc_toml="$tmp/frpc.toml" printf '%s' '${manifestB64}' | base64 -d > "$manifest" printf '%s' '${frpcTomlB64}' | base64 -d > "$frpc_toml" ns_out="$tmp/ns.out" ns_err="$tmp/ns.err" secret_out="$tmp/secret.out" secret_err="$tmp/secret.err" apply_out="$tmp/apply.out" apply_err="$tmp/apply.err" rollout_out="$tmp/rollout.out" rollout_err="$tmp/rollout.err" pods_json="$tmp/pods.json" pods_err="$tmp/pods.err" logs_out="$tmp/logs.out" logs_err="$tmp/logs.err" kubectl create namespace ${target.namespace} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$ns_out" 2>"$ns_err" ns_rc=$? secret_rc=1 apply_rc=1 rollout_rc=1 if [ "$ns_rc" -eq 0 ]; then kubectl -n ${target.namespace} create secret generic ${secret.secretName} \\ --from-file=${secret.secretKey}="$frpc_toml" \\ --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$secret_out" 2>"$secret_err" secret_rc=$? else : >"$secret_out" printf '%s\\n' 'skipped because namespace apply failed' >"$secret_err" fi if [ "$ns_rc" -eq 0 ] && [ "$secret_rc" -eq 0 ]; then kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$apply_out" 2>"$apply_err" apply_rc=$? if [ "$apply_rc" -eq 0 ]; then kubectl -n ${target.namespace} rollout status deployment/${target.publicExposure.frpc.deploymentName} --timeout=60s >"$rollout_out" 2>"$rollout_err" rollout_rc=$? else printf '%s\\n' 'skipped because apply failed' >"$rollout_err" fi else : >"$apply_out" printf '%s\\n' 'skipped because namespace or secret step failed' >"$apply_err" : >"$rollout_out" printf '%s\\n' 'skipped because namespace or secret step failed' >"$rollout_err" fi kubectl -n ${target.namespace} get pods -l app.kubernetes.io/name=${target.publicExposure.frpc.deploymentName} -o json >"$pods_json" 2>"$pods_err" pods_rc=$? kubectl -n ${target.namespace} logs deployment/${target.publicExposure.frpc.deploymentName} --tail=80 >"$logs_out" 2>"$logs_err" logs_rc=$? python3 - "$tmp" "$ns_rc" "$secret_rc" "$apply_rc" "$rollout_rc" "$pods_rc" "$logs_rc" <<'PY' import json import os import sys tmp = sys.argv[1] ns_rc, secret_rc, apply_rc, rollout_rc, pods_rc, logs_rc = [int(value) for value in sys.argv[2:]] def text(name, limit=4000): path = os.path.join(tmp, name) try: return open(path, encoding="utf-8", errors="replace").read()[-limit:] except FileNotFoundError: return "" def load_json(name): path = os.path.join(tmp, name) try: return json.load(open(path, encoding="utf-8")) except Exception: return None pods_obj = load_json("pods.json") pods = [] if isinstance(pods_obj, dict): for item in pods_obj.get("items") or []: status = item.get("status") or {} statuses = status.get("containerStatuses") or [] pods.append({ "name": item.get("metadata", {}).get("name"), "phase": status.get("phase"), "ready": all(cs.get("ready") is True for cs in statuses) if statuses else False, "restarts": sum(int(cs.get("restartCount") or 0) for cs in statuses), }) logs = text("logs.out", 4000) payload = { "ok": ns_rc == 0 and secret_rc == 0 and apply_rc == 0 and rollout_rc == 0 and pods_rc == 0 and len(pods) > 0 and all(p["ready"] for p in pods), "target": "${target.id}", "namespace": "${target.namespace}", "source": "${sub2apiConfigPath}.targets[${target.id}].publicExposure", "proxy": { "name": "${target.publicExposure.frpc.proxyName}", "remotePort": ${JSON.stringify(target.publicExposure.frpc.remotePort)}, "publicBaseUrl": "${target.publicExposure.publicBaseUrl}", "localIP": "${target.publicExposure.frpc.localIP}", "localPort": ${JSON.stringify(target.publicExposure.frpc.localPort)}, }, "resources": { "secret": "${secret.secretName}", "secretKey": "${secret.secretKey}", "deployment": "${target.publicExposure.frpc.deploymentName}", "image": "${target.publicExposure.frpc.image}", }, "steps": { "namespace": {"exitCode": ns_rc, "stdout": text("ns.out"), "stderr": text("ns.err")}, "secret": {"exitCode": secret_rc, "stdout": text("secret.out"), "stderr": text("secret.err"), "valuesPrinted": False}, "apply": {"exitCode": apply_rc, "stdout": text("apply.out"), "stderr": text("apply.err")}, "rollout": {"exitCode": rollout_rc, "stdout": text("rollout.out"), "stderr": text("rollout.err")}, "pods": {"exitCode": pods_rc, "items": pods, "stderr": text("pods.err")}, "logs": { "exitCode": logs_rc, "startProxySuccess": "start proxy success" in logs, "stdoutTail": logs, "stderrTail": text("logs.err"), }, }, "valuesPrinted": False, } print(json.dumps(payload, ensure_ascii=False, indent=2)) sys.exit(0 if payload["ok"] else 1) PY `; } async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): Promise<{ apiKey: string | null; error: string | null }> { const result = await capture(config, target.route, ["sh"], ` set -u kubectl -n ${target.namespace} get secret ${pool.apiKeySecretName} -o json `); if (result.exitCode !== 0) { return { apiKey: null, error: `read pool API key secret failed: ${result.stderr.slice(-1000)}` }; } const parsed = parseJsonOutput(result.stdout); const data = isRecord(parsed?.data) ? parsed.data : null; const encoded = typeof data?.[pool.apiKeySecretKey] === "string" ? data[pool.apiKeySecretKey] : null; if (encoded === null) return { apiKey: null, error: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey} missing` }; try { const apiKey = Buffer.from(encoded, "base64").toString("utf8"); return apiKey.length > 0 ? { apiKey, error: null } : { apiKey: null, error: "decoded API key is empty" }; } catch (error) { return { apiKey: null, error: error instanceof Error ? error.message : String(error) }; } } function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget()): Record { const codexDir = join(homedir(), ".codex"); mkdirSync(codexDir, { recursive: true, mode: 0o700 }); const configPath = join(codexDir, "config.toml"); const authPath = join(codexDir, "auth.json"); const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`); const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`); if (!existsSync(configPath)) throw new Error(`${configPath} missing`); if (!existsSync(authPath)) throw new Error(`${authPath} missing`); const configBackupAction = copyIfMissing(configPath, backupConfigPath); const authBackupAction = copyIfMissing(authPath, backupAuthPath); const currentToml = readFileSync(configPath, "utf8"); const nextToml = updateCodexConfigToml(currentToml, pool, target); writeFileSync(configPath, nextToml, { encoding: "utf8", mode: 0o600 }); chmodSync(configPath, 0o600); writeFileSync(authPath, `${JSON.stringify({ OPENAI_API_KEY: apiKey }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); chmodSync(authPath, 0o600); return { ok: true, codexDir, config: { path: configPath, bytes: statSync(configPath).size, backupPath: backupConfigPath, backupAction: configBackupAction, backupBytes: statSync(backupConfigPath).size, }, auth: { path: authPath, bytes: statSync(authPath).size, backupPath: backupAuthPath, backupAction: authBackupAction, backupBytes: statSync(backupAuthPath).size, openaiApiKeyShape: "string", openaiApiKeyBytes: Buffer.byteLength(apiKey, "utf8"), openaiApiKeyFingerprint: fingerprint(apiKey), valuesPrinted: false, }, provider: { name: pool.localCodex.providerName, baseUrl: codexConsumerBaseUrl(pool, target), wireApi: pool.localCodex.wireApi, modelContextWindow: pool.localCodex.modelContextWindow, modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit, supportsWebSockets: pool.localCodex.supportsWebSockets, responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2, }, valuesPrinted: false, }; } function copyIfMissing(source: string, target: string): "created" | "kept-existing" { if (existsSync(target)) return "kept-existing"; copyFileSync(source, target); chmodSync(target, 0o600); return "created"; } function updateCodexConfigToml(current: string, pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): string { return renderCodexLocalConsumerToml(current, { providerName: pool.localCodex.providerName, baseUrl: codexConsumerBaseUrl(pool, target), wireApi: pool.localCodex.wireApi, modelContextWindow: pool.localCodex.modelContextWindow, modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit, supportsWebSockets: pool.localCodex.supportsWebSockets, responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2, }); } function codexConsumerBaseUrl(pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): string { void pool; const baseUrl = target.publicBaseUrl; if (baseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure.enabled must be true before resolving the Codex consumer base URL`); return `${baseUrl.replace(/\/+$/u, "")}/`; } export function renderCodexLocalConsumerToml(current: string, options: CodexLocalConsumerTomlOptions): string { let next = upsertTopLevelTomlString(current, "model_provider", options.providerName); next = upsertTopLevelTomlInteger(next, "model_context_window", options.modelContextWindow); next = upsertTopLevelTomlInteger(next, "model_auto_compact_token_limit", options.modelAutoCompactTokenLimit); next = upsertProviderSection(next, options); next = upsertTomlSectionBoolean(next, "features", "responses_websockets_v2", options.responsesWebSocketsV2); return next.endsWith("\n") ? next : `${next}\n`; } function upsertTopLevelTomlString(text: string, key: string, value: string): string { const lines = text.split(/\r?\n/u); const firstSection = lines.findIndex((line) => /^\s*\[/.test(line)); const end = firstSection === -1 ? lines.length : firstSection; for (let index = 0; index < end; index += 1) { if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) { lines[index] = `${key} = ${tomlString(value)}`; return lines.join("\n"); } } lines.splice(0, 0, `${key} = ${tomlString(value)}`); return lines.join("\n"); } function upsertTopLevelTomlInteger(text: string, key: string, value: number): string { const lines = text.split(/\r?\n/u); const firstSection = lines.findIndex((line) => /^\s*\[/.test(line)); const end = firstSection === -1 ? lines.length : firstSection; const assignment = `${key} = ${value}`; for (let index = 0; index < end; index += 1) { if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) { lines[index] = assignment; return lines.join("\n"); } } lines.splice(end, 0, assignment); return lines.join("\n"); } function upsertProviderSection(text: string, options: CodexLocalConsumerTomlOptions): string { const { providerName, baseUrl, wireApi, supportsWebSockets } = options; const header = `[model_providers.${providerName}]`; const lines = text.split(/\r?\n/u); const start = lines.findIndex((line) => line.trim() === header); const canonical = [ `name = ${tomlString(providerName)}`, `base_url = ${tomlString(baseUrl)}`, `wire_api = ${tomlString(wireApi)}`, "requires_openai_auth = true", `supports_websockets = ${tomlBoolean(supportsWebSockets)}`, ]; if (start === -1) { const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== ""; return [...lines, ...(needsBlank ? [""] : []), header, ...canonical].join("\n"); } let end = lines.length; for (let index = start + 1; index < lines.length; index += 1) { if (/^\s*\[/.test(lines[index])) { end = index; break; } } const preserved = lines.slice(start + 1, end).filter((line) => { return !/^\s*(name|base_url|wire_api|requires_openai_auth|supports_websockets|env_key)\s*=/u.test(line); }); const replacement = [header, ...canonical, ...preserved.filter((line) => line.trim() !== "")]; return [...lines.slice(0, start), ...replacement, ...lines.slice(end)].join("\n"); } function upsertTomlSectionBoolean(text: string, sectionName: string, key: string, value: boolean): string { const header = `[${sectionName}]`; const lines = text.split(/\r?\n/u); const start = lines.findIndex((line) => line.trim() === header); const assignment = `${key} = ${tomlBoolean(value)}`; if (start === -1) { const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== ""; return [...lines, ...(needsBlank ? [""] : []), header, assignment].join("\n"); } let end = lines.length; for (let index = start + 1; index < lines.length; index += 1) { if (/^\s*\[/.test(lines[index])) { end = index; break; } } for (let index = start + 1; index < end; index += 1) { if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) { lines[index] = assignment; return lines.join("\n"); } } return [...lines.slice(0, start + 1), assignment, ...lines.slice(start + 1)].join("\n"); } async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget()): Promise> { const probe = await probePublicModels(pool, "with-api-key", apiKey, target); return { ...probe, ok: probe.ok === true, apiKeyFingerprint: fingerprint(apiKey), keyPreview: apiKeyPreview(apiKey), valuesPrinted: false, }; } async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, target = codexPoolRuntimeTarget()): Promise> { void pool; if (target.publicBaseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure.enabled must be true before probing the public gateway`); const baseUrl = target.publicBaseUrl; const url = `${baseUrl.replace(/\/+$/u, "")}/v1/models`; const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 8000); try { const response = await fetch(url, { method: "GET", headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined, signal: controller.signal, }); const body = await response.text(); let parsed: unknown = null; try { parsed = body.trim().length > 0 ? JSON.parse(body) : null; } catch { parsed = null; } const modelCount = isRecord(parsed) && Array.isArray(parsed.data) ? parsed.data.length : null; return { ok: response.ok, mode, method: "GET /v1/models", baseUrl, httpStatus: response.status, modelCount, bodyPreview: response.ok ? "" : body.slice(0, 500), valuesPrinted: false, }; } catch (error) { return { ok: false, mode, method: "GET /v1/models", baseUrl, httpStatus: 0, error: error instanceof Error ? error.message : String(error), valuesPrinted: false, }; } finally { clearTimeout(timeout); } } function apiKeyPreview(apiKey: string): string { if (apiKey.length <= 14) return "***"; return `${apiKey.slice(0, 10)}...${apiKey.slice(-4)}`; } function tomlString(value: string): string { return JSON.stringify(value); } function tomlBoolean(value: boolean): string { return value ? "true" : "false"; } function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); } function normalizeBaseUrl(value: string | null): string | null { if (value === null) return null; const trimmed = value.trim().replace(/\/+$/u, ""); if (trimmed.length === 0) return null; try { const parsed = new URL(trimmed); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; return parsed.toString().replace(/\/+$/u, ""); } catch { return null; } } function stringValue(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function requiredStringConfigField(obj: Record, key: string, path: string): string { const value = stringValue(obj[key]); const prefix = path.length > 0 ? `${path}.` : ""; if (value === null) throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be a non-empty string`); return value; } function readRequiredDescription(value: unknown, key: string, maxChars: number): string { const text = stringValue(value); if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a non-empty string`); if (text.length > maxChars) throw new Error(`${codexPoolConfigPath}.${key} must be at most ${maxChars} characters`); if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`); return text; } function readRequiredEmail(value: unknown, key: string): string { const text = stringValue(value); if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a non-empty string`); if (!/^[^@\s]+@[^@\s]+[.][^@\s]+$/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must be an email address`); return text; } function readRequiredBaseUrl(value: unknown, key: string): string { const baseUrl = normalizeBaseUrl(stringValue(value)); if (baseUrl === null) throw new Error(`${codexPoolConfigPath}.${key} must be a valid http(s) URL`); return baseUrl; } function readRequiredPort(value: unknown, key: string): number { const port = numberValue(value); if (port === null || !Number.isInteger(port) || port < 1 || port > 65535) { throw new Error(`${codexPoolConfigPath}.${key} must be an integer port`); } return port; } function readRequiredPositiveNumber(value: unknown, key: string): number { const number = numberValue(value); if (number === null || number <= 0) throw new Error(`${codexPoolConfigPath}.${key} must be > 0`); return number; } function requiredRecordConfigField(obj: Record, key: string, path: string): Record { const value = obj[key]; const prefix = path.length > 0 ? `${path}.` : ""; if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be a YAML object`); return value; } function integerConfigField(obj: Record, key: string, path: string): number { const value = obj[key]; const prefix = path.length > 0 ? `${path}.` : ""; if (typeof value !== "number" || !Number.isInteger(value)) throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be an integer`); return value; } function integerArrayConfigField(obj: Record, key: string, path: string): number[] { const value = obj[key]; const prefix = path.length > 0 ? `${path}.` : ""; if (!Array.isArray(value) || !value.every((item) => typeof item === "number" && Number.isInteger(item))) { throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be an array of integers`); } return value; } function numberValue(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim().length > 0) { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } return null; } function booleanValue(value: unknown): boolean | null { if (typeof value === "boolean") return value; if (typeof value === "string") { if (value.trim() === "true") return true; if (value.trim() === "false") return false; } return null; } function validateKubernetesName(value: string, key: string, strictDns: boolean): void { const pattern = strictDns ? /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u : /^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u; if (!pattern.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function fingerprint(value: string): string { return shortSha256Fingerprint(value); } export function codexPoolSentinelProbeConfigFingerprint(input: { accountName: string; profile: string; baseUrl: string; apiKeyFingerprint: string | null; upstreamUserAgent: string | null; openaiResponsesWebSocketsV2Mode: string | null; trustUpstream: boolean; sentinelProtect: CodexSentinelProtectPolicy; }): string { return fingerprint(JSON.stringify({ accountName: input.accountName, profile: input.profile, baseUrl: normalizeBaseUrl(input.baseUrl) ?? input.baseUrl, apiKeyFingerprint: input.apiKeyFingerprint, upstreamUserAgent: input.upstreamUserAgent, openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode, trustUpstream: input.trustUpstream, sentinelProtect: input.sentinelProtect, })); } function syncScript(payload: unknown, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); return remotePythonScript("sync", encoded, pool, target); } function validateScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { return remotePythonScript("validate", "", pool, target); } function traceScript(pool: CodexPoolConfig, options: TraceOptions, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify({ requestId: options.requestId, since: options.since, tail: options.tail, contextSeconds: options.contextSeconds, showLines: options.showLines, }), "utf8").toString("base64"); return remotePythonScript("trace", encoded, pool, target); } function sentinelProbeScript(payload: unknown, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); return remotePythonScript("sentinel-probe", encoded, pool, target); } function sentinelReportScript(pool: CodexPoolConfig, events: number, target: CodexPoolRuntimeTarget): string { const stateName = pool.sentinel.stateConfigMapName; const cronJobName = pool.sentinel.cronJobName; return ` set -eu python3 - <<'PY' import json import subprocess from datetime import datetime, timezone, timedelta NAMESPACE = ${JSON.stringify(target.namespace)} STATE_NAME = ${JSON.stringify(stateName)} CRONJOB_NAME = ${JSON.stringify(cronJobName)} EVENT_LIMIT = ${JSON.stringify(events)} def run(cmd): return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def text(data, limit=2000): if isinstance(data, bytes): data = data.decode("utf-8", errors="replace") return data[-limit:] def kube_json(args): proc = run(["kubectl", *args, "-o", "json"]) if proc.returncode != 0: return None, text(proc.stderr) try: return json.loads(proc.stdout.decode("utf-8")), None except Exception as exc: return None, str(exc) def parse_iso(value): if not isinstance(value, str) or not value: return None try: return datetime.fromisoformat(value.replace("Z", "+00:00")) except Exception: return None def minutes_between(a, b): left = parse_iso(a) right = parse_iso(b) if left is None or right is None: return None return round((right - left).total_seconds() / 60, 1) def day_ledgers(state): ledger = {} raw = state.get("ledger") if isinstance(raw, dict): for item in raw.values(): if not isinstance(item, dict): continue add_ledger(ledger, item) return ledger def add_ledger(target, source): target["inputTokens"] = target.get("inputTokens", 0) + int(source.get("inputTokens") or 0) target["outputTokens"] = target.get("outputTokens", 0) + int(source.get("outputTokens") or 0) target["totalTokens"] = target.get("totalTokens", 0) + int(source.get("totalTokens") or 0) target["estimatedCostUsd"] = target.get("estimatedCostUsd", 0.0) + float(source.get("estimatedCostUsd") or 0) target["requestCount"] = target.get("requestCount", 0) + int(source.get("requestCount") or 0) def account_ledger(account_state): total = {} raw = account_state.get("daily") if isinstance(raw, dict): for item in raw.values(): if isinstance(item, dict): add_ledger(total, item) return total def action_type(probe): action = probe.get("action") if isinstance(probe, dict) else None if isinstance(action, dict): value = action.get("type") if value: return value return "taken" if action.get("taken") is True else "" return "" def error_code(probe): details = probe.get("errorDetails") if isinstance(probe, dict) else None if not isinstance(details, dict): return "" openai_error = details.get("openaiError") if isinstance(openai_error, dict): return openai_error.get("code") or openai_error.get("type") or "" return details.get("kind") or "" def sentinel_protect_summary(account_state, probe): probe_protect = probe.get("sentinelProtect") if isinstance(probe.get("sentinelProtect"), dict) else {} config_protect = account_state.get("sentinelProtectConfig") if isinstance(account_state.get("sentinelProtectConfig"), dict) else {} source = probe_protect if probe_protect else config_protect if not isinstance(source, dict) or source.get("enabled") is not True: return { "enabled": False, "threshold": None, "decision": None, "failureCount": None, } return { "enabled": True, "threshold": source.get("threshold") or source.get("consecutiveFailures"), "decision": probe_protect.get("decision"), "failureCount": probe_protect.get("failureCount"), } def report(): cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME]) state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME]) state = {} parse_error = None if isinstance(state_cm, dict): raw_state = (state_cm.get("data") or {}).get("state.json") if isinstance(raw_state, str) and raw_state.strip(): try: state = json.loads(raw_state) except Exception as exc: parse_error = str(exc) accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} history = state.get("history") if isinstance(state.get("history"), list) else [] account_rows = [] for name, account_state in sorted(accounts.items()): if not isinstance(account_state, dict): continue probe = account_state.get("lastProbe") if isinstance(account_state.get("lastProbe"), dict) else {} quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else {} gateway_failure = account_state.get("lastGatewayFailure") if isinstance(account_state.get("lastGatewayFailure"), dict) else {} last_event_at = account_state.get("lastGatewayFailureAt") or account_state.get("lastProbeAt") last_failure_kind = quarantine.get("failureKind") if quarantine.get("active") is True and quarantine.get("failureKind") else probe.get("failureKind") last_action = action_type(probe) if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")): last_action = action_type(gateway_failure) ledger = account_ledger(account_state) protect = sentinel_protect_summary(account_state, probe) account_rows.append({ "account": name, "status": account_state.get("lastStatus"), "quarantineActive": quarantine.get("active") is True, "quarantineApplied": quarantine.get("applied") if isinstance(quarantine, dict) else None, "freezeIntervalMin": quarantine.get("intervalMinutes") if isinstance(quarantine, dict) else None, "freezeUntil": quarantine.get("until") if isinstance(quarantine, dict) else None, "runtimeAccountId": account_state.get("runtimeAccountId"), "runtimeStatus": account_state.get("runtimeStatus"), "runtimeSchedulable": account_state.get("runtimeSchedulable"), "runtimeMissing": account_state.get("runtimeMissing"), "runtimeTempUnschedulableSet": account_state.get("runtimeTempUnschedulableSet"), "runtimeTempUnschedulableUntil": account_state.get("runtimeTempUnschedulableUntil"), "runtimeSyncedAt": account_state.get("runtimeSyncedAt"), "runtimeSyncError": account_state.get("runtimeSyncError"), "trustUpstream": account_state.get("trustUpstream") if account_state.get("trustUpstream") is not None else probe.get("trustUpstream"), "successStreak": account_state.get("successStreak") or 0, "successIntervalMin": account_state.get("successIntervalMinutes") or 0, "successMaxIntervalMin": account_state.get("successMaxIntervalMinutes") or probe.get("successMaxIntervalMinutes"), "probeCount": ledger.get("requestCount", 0), "inputTokens": ledger.get("inputTokens", 0), "outputTokens": ledger.get("outputTokens", 0), "totalTokens": ledger.get("totalTokens", 0), "estimatedCostUsd": round(float(ledger.get("estimatedCostUsd", 0)), 6), "lastProbeAt": account_state.get("lastProbeAt"), "lastEventAt": last_event_at, "lastGatewayFailureAt": account_state.get("lastGatewayFailureAt"), "lastPurpose": probe.get("purpose"), "lastHttp": probe.get("httpStatus"), "lastMarker": probe.get("markerMatched"), "sentinelProtectEnabled": protect.get("enabled"), "sentinelProtectDecision": protect.get("decision"), "sentinelProtectFailureCount": protect.get("failureCount"), "sentinelProtectThreshold": protect.get("threshold"), "lastFailureKind": last_failure_kind, "lastErrorCode": error_code(probe), "lastAction": last_action, "nextProbeAfter": account_state.get("nextProbeAfter"), "observedLastToNextMin": minutes_between(last_event_at, account_state.get("nextProbeAfter")), "requestShape": probe.get("requestShape"), }) run_rows = [] for item in history: if not isinstance(item, dict): continue selection = item.get("selection") if isinstance(item.get("selection"), dict) else {} gateway = item.get("gatewayFailureMonitor") if isinstance(item.get("gatewayFailureMonitor"), dict) else {} run_rows.append({ "at": item.get("at"), "selected": item.get("selected"), "due": selection.get("due"), "ok": item.get("okCount"), "mismatch": item.get("mismatchCount") if item.get("mismatchCount") is not None else item.get("markerMismatchCount"), "transportFailures": item.get("transportFailureCount"), "actionsTaken": item.get("actionsTaken"), "gatewayFailures": gateway.get("newFailures"), "gatewayActions": gateway.get("actionsTaken"), "reasserts": len(item.get("reconcile") or []), }) quarantined = [item for item in account_rows if item.get("quarantineActive") is True] runtime_schedulable = [item for item in account_rows if item.get("runtimeSchedulable") is True] runtime_unschedulable = [item for item in account_rows if item.get("runtimeSchedulable") is False] cron_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {} cron_status = cronjob.get("status") if isinstance(cronjob, dict) else {} global_ledger = day_ledgers(state) result = { "ok": state_error is None and parse_error is None, "metadata": { "namespace": NAMESPACE, "stateConfigMapName": STATE_NAME, "cronJobName": CRONJOB_NAME, "generatedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "valuesPrinted": False, }, "cronJob": { "exists": isinstance(cronjob, dict), "schedule": cron_spec.get("schedule") if isinstance(cron_spec, dict) else None, "suspend": cron_spec.get("suspend") if isinstance(cron_spec, dict) else None, "lastScheduleTime": cron_status.get("lastScheduleTime") if isinstance(cron_status, dict) else None, "active": len(cron_status.get("active") or []) if isinstance(cron_status, dict) else None, "error": cron_error, }, "summary": { "accountCount": len(account_rows), "quarantinedCount": len(quarantined), "runtimeSchedulableCount": len(runtime_schedulable), "runtimeUnschedulableCount": len(runtime_unschedulable), "historyCount": len(history), "historyFrom": run_rows[0].get("at") if run_rows else None, "historyTo": run_rows[-1].get("at") if run_rows else None, "lastRun": state.get("lastRun"), "runtimeSchedulable": state.get("runtimeSchedulable"), }, "globalLedger": global_ledger, "accounts": account_rows, "runs": run_rows[-EVENT_LIMIT:], "errors": { "state": state_error, "parse": parse_error, }, "valuesPrinted": False, } return result payload = report() print(json.dumps(payload, ensure_ascii=False, indent=2)) raise SystemExit(0 if payload.get("ok") else 1) PY `; } function cleanupProbesScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { return remotePythonScript("cleanup-probes", "", pool, target); } function sentinelImageStatusScript(pool: CodexPoolConfig, targetRuntime: CodexPoolRuntimeTarget): string { const target = codexPoolSentinelRuntimeImage(pool.sentinel); return remoteSentinelImageScript("status", target, pool.sentinel, null, targetRuntime); } function sentinelImageBuildScript(pool: CodexPoolConfig, targetRuntime: CodexPoolRuntimeTarget): string { const target = codexPoolSentinelRuntimeImage(pool.sentinel); const dockerfile = readFileSync(sentinelImageDockerfilePath, "utf8"); return remoteSentinelImageScript("build", target, pool.sentinel, dockerfile, targetRuntime); } function remoteSentinelImageScript(mode: "status" | "build", target: ReturnType, sentinel: CodexPoolSentinelConfig, dockerfile: string | null, targetRuntime: CodexPoolRuntimeTarget): string { const dockerfileB64 = dockerfile === null ? "" : Buffer.from(dockerfile, "utf8").toString("base64"); return ` set -eu mode=${shQuote(mode)} image=${shQuote(target.runtimeImage)} repo=${shQuote("platform-infra/sub2api-account-sentinel")} tag=${shQuote(target.tag)} base_image=${shQuote(target.baseImage)} openai_version=${shQuote(sentinel.sdk.openaiPythonVersion)} base_image_cache_policy=${shQuote(targetRuntime.sentinelImageBuild.baseImageCachePolicy)} work=/tmp/unidesk-sub2api-sentinel-image mkdir -p "$work" dockerfile_path="$work/sentinel.Dockerfile" registry_has_tag=false if curl -fsS --max-time 10 "http://127.0.0.1:5000/v2/$repo/tags/list" 2>/dev/null | grep -F '"'"$tag"'"' >/dev/null 2>&1; then registry_has_tag=true fi local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)" if [ "$mode" = "status" ]; then if [ -n "$local_id" ]; then python_version="$(docker run --rm "$image" python3 --version 2>/dev/null || true)" openai_runtime_version="$(docker run --rm "$image" python3 -c 'import importlib.metadata; print(importlib.metadata.version("openai"))' 2>/dev/null || true)" else python_version= openai_runtime_version= fi python3 - </dev/null 2>&1 || true local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)" fi python3 - < "$dockerfile_path" <<'UNIDESK_SENTINEL_DOCKERFILE_B64' ${dockerfileB64} UNIDESK_SENTINEL_DOCKERFILE_B64 export NO_PROXY=${shQuote(targetRuntime.sentinelImageBuild.noProxy)} export no_proxy=$NO_PROXY set -- --pull base_image_source="registry" if [ "$base_image_cache_policy" = "local-if-present" ] && docker image inspect "$base_image" >/dev/null 2>&1; then set -- base_image_source="local-cache" fi docker build "$@" \\ --build-arg BASE_IMAGE="$base_image" \\ --build-arg OPENAI_PYTHON_VERSION="$openai_version" \\ --build-arg HTTP_PROXY= --build-arg HTTPS_PROXY= --build-arg http_proxy= --build-arg https_proxy= \\ --build-arg NO_PROXY --build-arg no_proxy \\ -f "$dockerfile_path" \\ -t "$image" \\ "$work" docker run --rm "$image" python3 -c 'import importlib.metadata, sys; expected=sys.argv[1]; actual=importlib.metadata.version("openai"); assert actual == expected, (actual, expected); print("openai", actual)' "$openai_version" docker push "$image" digest="$(docker image inspect "$image" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)" python3 - < total + value, 0); } function desiredAccountCapacityMap(pool: CodexPoolConfig): Record { const codexDir = join(homedir(), ".codex"); const seenAccountNames = new Set(); const capacities: Record = {}; for (const entry of pool.profiles) { const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames); seenAccountNames.add(accountName); capacities[accountName] = entry.capacity ?? pool.defaultAccountCapacity; } return capacities; } function desiredAccountLoadFactorMap(pool: CodexPoolConfig): Record { const codexDir = join(homedir(), ".codex"); const seenAccountNames = new Set(); const loadFactors: Record = {}; for (const entry of pool.profiles) { const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames); seenAccountNames.add(accountName); loadFactors[accountName] = entry.loadFactor ?? pool.defaultAccountLoadFactor; } return loadFactors; } function desiredAccountWebSocketsV2ModeMap(pool: CodexPoolConfig): Record { const codexDir = join(homedir(), ".codex"); const seenAccountNames = new Set(); const modes: Record = {}; for (const entry of pool.profiles) { const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames); seenAccountNames.add(accountName); modes[accountName] = entry.openaiResponsesWebSocketsV2Mode; } return modes; } function desiredAccountTempUnschedulableMap(pool: CodexPoolConfig): Record { const codexDir = join(homedir(), ".codex"); const seenAccountNames = new Set(); const policies: Record = {}; for (const entry of pool.profiles) { const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames); seenAccountNames.add(accountName); policies[accountName] = renderSub2ApiTempUnschedulableCredentials(entry.tempUnschedulable); } return policies; } function remotePythonScript(mode: "sync" | "validate" | "trace" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { return ` set -u python3 - <<'PY' import base64 import json import re import secrets import string import subprocess import sys import time from datetime import datetime, timezone, timedelta from urllib.parse import quote TARGET_ID = ${JSON.stringify(target.id)} NAMESPACE = ${JSON.stringify(target.namespace)} SERVICE_NAME = ${JSON.stringify(target.serviceName)} SERVICE_DNS = ${JSON.stringify(target.serviceDns)} FIELD_MANAGER = "${fieldManager}" APP_SECRET_NAME = ${JSON.stringify(target.appSecretName)} POOL_GROUP_NAME = "${pool.groupName}" POOL_GROUP_DESCRIPTION = ${JSON.stringify(pool.groupDescription)} POOL_API_KEY_NAME = "${pool.apiKeyName}" POOL_API_KEY_SECRET_NAME = "${pool.apiKeySecretName}" POOL_API_KEY_SECRET_KEY = "${pool.apiKeySecretKey}" POOL_ADMIN_EMAIL_DEFAULT = ${JSON.stringify(pool.adminEmailDefault)} MIN_OWNER_BALANCE_USD = ${JSON.stringify(pool.minOwnerBalanceUsd)} MIN_OWNER_CONCURRENCY = ${JSON.stringify(pool.minOwnerConcurrency)} MIN_OWNER_CONCURRENCY_SOURCE = ${JSON.stringify(pool.minOwnerConcurrencySource)} POOL_DEFAULT_ACCOUNT_PRIORITY = ${JSON.stringify(pool.defaultAccountPriority)} POOL_DEFAULT_ACCOUNT_CAPACITY = ${JSON.stringify(pool.defaultAccountCapacity)} POOL_DEFAULT_ACCOUNT_LOAD_FACTOR = ${JSON.stringify(pool.defaultAccountLoadFactor)} RESPONSES_SMOKE_MODEL = ${JSON.stringify(pool.localCodex.responsesSmokeModel)} EXPECTED_ACCOUNT_CAPACITIES = ${JSON.stringify(desiredAccountCapacityMap(pool))} EXPECTED_ACCOUNT_LOAD_FACTORS = ${JSON.stringify(desiredAccountLoadFactorMap(pool))} EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))}) EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))}) MANUAL_ACCOUNT_PROTECTIONS = json.loads(${JSON.stringify(JSON.stringify(resolvedManualAccountProtections(pool, target)))}) SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))}) TARGET_EGRESS_PROXY = json.loads(${JSON.stringify(JSON.stringify(target.egressProxy))}) MODE = "${mode}" PAYLOAD_B64 = "${encodedPayload}" def run(cmd, input_bytes=None): return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def text(data, limit=4000): if isinstance(data, bytes): data = data.decode("utf-8", errors="replace") return data[-limit:] def kubectl(args, input_obj=None): if isinstance(input_obj, str): input_bytes = input_obj.encode("utf-8") else: input_bytes = input_obj return run(["kubectl", *args], input_bytes) def require_kubectl(args, input_obj=None, label="kubectl"): proc = kubectl(args, input_obj) if proc.returncode != 0: raise RuntimeError(f"{label} failed: {text(proc.stderr, 1000)}") return proc.stdout def kube_json(args, label): raw = require_kubectl([*args, "-o", "json"], label=label) return json.loads(raw.decode("utf-8")) def decode_secret_value(name, key): data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {} if key not in data: return None return base64.b64decode(data[key]).decode("utf-8") def get_config_value(name, key): data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {} value = data.get(key) return value if isinstance(value, str) and value else None def select_app_pod(): pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or [] for pod in pods: status = pod.get("status") or {} if status.get("phase") != "Running": continue statuses = status.get("containerStatuses") or [] if statuses and all(item.get("ready") is True for item in statuses): return pod["metadata"]["name"] if pods: return pods[0]["metadata"]["name"] raise RuntimeError("sub2api app pod not found") APP_POD = select_app_pod() def parse_curl_output(proc): stdout = proc.stdout.decode("utf-8", errors="replace") marker = "\\n__HTTP_CODE__:" pos = stdout.rfind(marker) if pos < 0: return { "ok": False, "httpStatus": 0, "json": None, "body": stdout, "stderr": text(proc.stderr, 1000), "transportExitCode": proc.returncode, } body = stdout[:pos] status_text = stdout[pos + len(marker):].strip() try: http_status = int(status_text[-3:]) except ValueError: http_status = 0 try: parsed = json.loads(body) if body.strip() else None except json.JSONDecodeError: parsed = None return { "ok": proc.returncode == 0 and 200 <= http_status < 300, "httpStatus": http_status, "json": parsed, "body": body, "stderr": text(proc.stderr, 1000), "transportExitCode": proc.returncode, } def utc_iso(offset_seconds=0): return (datetime.now(timezone.utc) + timedelta(seconds=offset_seconds)).strftime("%Y-%m-%dT%H:%M:%SZ") def normalize_runtime_base_url(value): if not isinstance(value, str): return None value = value.strip().rstrip("/") return value or None def empty_to_none(value): return value if isinstance(value, str) and value else None def sentinel_quality_gate_enabled(): return (SENTINEL_CONFIG.get("monitor") or {}).get("enabled") is True and (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is True def account_notes_fingerprint(account): notes = account.get("notes") if isinstance(account, dict) else None if not isinstance(notes, str): return None match = re.search(r"fingerprint=([A-Za-z0-9_-]+)", notes) return match.group(1) if match else None def runtime_account_credentials(account): credentials = account.get("credentials") if isinstance(account, dict) and isinstance(account.get("credentials"), dict) else {} return credentials def runtime_account_extra(account): extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {} return extra def sentinel_probe_change_reasons(current, profile): if not isinstance(current, dict) or current.get("id") is None: return ["created"] credentials = runtime_account_credentials(current) extra = runtime_account_extra(current) expected_base_url = normalize_runtime_base_url(profile.get("baseUrl")) runtime_base_url = normalize_runtime_base_url(credentials.get("base_url")) expected_user_agent = empty_to_none(profile.get("upstreamUserAgent")) runtime_user_agent = empty_to_none(credentials.get("user_agent")) expected_ws_mode = empty_to_none(profile.get("openaiResponsesWebSocketsV2Mode")) runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode")) expected_trust_upstream = profile.get("trustUpstream") is True runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False} reasons = [] if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"): reasons.append("profile") if runtime_base_url != expected_base_url: reasons.append("base-url") if account_notes_fingerprint(current) != profile.get("apiKeyFingerprint"): reasons.append("api-key-fingerprint") if runtime_user_agent != expected_user_agent: reasons.append("upstream-user-agent") if runtime_ws_mode != expected_ws_mode: reasons.append("responses-websockets-v2-mode") if runtime_trust_upstream != expected_trust_upstream: reasons.append("trust-upstream") if runtime_protect != expected_protect: reasons.append("sentinel-protect") return reasons def curl_api(method, path, bearer=None, payload=None): body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") script = r''' set -eu method="$1" url="$2" token="\${3:-}" tmp="$(mktemp)" trap 'rm -f "$tmp"' EXIT cat > "$tmp" if [ -n "$token" ]; then if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url" else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url" fi else if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" "$url" else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" fi fi ''' proc = run([ "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or "", ], body) return parse_curl_output(proc) def envelope_data(parsed): if isinstance(parsed, dict) and "data" in parsed: return parsed.get("data") return parsed def ensure_success(resp, label): parsed = resp.get("json") code = parsed.get("code") if isinstance(parsed, dict) else None if not resp.get("ok") or (code is not None and code != 0): message = parsed.get("message") if isinstance(parsed, dict) else text(resp.get("body", ""), 500) raise RuntimeError(f"{label} failed: http={resp.get('httpStatus')} message={message}") return envelope_data(parsed) def ensure_admin_compliance(token): status_resp = curl_api("GET", "/api/v1/admin/compliance", bearer=token) if status_resp.get("httpStatus") == 404: return { "ok": True, "action": "not-supported-by-runtime", "valuesPrinted": False, } status = ensure_success(status_resp, "get admin compliance") if not isinstance(status, dict): return { "ok": True, "action": "status-unstructured", "valuesPrinted": False, } version = status.get("version") required = status.get("required") is True if not required: return { "ok": True, "action": "already-acknowledged", "version": version, "requiredBefore": False, "requiredAfter": False, "phrasePrinted": False, "valuesPrinted": False, } phrase = status.get("ack_phrase_zh") if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else status.get("ack_phrase_en") language = "zh" if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else "en" if not isinstance(phrase, str) or not phrase: raise RuntimeError("admin compliance acknowledgement phrase missing") accepted = ensure_success( curl_api("POST", "/api/v1/admin/compliance/accept", bearer=token, payload={"phrase": phrase, "language": language}), "accept admin compliance", ) required_after = accepted.get("required") if isinstance(accepted, dict) else None return { "ok": required_after is False, "action": "accepted", "version": version, "requiredBefore": True, "requiredAfter": required_after, "phrasePrinted": False, "valuesPrinted": False, } def extract_items(data): if isinstance(data, list): return data if isinstance(data, dict): if isinstance(data.get("items"), list): return data["items"] for key in ("groups", "accounts", "api_keys", "keys"): if isinstance(data.get(key), list): return data[key] return [] def find_access_token(data): if isinstance(data, dict): for key in ("access_token", "token"): if isinstance(data.get(key), str) and data[key]: return data[key] for value in data.values(): token = find_access_token(value) if token: return token return None def login(): admin_email = get_config_value("sub2api-config", "ADMIN_EMAIL") or POOL_ADMIN_EMAIL_DEFAULT admin_password = decode_secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") if not admin_password: raise RuntimeError("ADMIN_PASSWORD missing from sub2api-secrets") data = ensure_success(curl_api("POST", "/api/v1/auth/login", payload={"email": admin_email, "password": admin_password}), "admin login") token = find_access_token(data) if not token: raise RuntimeError("admin login response did not contain access_token") compliance = ensure_admin_compliance(token) if compliance.get("ok") is not True: raise RuntimeError("admin compliance acknowledgement failed") return admin_email, token, compliance def group_payload(): return { "name": POOL_GROUP_NAME, "description": POOL_GROUP_DESCRIPTION, "platform": "openai", "rate_multiplier": 1, "is_exclusive": False, "subscription_type": "standard", "allow_messages_dispatch": True, "require_oauth_only": False, "require_privacy_set": False, "rpm_limit": 0, } def list_groups(token): data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups") return extract_items(data) def ensure_group(token): existing = next((item for item in list_groups(token) if item.get("name") == POOL_GROUP_NAME), None) payload = group_payload() if existing is None: created = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=payload), "create group") return created, "created" group_id = existing.get("id") if group_id is None: raise RuntimeError("existing group has no id") payload["status"] = "active" updated = ensure_success(curl_api("PUT", f"/api/v1/admin/groups/{group_id}", bearer=token, payload=payload), "update group") return updated if isinstance(updated, dict) else existing, "updated" def list_accounts_for_group(token, group_id): path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500&platform=openai" data = ensure_success(curl_api("GET", path, bearer=token), f"list accounts for group {group_id}") return extract_items(data) def account_group_ids(token, account): if not isinstance(account, dict) or account.get("id") is None: return [] account_id = account.get("id") account_name = account.get("name") ids = [] for group in list_groups(token): group_id = group.get("id") if isinstance(group, dict) else None if group_id is None: continue members = list_accounts_for_group(token, group_id) if any(item.get("id") == account_id or item.get("name") == account_name for item in members if isinstance(item, dict)): ids.append(group_id) return sorted(set(ids)) def list_accounts(token): path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-codex-") data = ensure_success(curl_api("GET", path, bearer=token), "list accounts") return extract_items(data) def find_account_by_name(token, name): path = "/api/v1/admin/accounts?page=1&page_size=50&platform=openai&search=" + quote(str(name)) data = ensure_success(curl_api("GET", path, bearer=token), "find account " + str(name)) for item in extract_items(data): if isinstance(item, dict) and item.get("name") == name: return item return None def list_proxy_candidates(token, search=""): path = "/api/v1/admin/proxies?page=1&page_size=200" if search: path += "&search=" + quote(str(search)) data = ensure_success(curl_api("GET", path, bearer=token), "list proxies") return extract_items(data) def find_proxy_by_name(token, name): for item in list_proxy_candidates(token, name): if isinstance(item, dict) and item.get("name") == name: return item return None def manual_binding_plan(binding, expected_kind): if not isinstance(binding, dict): return None plan = binding.get("sourcePlan") if not isinstance(plan, dict): raise RuntimeError("manual account binding is missing resolved sourcePlan") if plan.get("enabled") is not True: raise RuntimeError(f"manual account binding source {plan.get('id')} is disabled") if plan.get("kind") != expected_kind: raise RuntimeError(f"manual account binding source {plan.get('id')} kind must be {expected_kind}") return plan def desired_manual_proxy_payload(protection): binding = protection.get("proxyBinding") if isinstance(protection, dict) else None if not isinstance(binding, dict) or binding.get("enabled") is not True: return None plan = manual_binding_plan(binding, "proxy") if plan.get("provider") != "target-egress-proxy": raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}") target_proxy = plan.get("targetEgressProxy") if not isinstance(target_proxy, dict): raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} requires an enabled target egressProxy") service_name = target_proxy.get("serviceName") listen_port = target_proxy.get("listenPort") namespace = target_proxy.get("namespace") if isinstance(target_proxy.get("namespace"), str) and target_proxy.get("namespace") else NAMESPACE proxy_name = binding.get("proxyName") if not isinstance(service_name, str) or not service_name: raise RuntimeError("target egressProxy serviceName is missing") if not isinstance(listen_port, int) or listen_port <= 0: raise RuntimeError("target egressProxy listenPort is missing") if not isinstance(proxy_name, str) or not proxy_name: raise RuntimeError("manual account proxyBinding proxyName is missing") return { "name": proxy_name, "protocol": "http", "host": f"{service_name}.{namespace}.svc.cluster.local", "port": listen_port, "fallback_mode": "none", "expiry_warn_days": 0, } def proxy_needs_update(proxy, payload): if not isinstance(proxy, dict): return True for key in ("name", "protocol", "host", "port", "fallback_mode", "expiry_warn_days"): if proxy.get(key) != payload.get(key): return True return proxy.get("status") != "active" def ensure_manual_proxy(token, payload): current = find_proxy_by_name(token, payload["name"]) if current is None: created = ensure_success(curl_api("POST", "/api/v1/admin/proxies", bearer=token, payload=payload), f"create proxy {payload['name']}") return created, "created" if proxy_needs_update(current, payload): update_payload = dict(payload) update_payload["status"] = "active" updated = ensure_success(curl_api("PUT", f"/api/v1/admin/proxies/{current['id']}", bearer=token, payload=update_payload), f"update proxy {payload['name']}") return updated if isinstance(updated, dict) else current, "updated" return current, "unchanged" def manual_proxy_status(token, account, protection): payload = desired_manual_proxy_payload(protection) if payload is None: return { "enabled": False, "ok": True, "action": "not-configured", "valuesPrinted": False, } binding = protection.get("proxyBinding") if isinstance(protection, dict) else None plan = manual_binding_plan(binding, "proxy") proxy = find_proxy_by_name(token, payload["name"]) runtime_proxy = account.get("proxy") if isinstance(account, dict) and isinstance(account.get("proxy"), dict) else None binding_aligned = ( isinstance(account, dict) and isinstance(proxy, dict) and account.get("proxy_id") == proxy.get("id") ) return { "enabled": True, "ok": binding_aligned, "action": "validate", "source": plan.get("id"), "sourceProvider": plan.get("provider"), "expectedProxyName": payload["name"], "expectedProtocol": payload["protocol"], "expectedHost": payload["host"], "expectedPort": payload["port"], "proxyRecordExists": isinstance(proxy, dict), "proxyId": proxy.get("id") if isinstance(proxy, dict) else None, "runtimeProxyId": account.get("proxy_id") if isinstance(account, dict) else None, "runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None, "bindingAligned": binding_aligned, "valuesPrinted": False, } def ensure_manual_account_proxy_bindings(token): items = [] for protection in MANUAL_ACCOUNT_PROTECTIONS: if not isinstance(protection, dict): continue name = protection.get("accountName") if not isinstance(name, str) or not name: continue account = find_account_by_name(token, name) payload = desired_manual_proxy_payload(protection) if payload is None: items.append({ "accountName": name, "enabled": False, "action": "not-configured", "ok": True, "valuesPrinted": False, }) continue if not isinstance(account, dict): items.append({ "accountName": name, "enabled": True, "action": "account-missing", "ok": False, "expectedProxyName": payload["name"], "valuesPrinted": False, }) continue extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} if extra.get("unidesk_managed") is True: items.append({ "accountName": name, "accountId": account.get("id"), "enabled": True, "action": "refused-managed-runtime-account", "ok": False, "expectedProxyName": payload["name"], "valuesPrinted": False, }) continue proxy, proxy_action = ensure_manual_proxy(token, payload) binding = protection.get("proxyBinding") if isinstance(protection, dict) else None plan = manual_binding_plan(binding, "proxy") proxy_id = proxy.get("id") if isinstance(proxy, dict) else None if proxy_id is None: raise RuntimeError(f"proxy {payload['name']} has no id") action = "unchanged" if account.get("proxy_id") != proxy_id: updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"proxy_id": proxy_id}), f"bind manual account proxy {name}") account = updated if isinstance(updated, dict) else account action = "bound" runtime_proxy = account.get("proxy") if isinstance(account.get("proxy"), dict) else None items.append({ "accountName": name, "accountId": account.get("id"), "enabled": True, "action": action, "proxyAction": proxy_action, "source": plan.get("id"), "sourceProvider": plan.get("provider"), "ok": account.get("proxy_id") == proxy_id, "expectedProxyName": payload["name"], "expectedProtocol": payload["protocol"], "expectedHost": payload["host"], "expectedPort": payload["port"], "proxyId": proxy_id, "runtimeProxyId": account.get("proxy_id"), "runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None, "bindingAligned": account.get("proxy_id") == proxy_id, "controlPolicy": "manual-protected: only proxy_id binding is YAML-controlled; credentials/status/schedulable are untouched", "valuesPrinted": False, }) return { "ok": all(item.get("ok") is True for item in items), "itemCount": len(items), "items": items, "valuesPrinted": False, } def manual_group_binding_plan(protection): binding = protection.get("groupBinding") if isinstance(protection, dict) else None if not isinstance(binding, dict) or binding.get("enabled") is not True: return None plan = manual_binding_plan(binding, "group") if plan.get("provider") != "pool-group": raise RuntimeError(f"manual account groupBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}") return plan def manual_group_binding_enabled(protection): return manual_group_binding_plan(protection) is not None def manual_group_status(token, account, protection, group_id): plan = manual_group_binding_plan(protection) if plan is None: return { "enabled": False, "ok": True, "action": "not-configured", "valuesPrinted": False, } group_accounts = list_accounts_for_group(token, group_id) account_id = account.get("id") if isinstance(account, dict) else None account_name = account.get("name") if isinstance(account, dict) else None binding_aligned = any( item.get("id") == account_id or item.get("name") == account_name for item in group_accounts if isinstance(item, dict) ) return { "enabled": True, "ok": binding_aligned, "action": "validate", "source": plan.get("id"), "sourceProvider": plan.get("provider"), "poolGroupName": POOL_GROUP_NAME, "poolGroupId": group_id, "bindingAligned": binding_aligned, "valuesPrinted": False, } def ensure_manual_account_group_bindings(token, group_id): items = [] for protection in MANUAL_ACCOUNT_PROTECTIONS: if not isinstance(protection, dict): continue name = protection.get("accountName") if not isinstance(name, str) or not name: continue if not manual_group_binding_enabled(protection): items.append({ "accountName": name, "enabled": False, "action": "not-configured", "ok": True, "valuesPrinted": False, }) continue plan = manual_group_binding_plan(protection) account = find_account_by_name(token, name) if not isinstance(account, dict): items.append({ "accountName": name, "enabled": True, "action": "account-missing", "ok": False, "poolGroupName": POOL_GROUP_NAME, "poolGroupId": group_id, "valuesPrinted": False, }) continue extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} if extra.get("unidesk_managed") is True: items.append({ "accountName": name, "accountId": account.get("id"), "enabled": True, "action": "refused-managed-runtime-account", "ok": False, "poolGroupName": POOL_GROUP_NAME, "poolGroupId": group_id, "valuesPrinted": False, }) continue existing_group_ids = account_group_ids(token, account) desired_group_ids = sorted(set(existing_group_ids + [group_id])) action = "unchanged" if group_id not in existing_group_ids: updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"group_ids": desired_group_ids}), f"bind manual account group {name}") account = updated if isinstance(updated, dict) else account action = "bound" binding_aligned = any( item.get("id") == account.get("id") or item.get("name") == name for item in list_accounts_for_group(token, group_id) if isinstance(item, dict) ) items.append({ "accountName": name, "accountId": account.get("id"), "enabled": True, "ok": binding_aligned, "action": action, "source": plan.get("id") if isinstance(plan, dict) else None, "sourceProvider": plan.get("provider") if isinstance(plan, dict) else None, "poolGroupName": POOL_GROUP_NAME, "poolGroupId": group_id, "previousGroupIds": existing_group_ids, "desiredGroupIds": desired_group_ids, "bindingAligned": binding_aligned, "controlPolicy": "manual-protected: only pool group membership is YAML-controlled; credentials/status/schedulable are untouched and sentinel does not probe it", "valuesPrinted": False, }) return { "ok": all(item.get("ok") is True for item in items), "itemCount": len(items), "items": items, "valuesPrinted": False, } def manual_account_protection_status(token, group_id=None): items = [] desired_names = set(EXPECTED_ACCOUNT_CAPACITIES.keys()) for protection in MANUAL_ACCOUNT_PROTECTIONS: if not isinstance(protection, dict): continue name = protection.get("accountName") if not isinstance(name, str) or not name: continue account = find_account_by_name(token, name) extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {} proxy_status = manual_proxy_status(token, account, protection) group_status = manual_group_status(token, account, protection, group_id) if group_id is not None else {"enabled": False, "ok": True, "action": "not-checked", "valuesPrinted": False} items.append({ "accountName": name, "reason": protection.get("reason") if isinstance(protection.get("reason"), str) else None, "exists": isinstance(account, dict), "accountId": account.get("id") if isinstance(account, dict) else None, "status": account.get("status") if isinstance(account, dict) else None, "schedulable": account.get("schedulable") if isinstance(account, dict) else None, "inYamlProfiles": name in desired_names, "runtimeMarkedUnideskManaged": extra.get("unidesk_managed") is True, "proxyBinding": proxy_status, "groupBinding": group_status, "ok": proxy_status.get("ok") is True and group_status.get("ok") is True, "controlPolicy": "manual-protected: no create/update/prune/probe/freeze; optional proxy_id and pool group membership binding only when configured", "valuesPrinted": False, }) return { "ok": all(item.get("ok") is True for item in items), "protectedCount": len(items), "items": items, "valuesPrinted": False, } def list_probe_accounts(token): path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-probe-") data = ensure_success(curl_api("GET", path, bearer=token), "list probe accounts") return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")] def list_probe_groups(token): data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups") return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")] def cleanup_probe_resources(token): api_key_results = [] account_results = [] group_results = [] for item in list_user_keys(token): name = item.get("name") key_id = item.get("id") if not isinstance(name, str) or not name.startswith("unidesk-probe-") or key_id is None: continue api_key_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{key_id}", "api-key")) for item in list_probe_accounts(token): account_id = item.get("id") if account_id is None: continue account_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account")) for item in list_probe_groups(token): group_id = item.get("id") if group_id is None: continue group_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group")) return { "ok": all(item.get("ok") is True for item in api_key_results + account_results + group_results), "apiKeysDeleted": len(api_key_results), "accountsDeleted": len(account_results), "groupsDeleted": len(group_results), "items": { "apiKeys": api_key_results, "accounts": account_results, "groups": group_results, }, "valuesPrinted": False, } def temp_unschedulable_credentials(profile): credentials = profile.get("tempUnschedulableCredentials") if not isinstance(credentials, dict): credentials = {} return normalize_temp_unschedulable_credentials(credentials) def account_payload(profile, group_id): extra = { "openai_responses_mode": "force_responses", "unidesk_codex_profile": profile["profile"], "unidesk_managed": True, "unidesk_trust_upstream": profile.get("trustUpstream") is True, "unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, } ws_mode = profile.get("openaiResponsesWebSocketsV2Mode") if ws_mode: extra["openai_apikey_responses_websockets_v2_mode"] = ws_mode extra["openai_apikey_responses_websockets_v2_enabled"] = ws_mode != "off" credentials = { "api_key": profile["apiKey"], "base_url": profile["baseUrl"], } upstream_user_agent = profile.get("upstreamUserAgent") if upstream_user_agent: credentials["user_agent"] = upstream_user_agent temp_unschedulable = temp_unschedulable_credentials(profile) if temp_unschedulable["enabled"]: credentials["temp_unschedulable_enabled"] = True credentials["temp_unschedulable_rules"] = temp_unschedulable["rules"] return { "name": profile["accountName"], "notes": f"UniDesk-managed Codex profile {profile['profile']} from {profile['configFile']} and {profile['authFile']}; secret source={profile['apiKeySource']}; fingerprint={profile['apiKeyFingerprint']}.", "platform": "openai", "type": "apikey", "credentials": credentials, "extra": extra, "concurrency": int(profile.get("capacity", 5) or 5), "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY), "rate_multiplier": 1, "load_factor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR), "group_ids": [group_id], "confirm_mixed_channel_risk": True, } def planned_sentinel_account_results(profiles, existing_accounts): existing = {item.get("name"): item for item in existing_accounts if isinstance(item, dict)} results = [] for profile in profiles: change_reasons = sentinel_probe_change_reasons(existing.get(profile["accountName"]), profile) quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0 results.append({ "profile": profile["profile"], "accountName": profile["accountName"], "profileConfig": { "accountName": profile["accountName"], "trustUpstream": profile.get("trustUpstream") is True, "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, }, "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), "sentinelProbeRequired": quality_gate_required, "sentinelChangeReasons": change_reasons if quality_gate_required else [], "sentinelProbePending": quality_gate_required, "sentinelDefaultFrozen": False, "valuesPrinted": False, }) return results def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_frozen_names=None, existing_accounts=None): if not isinstance(protected_frozen_names, set): protected_frozen_names = set() if not isinstance(existing_accounts, list): existing_accounts = list_accounts(token) existing = {item.get("name"): item for item in existing_accounts} desired_names = {profile["accountName"] for profile in profiles} results = [] for profile in profiles: payload = account_payload(profile, group_id) current = existing.get(profile["accountName"]) change_reasons = sentinel_probe_change_reasons(current, profile) quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0 keep_frozen = profile["accountName"] in protected_frozen_names if current and current.get("id") is not None: account_id = current["id"] update_payload = dict(payload) update_payload.pop("platform", None) update_payload["status"] = "active" data = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account_id}", bearer=token, payload=update_payload), f"update account {profile['accountName']}") action = "updated" else: data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}") action = "created" results.append({ "profile": profile["profile"], "accountName": profile["accountName"], "profileConfig": { "accountName": profile["accountName"], "trustUpstream": profile.get("trustUpstream") is True, "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, }, "accountId": data.get("id") if isinstance(data, dict) else None, "action": action, "baseUrl": profile["baseUrl"], "apiKeySource": profile["apiKeySource"], "apiKeyFingerprint": profile["apiKeyFingerprint"], "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), "sentinelProbeRequired": quality_gate_required, "sentinelChangeReasons": change_reasons if quality_gate_required else [], "sentinelProbePending": quality_gate_required, "sentinelDefaultFrozen": False, "sentinelFreezeProtected": keep_frozen, "schedulableControl": "sentinel-marker-restore", "openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"), "trustUpstream": profile.get("trustUpstream") is True, "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY), "capacity": int(profile.get("capacity", 5) or 5), "loadFactor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR), "runtimeConcurrency": data.get("concurrency") if isinstance(data, dict) else None, "runtimeLoadFactor": data.get("load_factor") if isinstance(data, dict) else None, "runtimeSchedulable": data.get("schedulable") if isinstance(data, dict) else None, "tempUnschedulableConfigured": bool(payload["credentials"].get("temp_unschedulable_enabled")), "tempUnschedulableRuleCount": len(payload["credentials"].get("temp_unschedulable_rules") or []), "upstreamUserAgentConfigured": bool(profile.get("upstreamUserAgent")), "valuesPrinted": False, }) prune_results = prune_removed_accounts(token, existing_accounts, desired_names) if prune_removed else [] return results, prune_results def prune_removed_accounts(token, existing_accounts, desired_names): results = [] for account in existing_accounts: name = account.get("name") account_id = account.get("id") extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} if not isinstance(name, str) or not name.startswith("unidesk-codex-"): continue if extra.get("unidesk_managed") is not True: continue if name in desired_names: continue if account_id is None: raise RuntimeError(f"removed account {name} has no id") ensure_success(curl_api("DELETE", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"delete removed account {name}") results.append({ "accountName": name, "accountId": account_id, "profile": extra.get("unidesk_codex_profile"), "action": "deleted", "reason": "removed-from-yaml", "valuesPrinted": False, }) return results def generate_api_key(): alphabet = string.ascii_letters + string.digits return "sk-unidesk-codex-" + "".join(secrets.choice(alphabet) for _ in range(48)) def ensure_api_key_secret(group_id): existing = None try: existing = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY) except Exception: existing = None api_key = existing if existing else generate_api_key() secret_action = "kept-existing" if existing else "created" manifest = { "apiVersion": "v1", "kind": "Secret", "metadata": { "name": POOL_API_KEY_SECRET_NAME, "namespace": NAMESPACE, "labels": { "app.kubernetes.io/name": "sub2api", "app.kubernetes.io/part-of": "platform-infra", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/secret-purpose": "sub2api-codex-pool-api-key", }, }, "type": "Opaque", "stringData": { POOL_API_KEY_SECRET_KEY: api_key, "GROUP_ID": str(group_id), "GROUP_NAME": POOL_GROUP_NAME, "SERVICE_DNS": SERVICE_DNS, }, } proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) if proc.returncode != 0: raise RuntimeError(f"apply API key secret failed: {text(proc.stderr, 1000)}") return api_key, secret_action, text(proc.stdout, 1000) def apply_sentinel_manifest(manifest): if not isinstance(manifest, str) or not manifest.strip(): return { "ok": False, "action": "missing-manifest", "valuesPrinted": False, } proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest) if proc.returncode != 0: return { "ok": False, "action": "apply-failed", "stdoutTail": text(proc.stdout, 2000), "stderrTail": text(proc.stderr, 4000), "valuesPrinted": False, } status = sentinel_runtime_status() return { "ok": status.get("ok") is True, "action": "applied", "stdoutTail": text(proc.stdout, 2000), "runtime": status, "valuesPrinted": False, } def safe_kube_json(args, label): proc = kubectl([*args, "-o", "json"]) if proc.returncode != 0: return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)} try: return json.loads(proc.stdout.decode("utf-8")), None except Exception as exc: return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)} def sentinel_runtime_status(): cfg = SENTINEL_CONFIG cronjob_name = cfg.get("cronJobName") secret_name = cfg.get("credentialsSecretName") configmap_name = cfg.get("configMapName") state_name = cfg.get("stateConfigMapName") cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}") secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}") configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}") state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}") state = None if isinstance(state_cm, dict): raw_state = (state_cm.get("data") or {}).get("state.json") if isinstance(raw_state, str) and raw_state: try: state = json.loads(raw_state) except Exception as exc: state = {"parseError": str(exc)} accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {} quarantined = [] recent_accounts = [] for name, account_state in accounts.items(): if not isinstance(account_state, dict): continue quarantine = account_state.get("quarantine") if isinstance(quarantine, dict) and quarantine.get("active") is True: quarantined.append({ "accountName": name, "until": quarantine.get("until"), "applied": quarantine.get("applied"), "reason": quarantine.get("reason"), "failureKind": quarantine.get("failureKind"), "errorDetails": quarantine.get("errorDetails"), "intervalMinutes": quarantine.get("intervalMinutes"), "sentinelProtect": quarantine.get("sentinelProtect"), }) last_probe = account_state.get("lastProbe") if isinstance(last_probe, dict): recent_accounts.append({ "accountName": name, "lastProbeAt": account_state.get("lastProbeAt"), "lastStatus": account_state.get("lastStatus"), "nextProbeAfter": account_state.get("nextProbeAfter"), "ok": last_probe.get("ok"), "purpose": last_probe.get("purpose"), "httpStatus": last_probe.get("httpStatus"), "durationMs": last_probe.get("durationMs"), "markerMatched": last_probe.get("markerMatched"), "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), "outputHash": last_probe.get("outputHash"), "outputPreview": last_probe.get("outputPreview"), "responseBodyHash": last_probe.get("responseBodyHash"), "responseBodyPreview": last_probe.get("responseBodyPreview"), "error": last_probe.get("error"), "errorDetails": last_probe.get("errorDetails"), "usage": last_probe.get("usage"), "failureKind": last_probe.get("failureKind"), "requestShape": last_probe.get("requestShape"), "action": last_probe.get("action"), }) recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "") last_run = state.get("lastRun") if isinstance(state, dict) else None cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {} secret_data = secret.get("data") if isinstance(secret, dict) else {} configmap_data = configmap.get("data") if isinstance(configmap, dict) else {} ok = cronjob is not None and secret is not None and configmap is not None return { "ok": ok, "desired": { "monitorEnabled": cfg.get("monitor", {}).get("enabled"), "actionsEnabled": cfg.get("actions", {}).get("enabled"), "schedule": cfg.get("schedule"), "cronJobName": cronjob_name, "configMapName": configmap_name, "credentialsSecretName": secret_name, "stateConfigMapName": state_name, }, "cronJob": { "exists": cronjob is not None, "schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None, "suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None, "lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None, "active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None, "error": cronjob_error, }, "secret": { "exists": secret is not None, "profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data, "valuesPrinted": False, "error": secret_error, }, "configMap": { "exists": configmap is not None, "configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data, "runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data, "error": configmap_error, }, "state": { "exists": state_cm is not None, "accountCount": len(accounts), "quarantinedCount": len(quarantined), "quarantined": quarantined[-10:], "recentAccounts": recent_accounts[-12:], "lastRun": last_run, "error": state_error, }, "valuesPrinted": False, } def parse_epoch_z(value): if not isinstance(value, str) or not value: return None try: return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() except Exception: return None def sentinel_state_object(): state_name = SENTINEL_CONFIG.get("stateConfigMapName") if not state_name: return None, None obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}") if not isinstance(obj, dict): return None, None raw_state = (obj.get("data") or {}).get("state.json") if not isinstance(raw_state, str) or not raw_state: return obj, None try: return obj, json.loads(raw_state) except Exception: return obj, None def active_sentinel_quarantine_names(): _, state = sentinel_state_object() if not isinstance(state, dict): return set() accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} names = set() for name, account_state in accounts_state.items(): if not isinstance(name, str) or not isinstance(account_state, dict): continue quarantine = account_state.get("quarantine") if isinstance(quarantine, dict) and quarantine.get("active") is True and quarantine.get("applied") is True: names.add(name) return names def default_sentinel_state(): return {"version": 1, "accounts": {}, "ledger": {}, "history": []} def clamp_sentinel_freezes_for_config(state, now): freeze_config = SENTINEL_CONFIG.get("freeze") if isinstance(SENTINEL_CONFIG.get("freeze"), dict) else {} try: max_interval = int(freeze_config.get("maxTtlMinutes") or 10) except Exception: max_interval = 10 accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} now_epoch = time.time() items = [] for name, account_state in accounts_state.items(): if not isinstance(name, str) or not isinstance(account_state, dict): continue quarantine = account_state.get("quarantine") if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True: continue try: interval = int(quarantine.get("intervalMinutes") or 0) except Exception: interval = 0 until_epoch = parse_epoch_z(quarantine.get("until")) old_until = quarantine.get("until") if interval <= max_interval and (until_epoch is None or until_epoch <= now_epoch + max_interval * 60): continue quarantine["previousIntervalMinutes"] = interval quarantine["intervalMinutes"] = max_interval quarantine["until"] = now quarantine["clampedAt"] = now quarantine["clampedBy"] = "sync-freeze-max-ttl" account_state["nextProbeAfter"] = now items.append({ "accountName": name, "previousIntervalMinutes": interval, "maxIntervalMinutes": max_interval, "previousUntil": old_until, "nextProbeAfter": now, }) return items def parse_iso_epoch(value): if not isinstance(value, str) or not value: return None try: return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() except Exception: return None def profile_success_max_interval(profile): cadence = SENTINEL_CONFIG.get("cadence") if isinstance(SENTINEL_CONFIG.get("cadence"), dict) else {} legacy = cadence.get("successMaxIntervalMinutes") if legacy is None: legacy = cadence.get("trustedSuccessMaxIntervalMinutes") or cadence.get("untrustedSuccessMaxIntervalMinutes") or 1 if profile.get("trustUpstream") is True: value = cadence.get("trustedSuccessMaxIntervalMinutes") or legacy else: value = cadence.get("untrustedSuccessMaxIntervalMinutes") or legacy try: return int(value) except Exception: return int(legacy) def clamp_sentinel_success_cadence_for_config(state, profiles, now): accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} profile_map = {item.get("accountName"): item for item in profiles if isinstance(item, dict) and isinstance(item.get("accountName"), str)} now_epoch = time.time() items = [] for name, profile in profile_map.items(): account_state = accounts_state.get(name) if not isinstance(account_state, dict): continue quarantine = account_state.get("quarantine") if isinstance(quarantine, dict) and quarantine.get("active") is True: account_state["trustUpstream"] = profile.get("trustUpstream") is True account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile) continue try: interval = int(account_state.get("successIntervalMinutes") or 0) except Exception: interval = 0 next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter")) max_interval = profile_success_max_interval(profile) account_state["trustUpstream"] = profile.get("trustUpstream") is True account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = max_interval if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60): continue old_next = account_state.get("nextProbeAfter") account_state["previousSuccessIntervalMinutes"] = interval account_state["successIntervalMinutes"] = min(interval, max_interval) if interval > 0 else interval account_state["nextProbeAfter"] = now account_state["cadenceClampedAt"] = now account_state["cadenceClampedBy"] = "sync-success-max-interval" items.append({ "accountName": name, "trustUpstream": profile.get("trustUpstream") is True, "previousSuccessIntervalMinutes": interval, "maxIntervalMinutes": max_interval, "previousNextProbeAfter": old_next, "nextProbeAfter": now, }) return items def update_sentinel_state_configmap(obj, state): state_name = SENTINEL_CONFIG.get("stateConfigMapName") if not state_name: return {"ok": False, "reason": "state-configmap-missing"} state_json = json.dumps(state, ensure_ascii=False, indent=2) manifest = { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": state_name, "namespace": NAMESPACE, "labels": { "app.kubernetes.io/name": SERVICE_NAME, "app.kubernetes.io/component": "account-sentinel", "app.kubernetes.io/managed-by": "unidesk-platform-infra", }, }, "data": {"state.json": state_json}, } proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) action = "applied" if isinstance(obj, dict) else "created" if proc.returncode != 0: return {"ok": False, "reason": f"{action}-failed", "stderrTail": text(proc.stderr, 2000)} return {"ok": True, "action": action} def ensure_sentinel_state_for_sync(account_results, pending_only=False): if not sentinel_quality_gate_enabled(): return {"ok": True, "skipped": True, "reason": "sentinel-quality-gate-disabled", "changedCount": 0, "items": [], "valuesPrinted": False} state_obj, state = sentinel_state_object() if not isinstance(state, dict): state = default_sentinel_state() state.setdefault("version", 1) accounts_state = state.setdefault("accounts", {}) if not isinstance(accounts_state, dict): accounts_state = {} state["accounts"] = accounts_state now = utc_iso() items = [] clamped_items = [] if pending_only else clamp_sentinel_freezes_for_config(state, now) cadence_clamped_items = [] if pending_only else clamp_sentinel_success_cadence_for_config(state, [item.get("profileConfig") for item in account_results if isinstance(item.get("profileConfig"), dict)], now) changed_count = 0 fingerprint_only_count = 0 for item in account_results: name = item.get("accountName") if not isinstance(name, str) or not name: continue account_state = accounts_state.setdefault(name, {}) if not isinstance(account_state, dict): account_state = {} accounts_state[name] = account_state fingerprint_value = item.get("sentinelProbeConfigFingerprint") if isinstance(fingerprint_value, str) and fingerprint_value: account_state["probeConfigFingerprint"] = fingerprint_value if item.get("sentinelProbeRequired") is not True: fingerprint_only_count += 1 continue changed_count += 1 reasons = item.get("sentinelChangeReasons") if isinstance(item.get("sentinelChangeReasons"), list) else [] quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else None if not (isinstance(quarantine, dict) and quarantine.get("active") is True): account_state["quarantine"] = { "active": False, "reason": "yaml-account-change-pending-sentinel-probe", "lastPendingAt": now, "changeReasons": reasons, } account_state["nextProbeAfter"] = now account_state["successStreak"] = 0 account_state["successIntervalMinutes"] = 0 profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {} account_state["trustUpstream"] = profile_config.get("trustUpstream") is True account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config) account_state["lastStatus"] = "pending-sentinel-quality-gate" account_state["qualityGate"] = { "pending": True, "reason": "yaml-account-change", "changeReasons": reasons, "markedAt": now, "pendingOnly": pending_only, "defaultFrozen": False, } items.append({"accountName": name, "changeReasons": reasons, "nextProbeAfter": now, "defaultFrozen": False, "defaultSchedulable": True, "pendingOnly": pending_only}) if changed_count <= 0 and len(clamped_items) <= 0 and len(cadence_clamped_items) <= 0: return {"ok": True, "skipped": False, "reason": "no-new-or-changed-accounts", "changedCount": 0, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": 0, "cadenceClampedCount": 0, "items": [], "valuesPrinted": False} update = update_sentinel_state_configmap(state_obj, state) if pending_only and changed_count > 0: reason = "new-or-changed-accounts-pending-probe-prepared-default-schedulable" elif changed_count > 0 and (len(clamped_items) > 0 or len(cadence_clamped_items) > 0): reason = "new-or-changed-accounts-default-schedulable-and-sentinel-cadence-clamped" elif changed_count > 0: reason = "new-or-changed-accounts-default-schedulable" elif len(cadence_clamped_items) > 0: reason = "success-cadence-clamped-to-current-config" else: reason = "freeze-backoff-clamped-to-current-config" return { "ok": update.get("ok") is True, "skipped": False, "reason": reason, "changedCount": changed_count, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": len(clamped_items), "cadenceClampedCount": len(cadence_clamped_items), "pendingOnly": pending_only, "items": items, "clampedItems": clamped_items, "cadenceClampedItems": cadence_clamped_items, "update": update, "valuesPrinted": False, } def sentinel_state_summary(): _, state = sentinel_state_object() if not isinstance(state, dict): return {"exists": False, "valuesPrinted": False} accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} quarantined = [] recent_accounts = [] for name, account_state in accounts.items(): if not isinstance(account_state, dict): continue quarantine = account_state.get("quarantine") if isinstance(quarantine, dict) and quarantine.get("active") is True: quarantined.append({ "accountName": name, "until": quarantine.get("until"), "applied": quarantine.get("applied"), "reason": quarantine.get("reason"), "failureKind": quarantine.get("failureKind"), "errorDetails": quarantine.get("errorDetails"), "intervalMinutes": quarantine.get("intervalMinutes"), "sentinelProtect": quarantine.get("sentinelProtect"), }) last_probe = account_state.get("lastProbe") if isinstance(last_probe, dict): recent_accounts.append({ "accountName": name, "lastProbeAt": account_state.get("lastProbeAt"), "lastStatus": account_state.get("lastStatus"), "nextProbeAfter": account_state.get("nextProbeAfter"), "ok": last_probe.get("ok"), "purpose": last_probe.get("purpose"), "httpStatus": last_probe.get("httpStatus"), "durationMs": last_probe.get("durationMs"), "markerMatched": last_probe.get("markerMatched"), "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), "usage": last_probe.get("usage"), "responseBodyHash": last_probe.get("responseBodyHash"), "responseBodyPreview": last_probe.get("responseBodyPreview"), "error": last_probe.get("error"), "errorDetails": last_probe.get("errorDetails"), "failureKind": last_probe.get("failureKind"), "requestShape": last_probe.get("requestShape"), "action": last_probe.get("action"), }) recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "") return { "exists": True, "accountCount": len(accounts), "quarantinedCount": len(quarantined), "quarantined": quarantined[-10:], "recentAccounts": recent_accounts[-12:], "lastRun": state.get("lastRun"), "valuesPrinted": False, } def reassert_sentinel_freezes_after_sync(token): if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True: return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False} _, state = sentinel_state_object() if not isinstance(state, dict): return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False} accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {} accounts = list_accounts(token) by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} items = [] for name, account_state in accounts_state.items(): if not isinstance(account_state, dict): continue quarantine = account_state.get("quarantine") if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True: continue account = by_name.get(name) if not account or account.get("id") is None: items.append({"accountName": name, "ok": False, "reason": "account-not-found"}) continue try: ensure_success( curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}), f"reassert sentinel freeze for {name}", ) items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")}) except Exception as exc: items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)}) return { "ok": all(item.get("ok") is True for item in items), "skipped": False, "items": items, "valuesPrinted": False, } def list_user_keys(token): data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys") return extract_items(data) def ensure_sub2api_api_key(token, api_key, group_id): keys = list_user_keys(token) existing = next((item for item in keys if item.get("key") == api_key), None) action = "kept-existing" if existing is None: existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None) if existing is None or existing.get("key") != api_key: payload = { "name": POOL_API_KEY_NAME, "group_id": group_id, "custom_key": api_key, "quota": 0, "rate_limit_5h": 0, "rate_limit_1d": 0, "rate_limit_7d": 0, } created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key") existing = created if isinstance(created, dict) else existing action = "created" elif existing.get("id") is not None and existing.get("group_id") != group_id: updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group") existing = updated if isinstance(updated, dict) else existing action = "updated-group" return { "action": action, "id": existing.get("id") if isinstance(existing, dict) else None, "name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME, "groupId": existing.get("group_id") if isinstance(existing, dict) else group_id, "userId": existing.get("user_id") if isinstance(existing, dict) else None, } def get_admin_user(token, user_id): data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner") if not isinstance(data, dict): raise RuntimeError("API key owner response is not an object") return data def ensure_pool_owner_balance(token, user_id): user = get_admin_user(token, user_id) current = float(user.get("balance") or 0) if current >= MIN_OWNER_BALANCE_USD: return { "action": "kept-existing", "userId": user_id, "balanceBefore": current, "balanceAfter": current, "minimumBalanceUsd": MIN_OWNER_BALANCE_USD, } updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={ "balance": MIN_OWNER_BALANCE_USD, "operation": "set", "notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.", }), "set API key owner balance") after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD return { "action": "set", "userId": user_id, "balanceBefore": current, "balanceAfter": after, "minimumBalanceUsd": MIN_OWNER_BALANCE_USD, } def ensure_pool_owner_concurrency(token, user_id): user = get_admin_user(token, user_id) try: current = int(user.get("concurrency") or 0) except Exception: current = 0 if current >= MIN_OWNER_CONCURRENCY: return { "ok": True, "action": "kept-existing", "userId": user_id, "concurrencyBefore": current, "concurrencyAfter": current, "minimumConcurrency": MIN_OWNER_CONCURRENCY, "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, } updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={ "concurrency": MIN_OWNER_CONCURRENCY, }), "set API key owner concurrency") try: after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY except Exception: after = MIN_OWNER_CONCURRENCY return { "ok": after >= MIN_OWNER_CONCURRENCY, "action": "set", "userId": user_id, "concurrencyBefore": current, "concurrencyAfter": after, "minimumConcurrency": MIN_OWNER_CONCURRENCY, "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, } def validate_gateway(api_key): resp = curl_api("GET", "/v1/models", bearer=api_key) parsed = resp.get("json") model_count = None if isinstance(parsed, dict) and isinstance(parsed.get("data"), list): model_count = len(parsed["data"]) return { "ok": resp.get("ok"), "httpStatus": resp.get("httpStatus"), "transportExitCode": resp.get("transportExitCode"), "modelCount": model_count, "bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "", "stderr": resp.get("stderr", ""), "method": "GET /v1/models", "serviceDns": SERVICE_DNS, "valuesPrinted": False, } def response_output_preview(parsed): if not isinstance(parsed, dict): return "" if isinstance(parsed.get("output_text"), str): return parsed["output_text"][:240] output = parsed.get("output") if not isinstance(output, list): return "" parts = [] for item in output: if not isinstance(item, dict): continue content = item.get("content") if not isinstance(content, list): continue for block in content: if not isinstance(block, dict): continue text_value = block.get("text") if isinstance(text_value, str) and text_value: parts.append(text_value) return "\\n".join(parts)[:240] def request_log_evidence(request_id): proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=5m", "--tail=800"]) stdout = proc.stdout.decode("utf-8", errors="replace") lines = [line for line in stdout.splitlines() if request_id in line] failovers = [] final = None for line in lines: json_start = line.find("{") if json_start < 0: continue try: item = json.loads(line[json_start:]) except Exception: continue if "upstream_failover_switching" in line: failovers.append({ "accountId": item.get("account_id"), "upstreamStatus": item.get("upstream_status"), "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches"), }) if "http request completed" in line: final = { "accountId": item.get("account_id"), "statusCode": item.get("status_code"), "latencyMs": item.get("latency_ms"), "path": item.get("path"), } return { "requestId": request_id, "matchedLogLineCount": len(lines), "failovers": failovers, "final": final, "logsExitCode": proc.returncode, "logsStderr": text(proc.stderr, 1000), } def recent_compact_gateway_evidence(): proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"]) stdout = proc.stdout.decode("utf-8", errors="replace") failures = [] successes = [] failovers = [] final_errors = [] context_canceled = [] for line in stdout.splitlines(): if "/responses/compact" not in line and "remote_compact" not in line: continue json_start = line.find("{") if json_start < 0: continue try: item = json.loads(line[json_start:]) except Exception: continue path = item.get("path") entry = { "requestId": item.get("request_id"), "clientRequestId": item.get("client_request_id"), "accountId": item.get("account_id"), "statusCode": item.get("status_code"), "upstreamStatus": item.get("upstream_status"), "latencyMs": item.get("latency_ms"), "path": path, } if "codex.remote_compact.failed" in line: failures.append(entry) elif "codex.remote_compact.succeeded" in line: successes.append(entry) elif "upstream_failover_switching" in line and path == "/responses/compact": failovers.append({ **entry, "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches"), }) elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400: final_errors.append(entry) if "context canceled" in line and path == "/responses/compact": context_canceled.append(entry) return { "ok": True, "degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0, "window": "6h", "tailLines": 2500, "failureCount": len(failures), "successCount": len(successes), "failoverCount": len(failovers), "finalErrorCount": len(final_errors), "contextCanceledCount": len(context_canceled), "recentFailures": failures[-5:], "recentSuccesses": successes[-5:], "recentFailovers": failovers[-8:], "recentFinalErrors": final_errors[-5:], "recentContextCanceled": context_canceled[-5:], "logsExitCode": proc.returncode, "logsStderr": text(proc.stderr, 1000), "valuesPrinted": False, } def is_ignored_probe_noise(entry): for value in (entry.get("requestId"), entry.get("clientRequestId")): if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"): return True return False def filter_ignored_probe_noise(items): return [item for item in items if not is_ignored_probe_noise(item)] def ignored_probe_noise(items, section): result = [] for item in items: if is_ignored_probe_noise(item): probe = dict(item) probe["section"] = section result.append(probe) return result def group_by_request_id(items): grouped = {} for item in items: request_id = item.get("requestId") if not isinstance(request_id, str) or not request_id: continue grouped.setdefault(request_id, []).append(item) return grouped def failover_budget_exhausted_evidence(failovers, final_errors): final_by_request = {} for item in final_errors: request_id = item.get("requestId") if isinstance(request_id, str) and request_id: final_by_request[request_id] = item exhausted = [] for request_id, request_failovers in group_by_request_id(failovers).items(): final = final_by_request.get(request_id) if not final: continue last = request_failovers[-1] switch_count = last.get("switchCount") max_switches = last.get("maxSwitches") final_status = final.get("statusCode") if ( isinstance(switch_count, int) and isinstance(max_switches, int) and max_switches > 0 and switch_count >= max_switches and isinstance(final_status, int) and final_status >= 500 ): exhausted.append({ "requestId": request_id, "clientRequestId": final.get("clientRequestId") or last.get("clientRequestId"), "path": final.get("path") or last.get("path"), "finalAccountId": final.get("accountId"), "finalStatusCode": final_status, "switchCount": switch_count, "maxSwitches": max_switches, "lastFailoverAccountId": last.get("accountId"), "lastUpstreamStatus": last.get("upstreamStatus"), }) return exhausted def recent_responses_gateway_evidence(): proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"]) stdout = proc.stdout.decode("utf-8", errors="replace") failovers = [] forward_failures = [] final_errors = [] context_canceled = [] slow_final_errors = [] for line in stdout.splitlines(): if '"/responses"' not in line and '"/v1/responses"' not in line: continue json_start = line.find("{") if json_start < 0: continue try: item = json.loads(line[json_start:]) except Exception: continue path = item.get("path") if path not in ("/responses", "/v1/responses"): continue entry = { "requestId": item.get("request_id"), "clientRequestId": item.get("client_request_id"), "accountId": item.get("account_id"), "statusCode": item.get("status_code"), "upstreamStatus": item.get("upstream_status"), "latencyMs": item.get("latency_ms"), "path": path, } if "upstream_failover_switching" in line: failovers.append({ **entry, "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches"), }) elif "openai.forward_failed" in line: forward_failures.append({ **entry, "errorPreview": text(str(item.get("error") or ""), 500), "fallbackErrorResponseWritten": item.get("fallback_error_response_written"), "upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"), }) elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400: final_errors.append(entry) latency_ms = item.get("latency_ms") if isinstance(latency_ms, int) and latency_ms >= 30000: slow_final_errors.append(entry) if "context canceled" in line: context_canceled.append(entry) visible_failovers = filter_ignored_probe_noise(failovers) visible_forward_failures = filter_ignored_probe_noise(forward_failures) visible_final_errors = filter_ignored_probe_noise(final_errors) visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors) visible_context_canceled = filter_ignored_probe_noise(context_canceled) failover_budget_exhausted = failover_budget_exhausted_evidence(visible_failovers, visible_final_errors) probe_noise = ( ignored_probe_noise(failovers, "failovers") + ignored_probe_noise(forward_failures, "forwardFailures") + ignored_probe_noise(final_errors, "finalErrors") + ignored_probe_noise(slow_final_errors, "slowFinalErrors") + ignored_probe_noise(context_canceled, "contextCanceled") ) return { "ok": True, "degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0, "window": "6h", "tailLines": 2500, "failoverCount": len(visible_failovers), "forwardFailureCount": len(visible_forward_failures), "finalErrorCount": len(visible_final_errors), "slowFinalErrorCount": len(visible_slow_final_errors), "contextCanceledCount": len(visible_context_canceled), "ignoredProbeNoiseCount": len(probe_noise), "failoverBudgetExhausted": failover_budget_exhausted[-8:], "rawCounts": { "failoverCount": len(failovers), "forwardFailureCount": len(forward_failures), "finalErrorCount": len(final_errors), "slowFinalErrorCount": len(slow_final_errors), "contextCanceledCount": len(context_canceled), }, "recentFailovers": visible_failovers[-8:], "recentForwardFailures": visible_forward_failures[-8:], "recentFinalErrors": visible_final_errors[-8:], "recentSlowFinalErrors": visible_slow_final_errors[-5:], "recentContextCanceled": visible_context_canceled[-5:], "recentProbeNoise": probe_noise[-5:], "logsExitCode": proc.returncode, "logsStderr": text(proc.stderr, 1000), "valuesPrinted": False, } def validate_gateway_responses(api_key): request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000)) payload = { "model": RESPONSES_SMOKE_MODEL, "input": "Reply exactly: unidesk-sub2api-validate-ok", "stream": False, "store": False, "max_output_tokens": 32, } body = json.dumps(payload, separators=(",", ":")).encode("utf-8") script = r''' set -eu token="$1" request_id="$2" tmp="$(mktemp)" trap 'rm -f "$tmp"' EXIT cat > "$tmp" curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \ -H "Authorization: Bearer $token" \ -H 'Content-Type: application/json' \ -H "X-Request-ID: $request_id" \ -H "OpenAI-Client-Request-ID: $request_id" \ --data-binary @"$tmp" \ http://127.0.0.1:8080/v1/responses ''' started = time.time() proc = run([ "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", "sh", "-c", script, "sh", api_key, request_id, ], body) resp = parse_curl_output(proc) evidence = request_log_evidence(request_id) parsed = resp.get("json") failover_count = len(evidence.get("failovers") or []) return { "ok": resp.get("ok"), "degraded": failover_count > 0, "outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"), "httpStatus": resp.get("httpStatus"), "transportExitCode": resp.get("transportExitCode"), "method": "POST /v1/responses", "model": RESPONSES_SMOKE_MODEL, "requestId": request_id, "durationMs": int((time.time() - started) * 1000), "outputTextPreview": response_output_preview(parsed), "bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800), "stderr": resp.get("stderr", ""), "evidence": evidence, "valuesPrinted": False, } def bool_value(value): if isinstance(value, bool): return value if isinstance(value, str): if value.lower() == "true": return True if value.lower() == "false": return False return False def normalize_temp_unschedulable_credentials(credentials): if not isinstance(credentials, dict): credentials = {} enabled = bool_value(credentials.get("temp_unschedulable_enabled")) raw_rules = credentials.get("temp_unschedulable_rules") if isinstance(raw_rules, str): try: raw_rules = json.loads(raw_rules) except json.JSONDecodeError: raw_rules = [] rules = [] if isinstance(raw_rules, list): for rule in raw_rules: if not isinstance(rule, dict): continue error_code = rule.get("error_code", rule.get("statusCode")) duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes")) keywords = rule.get("keywords") if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list): continue clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()] if not clean_keywords: continue description = rule.get("description") if isinstance(rule.get("description"), str) else "" rules.append({ "error_code": error_code, "keywords": clean_keywords, "duration_minutes": duration_minutes, "description": description, }) return { "enabled": enabled, "rules": rules, } def summarize_temp_unschedulable_rules(rules): return [{ "errorCode": rule.get("error_code"), "durationMinutes": rule.get("duration_minutes"), "keywordCount": len(rule.get("keywords") or []), "keywords": rule.get("keywords") or [], "hasDescription": bool(rule.get("description")), } for rule in rules] def success_body_reclassification_requirement(): for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) if expected["enabled"] is not True: continue for rule in expected["rules"]: error_code = rule.get("error_code") keywords = rule.get("keywords") or [] if isinstance(error_code, int) and 200 <= error_code < 300 and keywords: return { "required": True, "sourceAccountName": name, "statusCode": error_code, "keywords": keywords, "representativeKeyword": keywords[0], "durationMinutes": rule.get("duration_minutes"), } return { "required": False, "sourceAccountName": None, "statusCode": None, "keywords": [], "representativeKeyword": None, "durationMinutes": None, } def model_routing_400_failover_requirement(): preferred = ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"] for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) if expected["enabled"] is not True: continue for rule in expected["rules"]: error_code = rule.get("error_code") keywords = rule.get("keywords") or [] if error_code != 400 or not keywords: continue representative_keyword = next((item for item in preferred if item in keywords), keywords[0]) return { "required": True, "sourceAccountName": name, "statusCode": error_code, "keywords": keywords, "representativeKeyword": representative_keyword, "durationMinutes": rule.get("duration_minutes"), } return { "required": False, "sourceAccountName": None, "statusCode": None, "keywords": [], "representativeKeyword": None, "durationMinutes": None, } def delete_probe_resource(token, method, path, label): if not path: return {"label": label, "ok": True, "skipped": True} resp = curl_api(method, path, bearer=token) ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410) return { "label": label, "ok": ok, "method": method, "path": path, "httpStatus": resp.get("httpStatus"), "transportExitCode": resp.get("transportExitCode"), "bodyPreview": "" if ok else text(resp.get("body", ""), 500), "valuesPrinted": False, } def validate_runtime_capabilities(token): success_body = success_body_reclassification_requirement() model_routing_400 = model_routing_400_failover_requirement() return { "ok": success_body.get("required") is not True and model_routing_400.get("required") is True, "runtimeImage": app_pod_runtime_image(), "successBodyReclassification": { "ok": success_body.get("required") is not True, "required": success_body.get("required"), "outcome": "not-required-by-yaml" if success_body.get("required") is not True else "requires-source-review-and-real-traffic-evidence", "requirement": success_body, "valuesPrinted": False, }, "modelRouting400Failover": { "ok": model_routing_400.get("required") is True, "required": model_routing_400.get("required"), "outcome": "yaml-rules-present-runtime-observed-by-real-traffic", "requirement": model_routing_400, "evidence": "Use Sub2API source review plus validation.gatewayResponsesRecent and Artificer/real request ids for runtime proof; default validate does not create mock upstreams or temporary failover accounts.", "valuesPrinted": False, }, "valuesPrinted": False, } def app_pod_runtime_image(): try: pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}") spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else [] status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else [] spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {}) status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {}) return { "pod": APP_POD, "image": spec.get("image"), "imageID": status.get("imageID"), "ready": status.get("ready"), "restartCount": status.get("restartCount"), } except Exception as exc: return {"pod": APP_POD, "error": str(exc)} def get_account_detail(token, account): account_id = account.get("id") if isinstance(account, dict) else None if account_id is None: return account data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}") return data if isinstance(data, dict) else account def account_temp_unschedulable_status(token): accounts = list_accounts(token) by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} items = [] missing = [] mismatched = [] enabled_names = [] for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE): expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name]) account = by_name.get(name) if account is None: missing.append(name) items.append({ "accountName": name, "accountId": None, "expectedEnabled": expected["enabled"], "runtimeEnabled": None, "expectedRuleCount": len(expected["rules"]), "runtimeRuleCount": None, "ok": False, }) continue detail = get_account_detail(token, account) credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {} runtime = normalize_temp_unschedulable_credentials(credentials) temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil") temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or "" ok = runtime == expected if expected["enabled"]: enabled_names.append(name) if not ok: mismatched.append(name) items.append({ "accountName": name, "accountId": account.get("id"), "expectedEnabled": expected["enabled"], "runtimeEnabled": runtime["enabled"], "expectedRuleCount": len(expected["rules"]), "runtimeRuleCount": len(runtime["rules"]), "expectedRules": summarize_temp_unschedulable_rules(expected["rules"]), "runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]), "status": account.get("status"), "schedulable": account.get("schedulable"), "tempUnschedulableUntil": temp_until, "tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "", "tempUnschedulableSet": temp_until is not None or bool(temp_reason), "ok": ok, }) return { "ok": len(missing) == 0 and len(mismatched) == 0, "desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE), "enabled": enabled_names, "missing": missing, "mismatched": mismatched, "items": items, "valuesPrinted": False, } def account_capacity_status(token): accounts = list_accounts(token) by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} items = [] missing = [] mismatched = [] expected_capacity_total = 0 runtime_concurrency_total = 0 schedulable_runtime_concurrency_total = 0 available_runtime_concurrency_total = 0 temp_unschedulable_runtime_concurrency_total = 0 for name in sorted(EXPECTED_ACCOUNT_CAPACITIES): expected = int(EXPECTED_ACCOUNT_CAPACITIES[name]) expected_capacity_total += expected account = by_name.get(name) if account is None: missing.append(name) items.append({ "accountName": name, "accountId": None, "expectedCapacity": expected, "runtimeConcurrency": None, "ok": False, }) continue runtime = account.get("concurrency") runtime_int = runtime if isinstance(runtime, int) else 0 runtime_concurrency_total += runtime_int if account.get("schedulable") is True: runtime_status = account.get("status") if runtime_status is None or runtime_status == "active": runtime_status_ok = True else: runtime_status_ok = False if runtime_status_ok: schedulable_runtime_concurrency_total += runtime_int temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil") temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason") if temp_until is None and not bool(temp_reason): available_runtime_concurrency_total += runtime_int else: temp_unschedulable_runtime_concurrency_total += runtime_int ok = runtime == expected if not ok: mismatched.append(name) items.append({ "accountName": name, "accountId": account.get("id"), "expectedCapacity": expected, "runtimeConcurrency": runtime, "priority": account.get("priority"), "status": account.get("status"), "schedulable": account.get("schedulable"), "ok": ok, }) return { "ok": len(missing) == 0 and len(mismatched) == 0, "defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY, "desired": len(EXPECTED_ACCOUNT_CAPACITIES), "totals": { "expectedCapacityTotal": expected_capacity_total, "runtimeConcurrencyTotal": runtime_concurrency_total, "schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total, "availableRuntimeConcurrencyTotal": available_runtime_concurrency_total, "tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total, "minOwnerConcurrency": MIN_OWNER_CONCURRENCY, "minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE, }, "missing": missing, "mismatched": mismatched, "items": items, "valuesPrinted": False, } def account_load_factor_status(token): accounts = list_accounts(token) by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} items = [] missing = [] mismatched = [] for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS): expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name]) account = by_name.get(name) if account is None: missing.append(name) items.append({ "accountName": name, "accountId": None, "expectedLoadFactor": expected, "runtimeLoadFactor": None, "ok": False, }) continue runtime_raw = account.get("load_factor") if runtime_raw is None: detail = get_account_detail(token, account) runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None try: runtime = int(runtime_raw) except Exception: runtime = runtime_raw ok = runtime == expected if not ok: mismatched.append(name) items.append({ "accountName": name, "accountId": account.get("id"), "expectedLoadFactor": expected, "runtimeLoadFactor": runtime, "priority": account.get("priority"), "status": account.get("status"), "schedulable": account.get("schedulable"), "ok": ok, }) return { "ok": len(missing) == 0 and len(mismatched) == 0, "defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR, "desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS), "missing": missing, "mismatched": mismatched, "items": items, "valuesPrinted": False, } def account_ws_v2_status(token): accounts = list_accounts(token) by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} items = [] missing = [] mismatched = [] enabled_names = [] unschedulable_enabled = [] schedulable_enabled = [] for name in sorted(EXPECTED_ACCOUNT_WS_MODES): expected_mode = EXPECTED_ACCOUNT_WS_MODES[name] expected_enabled = expected_mode not in (None, "", "off") account = by_name.get(name) if account is None: missing.append(name) items.append({ "accountName": name, "accountId": None, "expectedMode": expected_mode, "expectedEnabled": expected_enabled, "runtimeMode": None, "runtimeEnabled": None, "ok": False, }) continue extra = account.get("extra") if isinstance(account.get("extra"), dict) else {} runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode") runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled") schedulable = account.get("schedulable") if expected_mode == "off": ok = runtime_mode == "off" and runtime_enabled is False elif expected_mode is None: ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False) else: ok = runtime_mode == expected_mode and runtime_enabled is True if not ok: mismatched.append(name) if expected_enabled: enabled_names.append(name) if schedulable is True: schedulable_enabled.append(name) else: unschedulable_enabled.append(name) items.append({ "accountName": name, "accountId": account.get("id"), "expectedMode": expected_mode, "expectedEnabled": expected_enabled, "runtimeMode": runtime_mode, "runtimeEnabled": runtime_enabled, "status": account.get("status"), "schedulable": schedulable, "ok": ok and ((not expected_enabled) or schedulable is True), }) availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0 return { "ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok, "desired": len(EXPECTED_ACCOUNT_WS_MODES), "enabled": enabled_names, "schedulableEnabled": schedulable_enabled, "unschedulableEnabled": unschedulable_enabled, "missing": missing, "mismatched": mismatched, "items": items, "valuesPrinted": False, } def api_key_preview(api_key): if len(api_key) <= 14: return "***" return api_key[:10] + "..." + api_key[-4:] def run_sync(): global MANUAL_ACCOUNT_PROTECTIONS payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) manual_accounts_payload = payload.get("manualAccounts") if isinstance(payload.get("manualAccounts"), dict) else {} resolved_manual_protections = manual_accounts_payload.get("protected") if not isinstance(resolved_manual_protections, list): raise RuntimeError("sync payload has no manualAccounts.protected binding plan") MANUAL_ACCOUNT_PROTECTIONS = resolved_manual_protections profiles = payload.get("profiles") or [] prune_removed = bool(payload.get("pruneRemoved")) sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {} if not profiles: raise RuntimeError("sync payload has no profiles") admin_email, token, admin_compliance = login() group, group_action = ensure_group(token) group_id = group.get("id") if isinstance(group, dict) else None if group_id is None: raise RuntimeError("pool group id missing after ensure") existing_accounts = list_accounts(token) planned_account_results = planned_sentinel_account_results(profiles, existing_accounts) sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True) if sentinel_quality_prepare.get("ok") is not True: raise RuntimeError("prepare sentinel pending probe failed: " + json.dumps(sentinel_quality_prepare, ensure_ascii=False)) protected_frozen_names = active_sentinel_quarantine_names() account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed, protected_frozen_names, existing_accounts) manual_account_proxy_bindings = ensure_manual_account_proxy_bindings(token) manual_account_group_bindings = ensure_manual_account_group_bindings(token, group_id) manual_account_protections = manual_account_protection_status(token, group_id) capacity_status = account_capacity_status(token) load_factor_status = account_load_factor_status(token) ws_v2_status = account_ws_v2_status(token) temp_unschedulable_status = account_temp_unschedulable_status(token) api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id) api_key_result = ensure_sub2api_api_key(token, api_key, group_id) owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"]) owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"]) gateway = validate_gateway(api_key) responses_smoke = validate_gateway_responses(api_key) compact_evidence = recent_compact_gateway_evidence() responses_evidence = recent_responses_gateway_evidence() runtime_capabilities = validate_runtime_capabilities(token) sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest")) sentinel_quality = ensure_sentinel_state_for_sync(account_results) sentinel_reassert = reassert_sentinel_freezes_after_sync(token) return { "ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_proxy_bindings.get("ok") is True and manual_account_group_bindings.get("ok") is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and sentinel_quality_prepare.get("ok") is True and sentinel_quality.get("ok") is True and sentinel_reassert.get("ok") is True and runtime_capabilities.get("ok") is True, "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True, "mode": "sync", "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": APP_POD, "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, "pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"}, "accounts": { "desired": len(profiles), "created": sum(1 for item in account_results if item["action"] == "created"), "updated": sum(1 for item in account_results if item["action"] == "updated"), "pruned": len(pruned_account_results), "pruneMode": "explicit" if prune_removed else "disabled-by-default", "items": account_results, "prunedItems": pruned_account_results, "processControl": {"schedulableRestore": "sentinel marker probe only; sync does not restore schedulable for existing accounts", "durableConfig": False}, "valuesPrinted": False, }, "manualAccounts": {**manual_account_protections, "proxySync": manual_account_proxy_bindings, "groupSync": manual_account_group_bindings}, "capacity": capacity_status, "loadFactor": load_factor_status, "webSocketsV2": ws_v2_status, "tempUnschedulable": temp_unschedulable_status, "apiKey": { "name": POOL_API_KEY_NAME, "secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}", "secretAction": secret_action, "secretApply": secret_apply_stdout, "sub2apiAction": api_key_result["action"], "sub2apiId": api_key_result["id"], "groupId": api_key_result["groupId"], "userId": api_key_result["userId"], "keyPreview": api_key_preview(api_key), "valuesPrinted": False, }, "ownerBalance": owner_balance, "ownerConcurrency": owner_concurrency, "sentinel": {**sentinel, "qualityGatePrepare": sentinel_quality_prepare, "qualityGate": sentinel_quality, "freezeReassert": sentinel_reassert}, "runtimeCapabilities": runtime_capabilities, "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, } def run_validate(): api_key = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY) if not api_key: raise RuntimeError(f"{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY} missing") admin_email, token, admin_compliance = login() key_item = next((item for item in list_user_keys(token) if item.get("key") == api_key), None) owner_balance = None owner_concurrency = None if key_item is not None and key_item.get("user_id") is not None: owner_balance = ensure_pool_owner_balance(token, key_item["user_id"]) owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"]) capacity_status = account_capacity_status(token) load_factor_status = account_load_factor_status(token) ws_v2_status = account_ws_v2_status(token) temp_unschedulable_status = account_temp_unschedulable_status(token) pool_group_id = key_item.get("group_id") if isinstance(key_item, dict) else None manual_account_protections = manual_account_protection_status(token, pool_group_id) gateway = validate_gateway(api_key) responses_smoke = validate_gateway_responses(api_key) compact_evidence = recent_compact_gateway_evidence() responses_evidence = recent_responses_gateway_evidence() runtime_capabilities = validate_runtime_capabilities(token) sentinel = sentinel_runtime_status() return { "ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and runtime_capabilities.get("ok") is True, "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True, "mode": "validate", "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": APP_POD, "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, "apiKey": { "secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}", "sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None, "userId": key_item.get("user_id") if isinstance(key_item, dict) else None, "groupId": key_item.get("group_id") if isinstance(key_item, dict) else None, "keyPreview": api_key_preview(api_key), "valuesPrinted": False, }, "ownerBalance": owner_balance, "ownerConcurrency": owner_concurrency, "capacity": capacity_status, "loadFactor": load_factor_status, "webSocketsV2": ws_v2_status, "tempUnschedulable": temp_unschedulable_status, "manualAccounts": manual_account_protections, "sentinel": sentinel, "runtimeCapabilities": runtime_capabilities, "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, } def parse_log_line(line): json_start = line.find("{") if json_start < 0: return None prefix = line[:json_start].rstrip() try: item = json.loads(line[json_start:]) except Exception: return None if not isinstance(item, dict): return None at = None parts = prefix.split() if parts: at = parts[0] message = "" if len(parts) >= 4: message = " ".join(parts[3:]) elif len(parts) >= 3: message = parts[2] elif len(parts) >= 1: message = parts[-1] item["_at"] = at item["_message"] = message item["_line"] = line return item def log_time_epoch(item): at = item.get("_at") if isinstance(item, dict) else None if not isinstance(at, str) or not at: return None try: return datetime.strptime(at, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp() except Exception: try: return datetime.fromisoformat(at.replace("Z", "+00:00")).timestamp() except Exception: return None def event_base(item, account_names_by_id): account_id = item.get("account_id") if isinstance(account_id, str) and account_id.isdigit(): account_id = int(account_id) account_name = account_names_by_id.get(account_id) return { "at": item.get("_at"), "message": item.get("_message"), "requestId": item.get("request_id"), "clientRequestId": item.get("client_request_id"), "path": item.get("path"), "method": item.get("method"), "model": item.get("model"), "accountId": account_id, "accountName": account_name, "statusCode": item.get("status_code"), "upstreamStatus": item.get("upstream_status"), "latencyMs": item.get("latency_ms"), } def classify_trace_event(item, account_names_by_id): message = str(item.get("_message") or "") event = event_base(item, account_names_by_id) if "content_moderation.gateway_check_start" in message: event.update({ "type": "request-start", "stream": item.get("stream"), "bodyBytes": item.get("body_bytes"), "groupId": item.get("group_id"), "groupName": item.get("group_name"), "apiKeyName": item.get("api_key_name"), }) elif "content_moderation.gateway_check_done" in message: event.update({ "type": "gateway-check", "allowed": item.get("allowed"), "blocked": item.get("blocked"), "action": item.get("action"), }) elif "openai.upstream_failover_switching" in message: event.update({ "type": "failover", "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches"), }) elif "openai.account_select_failed" in message: event.update({ "type": "select-failed", "error": item.get("error"), "excludedAccountCount": item.get("excluded_account_count"), }) elif "account_upstream_error" in message: event.update({ "type": "upstream-error", "error": item.get("error"), }) elif "account_temp_unschedulable" in message: event.update({ "type": "temp-unschedulable", "until": item.get("until") or item.get("temp_unschedulable_until"), "ruleIndex": item.get("rule_index"), "matchedKeyword": item.get("matched_keyword"), "reason": item.get("reason") or item.get("error"), }) elif "http request completed" in message: event.update({ "type": "final", "clientIp": item.get("client_ip"), "protocol": item.get("protocol"), "platform": item.get("platform"), "completedAt": item.get("completed_at"), }) elif "admin account schedulable updated" in message or "account schedulable updated" in message or "/schedulable" in str(item.get("path") or ""): event.update({ "type": "admin-schedulable", "schedulable": item.get("schedulable"), }) else: event.update({"type": "other"}) return event def with_trace_phase(event, first_epoch, last_epoch): epoch = None at = event.get("at") if isinstance(event, dict) else None if isinstance(at, str) and at: epoch = log_time_epoch({"_at": at}) if epoch is None or first_epoch is None: phase = "unknown" elif epoch < first_epoch: phase = "before-request" elif last_epoch is not None and epoch > last_epoch: phase = "after-request" else: phase = "during-request" event["phase"] = phase return event def account_snapshot_from_runtime(token): try: accounts = list_accounts(token) except Exception as exc: return [], {"error": str(exc)} rows = [] for item in accounts: if not isinstance(item, dict): continue rows.append({ "accountId": item.get("id"), "accountName": item.get("name"), "schedulable": item.get("schedulable"), "status": item.get("status"), "concurrency": item.get("concurrency"), "priority": item.get("priority"), "tempUnschedulableUntil": item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil"), "tempUnschedulableSet": (item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil")) is not None or bool(item.get("temp_unschedulable_reason") or item.get("tempUnschedulableReason")), }) rows.sort(key=lambda row: (str(row.get("accountName") or ""), int(row.get("accountId") or 0))) return rows, None def trace_reason(events, final_event): failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] if failover_budget_exhausted(failovers, final_event): return "failover-budget-exhausted" if failovers and select_failures: return "failover-attempted-no-candidate" if failovers: return "failover-attempted" if select_failures: return "account-select-failed" if upstream_errors: return "upstream-error" if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") >= 400: return "final-http-error" if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int): return "completed" return "unknown" def failover_budget_exhausted(failovers, final_event): if not failovers or not isinstance(final_event, dict): return False last = failovers[-1] switch_count = last.get("switchCount") max_switches = last.get("maxSwitches") final_status = final_event.get("statusCode") return ( isinstance(switch_count, int) and isinstance(max_switches, int) and max_switches > 0 and switch_count >= max_switches and isinstance(final_status, int) and final_status >= 500 ) def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot): if not failover_budget_exhausted(failovers, final_event): return [] tried = set() for item in failovers: account_id = item.get("accountId") if isinstance(account_id, int): tried.add(account_id) final_account = final_event.get("accountId") if isinstance(final_event, dict) else None if isinstance(final_account, int): tried.add(final_account) result = [] for item in account_snapshot: account_id = item.get("accountId") if not isinstance(account_id, int) or account_id in tried: continue if item.get("schedulable") is True and item.get("status") == "active" and item.get("tempUnschedulableSet") is not True: result.append({ "accountId": account_id, "accountName": item.get("accountName"), "priority": item.get("priority"), "concurrency": item.get("concurrency"), }) return result def run_trace(): payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} request_id = payload.get("requestId") since = payload.get("since") or "24h" tail = int(payload.get("tail") or 20000) context_seconds = int(payload.get("contextSeconds") or 300) show_lines = bool(payload.get("showLines")) if not isinstance(request_id, str) or not request_id: raise RuntimeError("trace payload missing requestId") admin_email, token, admin_compliance = login() account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token) account_names_by_id = {} for row in account_snapshot: account_id = row.get("accountId") if isinstance(account_id, str) and account_id.isdigit(): account_id = int(account_id) if isinstance(account_id, int) and isinstance(row.get("accountName"), str): account_names_by_id[account_id] = row.get("accountName") proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) stdout = proc.stdout.decode("utf-8", errors="replace") parsed_lines = [] matched = [] for line in stdout.splitlines(): parsed = parse_log_line(line) if parsed is None: continue parsed_lines.append(parsed) if request_id in line: matched.append(parsed) first_epoch = None last_epoch = None for item in matched: epoch = log_time_epoch(item) if epoch is None: continue first_epoch = epoch if first_epoch is None else min(first_epoch, epoch) last_epoch = epoch if last_epoch is None else max(last_epoch, epoch) window_lines = [] if first_epoch is not None: start_epoch = first_epoch - context_seconds end_epoch = (last_epoch if last_epoch is not None else first_epoch) + context_seconds for item in parsed_lines: epoch = log_time_epoch(item) if epoch is not None and start_epoch <= epoch <= end_epoch: window_lines.append(item) else: window_lines = matched events = [classify_trace_event(item, account_names_by_id) for item in matched] request_start = next((item for item in events if item.get("type") == "request-start"), None) final_event = next((item for item in reversed(events) if item.get("type") == "final"), None) failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")] admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))] window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines] final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400] window_failovers = [item for item in window_events if item.get("type") == "failover"] window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) reason = trace_reason(events, final_event) if not matched: outcome = "not-found" elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: outcome = "succeeded" elif isinstance(final_event, dict): outcome = "failed" else: outcome = "incomplete" return { "ok": proc.returncode == 0 and len(matched) > 0, "mode": "trace", "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": APP_POD, "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, "requestId": request_id, "summary": { "outcome": outcome, "reason": reason, "eventCount": len(events), "matchedLineCount": len(matched), "firstAt": events[0].get("at") if events else None, "lastAt": events[-1].get("at") if events else None, }, "window": { "since": since, "tail": tail, "beforeSeconds": context_seconds, "afterSeconds": context_seconds, "lineCount": len(window_lines), }, "request": request_start or {}, "final": final_event or {}, "events": events, "failovers": failovers, "selectFailures": select_failures, "upstreamErrors": upstream_errors, "tempUnschedulable": temp_unsched, "adminSchedulable": admin_sched[-20:], "windowStats": { "matchedLines": len(matched), "eventCount": len(window_events), "finalErrorCount": len(final_errors), "failoverCount": len(window_failovers), "selectFailedCount": len(window_select_failures), "tempUnschedulableCount": len(temp_unsched), "adminSchedulableCount": len(admin_sched), }, "diagnostics": { "failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}), "untriedSchedulableAccounts": untried_schedulable_accounts, }, "accountSnapshot": account_snapshot, "accountSnapshotError": account_snapshot_error, "rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [], "showLines": show_lines, "logs": { "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000), "stdoutLineCount": len(stdout.splitlines()), }, "valuesPrinted": False, } def parse_embedded_json(stdout): if not isinstance(stdout, str) or not stdout.strip(): return None start = stdout.find("{") end = stdout.rfind("}") if start < 0 or end <= start: return None try: return json.loads(stdout[start:end + 1]) except Exception: return None def inject_env(template, name, value): spec = template.setdefault("spec", {}) containers = spec.setdefault("containers", []) if not containers: raise RuntimeError("sentinel job template has no containers") container = containers[0] env = container.setdefault("env", []) env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)] env.append({"name": name, "value": value}) def sentinel_probe_job_manifest(accounts): cronjob_name = SENTINEL_CONFIG.get("cronJobName") if not isinstance(cronjob_name, str) or not cronjob_name: raise RuntimeError("sentinel cronJobName missing from config") cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}") job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {} job_spec = copy_json(job_template.get("spec") or {}) if not isinstance(job_spec, dict) or not job_spec: raise RuntimeError("sentinel CronJob jobTemplate.spec missing") template = job_spec.get("template") if not isinstance(template, dict): raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing") inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts)) job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600) suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5)) job_name = ("sub2api-sentinel-probe-" + suffix)[:63] return job_name, { "apiVersion": "batch/v1", "kind": "Job", "metadata": { "name": job_name, "namespace": NAMESPACE, "labels": { "app.kubernetes.io/name": cronjob_name, "app.kubernetes.io/part-of": "platform-infra", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe", }, }, "spec": job_spec, } def copy_json(value): return json.loads(json.dumps(value)) def job_condition(job, cond_type): for item in ((job.get("status") or {}).get("conditions") or []): if item.get("type") == cond_type and item.get("status") == "True": return item return None def wait_sentinel_probe_job(job_name, timeout_seconds): deadline = time.time() + timeout_seconds latest = None while time.time() < deadline: latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}") if isinstance(latest, dict): complete = job_condition(latest, "Complete") failed = job_condition(latest, "Failed") if complete is not None: return "succeeded", latest, complete if failed is not None: return "failed", latest, failed time.sleep(2) return "timeout", latest, None def job_logs(job_name): proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"]) return { "exitCode": proc.returncode, "stdout": proc.stdout.decode("utf-8", errors="replace"), "stderr": text(proc.stderr, 4000), } def run_sentinel_probe(): payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} accounts = payload.get("accounts") if isinstance(payload, dict) else None if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts): raise RuntimeError("sentinel-probe payload requires non-empty accounts") job_name, manifest = sentinel_probe_job_manifest(accounts) proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest)) if proc.returncode != 0: raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}") timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240) status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds) logs = job_logs(job_name) parsed = parse_embedded_json(logs.get("stdout") or "") results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else [] requested = set(accounts) measured = {item.get("accountName") for item in results if isinstance(item, dict)} missing = sorted(name for name in requested if name not in measured) marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested) job_ok = status == "succeeded" and isinstance(parsed, dict) and parsed.get("ok") is True return { "ok": job_ok and marker_ok, "jobExecutionOk": job_ok, "markerOk": marker_ok, "mode": "sentinel-probe", "namespace": NAMESPACE, "requestedAccounts": accounts, "missingAccounts": missing, "job": { "name": job_name, "status": status, "condition": condition, "timeoutSeconds": timeout_seconds, "applyStdoutTail": text(proc.stdout, 1200), "logsExitCode": logs.get("exitCode"), "logsStderrTail": logs.get("stderr"), }, "probe": parsed, "sentinelState": sentinel_state_summary(), "valuesPrinted": False, } def run_cleanup_probes(): admin_email, token, admin_compliance = login() cleanup = cleanup_probe_resources(token) return { "ok": cleanup.get("ok") is True, "mode": "cleanup-probes", "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": APP_POD, "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, "cleanup": cleanup, "valuesPrinted": False, } try: if MODE == "sync": result = run_sync() elif MODE == "trace": result = run_trace() elif MODE == "cleanup-probes": result = run_cleanup_probes() elif MODE == "sentinel-probe": result = run_sentinel_probe() else: result = run_validate() except Exception as exc: result = { "ok": False, "mode": MODE, "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": globals().get("APP_POD"), "error": str(exc), "valuesPrinted": False, } print(json.dumps(result, ensure_ascii=False, indent=2)) sys.exit(0 if result.get("ok") else 1) PY `; } async function capture(config: UniDeskConfig, target: string, args: string[], input?: string): Promise { return await runSshCommandCapture(config, target, args, input); } type RemoteCodexPoolMode = "sync" | "validate" | "sentinel-probe" | "sentinel-image-status" | "sentinel-image-build"; async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: RemoteCodexPoolMode, script: string, target = codexPoolRuntimeTarget()): Promise { const jobName = `codex-pool-${mode}-${Date.now().toString(36)}`.slice(0, 63); const startedAtMs = Date.now(); const start = await capture(config, target.route, ["sh"], remoteJobStartScript(jobName, script)); const started = parseJsonOutput(start.stdout); if (start.exitCode !== 0 || boolField(started, "ok", false) !== true) return start; let latest: RemoteCodexPoolJobStatus | null = null; while (Date.now() - startedAtMs <= remoteJobTimeoutMs) { await sleep(remoteJobPollMs); const probe = await capture(config, target.route, ["sh"], remoteJobStatusScript(jobName)); const parsed = parseJsonOutput(probe.stdout); latest = normalizeRemoteJobStatus(parsed); process.stderr.write(`${JSON.stringify({ event: "platform-infra.sub2api.codex-pool.progress", at: new Date().toISOString(), mode, jobName, status: latest?.status ?? "unknown", exitCode: latest?.exitCode ?? null, elapsedMs: Date.now() - startedAtMs, stdoutBytes: latest?.stdoutBytes ?? null, stderrBytes: latest?.stderrBytes ?? null, })}\n`); if (probe.exitCode !== 0) { return { exitCode: probe.exitCode, stdout: probe.stdout, stderr: `${start.stderr}${probe.stderr}`, }; } if (latest?.status === "succeeded" || latest?.status === "failed") { return { exitCode: latest.exitCode ?? (latest.status === "succeeded" ? 0 : 1), stdout: latest.stdout ?? "", stderr: latest.stderr ?? "", }; } } return { exitCode: 124, stdout: latest?.stdout ?? "", stderr: [ latest?.stderr ?? "", `remote codex-pool ${mode} job ${jobName} did not finish within ${remoteJobTimeoutMs}ms`, `status command: ${codexPoolModeCommand(mode)}`, ].filter(Boolean).join("\n"), }; } function codexPoolModeCommand(mode: RemoteCodexPoolMode): string { if (mode === "sync") return "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm"; if (mode === "sentinel-probe") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account --confirm"; return "bun scripts/cli.ts platform-infra sub2api codex-pool validate"; } interface RemoteCodexPoolJobStatus { status: "running" | "succeeded" | "failed" | "unknown"; exitCode: number | null; stdout: string | null; stderr: string | null; stdoutBytes: number | null; stderrBytes: number | null; } function normalizeRemoteJobStatus(parsed: Record | null): RemoteCodexPoolJobStatus | null { if (parsed === null) return null; const status = parsed.status === "running" || parsed.status === "succeeded" || parsed.status === "failed" || parsed.status === "unknown" ? parsed.status : "unknown"; return { status, exitCode: typeof parsed.exitCode === "number" ? parsed.exitCode : null, stdout: typeof parsed.stdout === "string" ? parsed.stdout : null, stderr: typeof parsed.stderr === "string" ? parsed.stderr : null, stdoutBytes: typeof parsed.stdoutBytes === "number" ? parsed.stdoutBytes : null, stderrBytes: typeof parsed.stderrBytes === "number" ? parsed.stderrBytes : null, }; } function remoteJobStartScript(jobName: string, script: string): string { const scriptB64 = Buffer.from(script, "utf8").toString("base64"); return ` set -eu job=${shQuote(jobName)} dir=${shQuote(remoteJobDir)} mkdir -p "$dir" chmod 700 "$dir" script_path="$dir/$job.sh" stdout_path="$dir/$job.stdout" stderr_path="$dir/$job.stderr" exit_path="$dir/$job.exit" done_path="$dir/$job.done" pid_path="$dir/$job.pid" rm -f "$script_path" "$stdout_path" "$stderr_path" "$exit_path" "$done_path" "$pid_path" base64 -d > "$script_path" <<'UNIDESK_REMOTE_SCRIPT_B64' ${scriptB64} UNIDESK_REMOTE_SCRIPT_B64 chmod 700 "$script_path" ( set +e trap '' HUP sh "$script_path" > "$stdout_path" 2> "$stderr_path" code=$? printf '%s' "$code" > "$exit_path" date -u +%Y-%m-%dT%H:%M:%SZ > "$done_path" rm -f "$script_path" ) >/dev/null 2>&1 & pid=$! printf '%s' "$pid" > "$pid_path" python3 - < limit: data = data[-limit:] return data.decode("utf-8", errors="replace") def file_size(path): try: return os.path.getsize(path) except FileNotFoundError: return 0 pid_text = read_text(pid_path) done = os.path.exists(done_path) exit_text = read_text(exit_path) exit_code = None if exit_text is not None: try: exit_code = int(exit_text.strip()) except ValueError: exit_code = None running = False if pid_text is not None and not done: try: os.kill(int(pid_text.strip()), 0) running = True except Exception: running = False status = "succeeded" if done and exit_code == 0 else "failed" if done else "running" if running else "unknown" print(json.dumps({ "ok": True, "jobName": job, "status": status, "pid": int(pid_text.strip()) if pid_text and pid_text.strip().isdigit() else None, "exitCode": exit_code, "stdoutBytes": file_size(stdout_path), "stderrBytes": file_size(stderr_path), "stdout": read_text(stdout_path, 500000) if done else read_text(stdout_path, 12000), "stderr": read_text(stderr_path, 120000) if done else read_text(stderr_path, 12000), "doneAt": read_text(done_path), }, ensure_ascii=False)) PY `; } function shQuote(value: string): string { return `'${value.replace(/'/gu, `'\\''`)}'`; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function parseJsonOutput(stdout: string): Record | null { const trimmed = stdout.trim(); if (trimmed.length === 0) return null; const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start === -1 || end === -1 || end <= start) return null; try { const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } } function boolField(value: Record | null, key: string, defaultValue: boolean): boolean { if (value === null) return defaultValue; const field = value[key]; return typeof field === "boolean" ? field : defaultValue; } function compactCapture(result: SshCaptureResult, options: { full?: boolean } = {}): Record { const full = options.full ?? false; return { exitCode: result.exitCode, stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), stderrBytes: Buffer.byteLength(result.stderr, "utf8"), stdoutTail: full || result.exitCode !== 0 ? result.stdout.slice(-8000) : "", stderrTail: full || result.exitCode !== 0 ? result.stderr.slice(-4000) : "", }; }