Codex
7458 lines
324 KiB
TypeScript
7458 lines
324 KiB
TypeScript
import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
import { homedir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { UniDeskConfig } from "./config";
|
|
import { rootPath } from "./config";
|
|
import type { RenderedCliResult } from "./output";
|
|
import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "./platform-infra-public-service";
|
|
import { shortSha256Fingerprint } from "./platform-infra-ops-library";
|
|
import {
|
|
codexPoolSentinelSummary,
|
|
codexPoolSentinelRuntimeImage,
|
|
readCodexPoolSentinelConfig,
|
|
renderCodexPoolSentinelManifest,
|
|
type CodexPoolSentinelConfig,
|
|
type CodexPoolSentinelProfileSecret,
|
|
} from "./platform-infra-sub2api-codex-sentinel";
|
|
import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "./secrets";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
|
|
|
const serviceName = "sub2api";
|
|
const fieldManager = "unidesk-platform-infra";
|
|
const sub2apiConfigPath = rootPath("config", "platform-infra", "sub2api.yaml");
|
|
const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml");
|
|
const sentinelImageDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile");
|
|
const remoteJobDir = "/tmp/unidesk-platform-infra-sub2api-codex-pool";
|
|
const remoteJobTimeoutMs = 15 * 60_000;
|
|
const remoteJobPollMs = 5_000;
|
|
|
|
interface DisclosureOptions {
|
|
full: boolean;
|
|
raw: boolean;
|
|
targetId: string;
|
|
}
|
|
|
|
interface SyncOptions extends DisclosureOptions {
|
|
confirm: boolean;
|
|
pruneRemoved: boolean;
|
|
}
|
|
|
|
interface ConfirmOptions extends DisclosureOptions {
|
|
confirm: boolean;
|
|
}
|
|
|
|
interface SentinelProbeOptions extends ConfirmOptions {
|
|
accounts: string[];
|
|
}
|
|
|
|
interface SentinelReportOptions extends DisclosureOptions {
|
|
events: number;
|
|
}
|
|
|
|
interface TraceOptions extends DisclosureOptions {
|
|
requestId: string | null;
|
|
since: string;
|
|
tail: number;
|
|
contextSeconds: number;
|
|
showLines: boolean;
|
|
}
|
|
|
|
interface SentinelImageOptions extends DisclosureOptions {
|
|
action: "status" | "build";
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface CodexPoolRuntimeTarget {
|
|
id: string;
|
|
route: string;
|
|
namespace: string;
|
|
serviceName: string;
|
|
serviceDns: string;
|
|
publicBaseUrl: string | null;
|
|
publicExposure: PublicServiceExposure | null;
|
|
appSecretName: string;
|
|
secretsRoot: string;
|
|
sentinelImageBuild: {
|
|
baseImageCachePolicy: "pull" | "local-if-present";
|
|
noProxy: string;
|
|
};
|
|
egressProxy: {
|
|
enabled: boolean;
|
|
applyToSentinel: boolean;
|
|
serviceName: string;
|
|
listenPort: number;
|
|
httpProxy: string;
|
|
noProxy: string;
|
|
} | null;
|
|
}
|
|
|
|
interface CodexProfile {
|
|
profile: string;
|
|
accountName: string;
|
|
configFile: string;
|
|
authFile: string;
|
|
provider: string;
|
|
baseUrl: string;
|
|
wireApi: string | null;
|
|
model: string | null;
|
|
envKey: string | null;
|
|
apiKey: string | null;
|
|
apiKeySource: "auth-json" | "env" | null;
|
|
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
|
|
upstreamUserAgent: string | null;
|
|
trustUpstream: boolean;
|
|
sentinelProtect: CodexSentinelProtectPolicy;
|
|
priority: number;
|
|
capacity: number;
|
|
loadFactor: number;
|
|
tempUnschedulable: CodexTempUnschedulablePolicy;
|
|
authOpenAIKeyShape: string;
|
|
ok: boolean;
|
|
error: string | null;
|
|
}
|
|
|
|
type OpenAIResponsesWebSocketsV2Mode = "off" | "ctx_pool" | "passthrough";
|
|
|
|
export interface CodexTempUnschedulableRule {
|
|
statusCode: number;
|
|
keywords: string[];
|
|
durationMinutes: number;
|
|
description: string | null;
|
|
}
|
|
|
|
export interface CodexTempUnschedulablePolicy {
|
|
enabled: boolean;
|
|
rules: CodexTempUnschedulableRule[];
|
|
}
|
|
|
|
interface CodexPoolConfig {
|
|
version: number;
|
|
kind: string;
|
|
metadata: {
|
|
id: string;
|
|
owner: string;
|
|
relatedIssues: number[];
|
|
};
|
|
groupName: string;
|
|
groupDescription: string;
|
|
apiKeyName: string;
|
|
apiKeySecretName: string;
|
|
apiKeySecretKey: string;
|
|
adminEmailDefault: string;
|
|
minOwnerBalanceUsd: number;
|
|
minOwnerConcurrency: number;
|
|
minOwnerConcurrencySource: "auto" | "yaml";
|
|
defaultAccountPriority: number;
|
|
defaultAccountCapacity: number;
|
|
defaultAccountLoadFactor: number;
|
|
defaultTempUnschedulable: CodexTempUnschedulablePolicy;
|
|
defaultSentinelProtect: CodexSentinelProtectPolicy;
|
|
profiles: CodexPoolProfileConfig[];
|
|
manualAccounts: CodexPoolManualAccountsConfig;
|
|
publicExposure: CodexPoolPublicExposureConfig;
|
|
localCodex: CodexPoolLocalCodexConfig;
|
|
sentinel: CodexPoolSentinelConfig;
|
|
}
|
|
|
|
interface CodexPoolManualAccountsConfig {
|
|
bindingSources: CodexPoolManualBindingSourcesConfig;
|
|
protected: CodexPoolManualAccountProtection[];
|
|
}
|
|
|
|
type CodexPoolManualBindingKind = "proxy" | "group";
|
|
type CodexPoolManualBindingProvider = "target-egress-proxy" | "pool-group";
|
|
|
|
interface CodexPoolManualBindingSource {
|
|
id: string;
|
|
enabled: boolean;
|
|
kind: CodexPoolManualBindingKind;
|
|
provider: CodexPoolManualBindingProvider;
|
|
description: string | null;
|
|
}
|
|
|
|
interface CodexPoolManualBindingSourcesConfig {
|
|
items: CodexPoolManualBindingSource[];
|
|
byId: Record<string, CodexPoolManualBindingSource>;
|
|
}
|
|
|
|
interface CodexPoolManualAccountProxyBinding {
|
|
enabled: boolean;
|
|
source: string;
|
|
proxyName: string;
|
|
}
|
|
|
|
interface CodexPoolManualAccountGroupBinding {
|
|
enabled: boolean;
|
|
source: string;
|
|
}
|
|
|
|
interface CodexPoolManualAccountProtection {
|
|
accountName: string;
|
|
reason: string | null;
|
|
proxyBinding: CodexPoolManualAccountProxyBinding | null;
|
|
groupBinding: CodexPoolManualAccountGroupBinding | null;
|
|
}
|
|
|
|
interface CodexPoolProfileConfig {
|
|
profile: string;
|
|
accountName: string | null;
|
|
configFile: string;
|
|
authFile: string;
|
|
fallbackConfigFile: string | null;
|
|
fallbackAuthFile: string | null;
|
|
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
|
|
upstreamUserAgent: string | null;
|
|
trustUpstream: boolean;
|
|
sentinelProtect: CodexSentinelProtectPolicy;
|
|
priority: number;
|
|
capacity: number | null;
|
|
loadFactor: number | null;
|
|
tempUnschedulable: CodexTempUnschedulablePolicy;
|
|
}
|
|
|
|
export interface CodexSentinelProtectPolicy {
|
|
enabled: boolean;
|
|
consecutiveFailures: number;
|
|
initialRetryDelaySeconds: number;
|
|
maxRetryDelaySeconds: number;
|
|
backoffMultiplier: number;
|
|
}
|
|
|
|
interface CodexPoolPublicExposureConfig {
|
|
enabled: boolean;
|
|
proxyName: string;
|
|
configMapName: string;
|
|
deploymentName: string;
|
|
frpcImage: string;
|
|
serverAddr: string;
|
|
serverPort: number;
|
|
remotePort: number;
|
|
localIP: string;
|
|
localPort: number;
|
|
publicBaseUrl: string;
|
|
masterBaseUrl: string;
|
|
masterFrps: {
|
|
configPath: string;
|
|
containerName: string;
|
|
};
|
|
masterCaddy: {
|
|
enabled: boolean;
|
|
domain: string;
|
|
configPath: string;
|
|
serviceName: string;
|
|
upstreamBaseUrl: string;
|
|
responseHeaderTimeoutSeconds: number;
|
|
edgeRetry: CodexPoolCaddyEdgeRetryConfig;
|
|
};
|
|
}
|
|
|
|
interface CodexPoolCaddyEdgeRetryConfig {
|
|
enabled: boolean;
|
|
tryDurationSeconds: number;
|
|
tryIntervalMilliseconds: number;
|
|
retryMatch: {
|
|
methods: string[];
|
|
paths: string[];
|
|
};
|
|
}
|
|
|
|
interface CodexPoolLocalCodexConfig {
|
|
backupSuffix: string;
|
|
providerName: string;
|
|
wireApi: string;
|
|
modelContextWindow: number;
|
|
modelAutoCompactTokenLimit: number;
|
|
supportsWebSockets: boolean;
|
|
responsesWebSocketsV2: boolean;
|
|
responsesSmokeModel: string;
|
|
}
|
|
|
|
interface CodexLocalConsumerTomlOptions {
|
|
providerName: string;
|
|
baseUrl: string;
|
|
wireApi: string;
|
|
modelContextWindow: number;
|
|
modelAutoCompactTokenLimit: number;
|
|
supportsWebSockets: boolean;
|
|
responsesWebSocketsV2: boolean;
|
|
}
|
|
|
|
export function codexPoolHelp(): unknown {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget();
|
|
return {
|
|
command: "platform-infra sub2api codex-pool plan|sync|validate|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
|
|
output: "json, except trace and sentinel-report default to low-noise text tables",
|
|
usage: [
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id <requestId> [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report [--target D601] [--events 20|--full|--raw]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes [--target D601] --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
|
|
],
|
|
description: "Import YAML-selected ~/.codex API-key profiles into one Sub2API OpenAI pool, expose one unified API_KEY, and optionally configure this master server's ~/.codex consumer endpoint.",
|
|
target: {
|
|
default: runtimeTarget.id,
|
|
route: runtimeTarget.route,
|
|
namespace: runtimeTarget.namespace,
|
|
serviceDns: runtimeTarget.serviceDns,
|
|
configPath: codexPoolConfigPath,
|
|
poolGroupName: pool.groupName,
|
|
poolApiKeySecretName: pool.apiKeySecretName,
|
|
poolApiKeySecretKey: pool.apiKeySecretKey,
|
|
publicBaseUrl: runtimeTarget.publicBaseUrl,
|
|
publicExposureSource: `${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure`,
|
|
sentinelMonitorEnabled: pool.sentinel.monitor.enabled,
|
|
sentinelActionsEnabled: pool.sentinel.actions.enabled,
|
|
secretValuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const [action = "plan"] = args;
|
|
if (action === "plan") return codexPoolPlan(parseDisclosureOptions(args.slice(1)));
|
|
if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1)));
|
|
if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1)));
|
|
if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1)));
|
|
if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1)));
|
|
if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1)));
|
|
if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1)));
|
|
if (action === "cleanup-probes") return await codexPoolCleanupProbes(config, parseConfirmOptions(args.slice(1)));
|
|
if (action === "expose") return await codexPoolExpose(config, parseConfirmOptions(args.slice(1)));
|
|
if (action === "configure-local") return await codexPoolConfigureLocal(config, parseConfirmOptions(args.slice(1)));
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-platform-infra-sub2api-codex-pool-command",
|
|
args,
|
|
help: codexPoolHelp(),
|
|
};
|
|
}
|
|
|
|
function parseSyncOptions(args: string[]): SyncOptions {
|
|
validateOptions(args, new Set(["--confirm", "--prune-removed", "--full", "--raw", "--target"]));
|
|
const disclosure = parseDisclosureOptions(stripBooleanOptions(args, new Set(["--confirm", "--prune-removed"])));
|
|
return { ...disclosure, confirm: args.includes("--confirm"), pruneRemoved: args.includes("--prune-removed") };
|
|
}
|
|
|
|
function parseConfirmOptions(args: string[]): ConfirmOptions {
|
|
validateOptions(args, new Set(["--confirm", "--full", "--raw", "--target"]));
|
|
const disclosure = parseDisclosureOptions(stripBooleanOptions(args, new Set(["--confirm"])));
|
|
return { ...disclosure, confirm: args.includes("--confirm") };
|
|
}
|
|
|
|
function parseSentinelImageOptions(args: string[]): SentinelImageOptions {
|
|
const [actionRaw = "status", ...rest] = args;
|
|
if (actionRaw !== "status" && actionRaw !== "build") throw new Error("sentinel-image usage: status|build [--dry-run|--confirm] [--full|--raw]");
|
|
let confirm = false;
|
|
let explicitDryRun = false;
|
|
const disclosureArgs: string[] = [];
|
|
for (let index = 0; index < rest.length; index += 1) {
|
|
const arg = rest[index]!;
|
|
if (arg === "--confirm") {
|
|
confirm = true;
|
|
continue;
|
|
}
|
|
if (arg === "--dry-run") {
|
|
explicitDryRun = true;
|
|
continue;
|
|
}
|
|
if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) {
|
|
disclosureArgs.push(arg);
|
|
if (arg === "--target") {
|
|
const value = rest[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
|
disclosureArgs.push(value);
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
if (confirm && explicitDryRun) throw new Error("sentinel-image accepts only one of --confirm or --dry-run");
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
return {
|
|
...disclosure,
|
|
action: actionRaw,
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
};
|
|
}
|
|
|
|
function parseSentinelProbeOptions(args: string[]): SentinelProbeOptions {
|
|
const accounts: string[] = [];
|
|
const disclosureArgs: string[] = [];
|
|
let confirm = false;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--confirm") {
|
|
confirm = true;
|
|
continue;
|
|
}
|
|
if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) {
|
|
disclosureArgs.push(arg);
|
|
if (arg === "--target") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
|
disclosureArgs.push(value);
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg === "--account") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--account requires an account name");
|
|
accounts.push(...splitAccountNames(value));
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--account=")) {
|
|
accounts.push(...splitAccountNames(arg.slice("--account=".length)));
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
const uniqueAccounts = [...new Set(accounts)];
|
|
if (uniqueAccounts.length === 0) throw new Error("sentinel-probe requires --account <accountName>");
|
|
for (const account of uniqueAccounts) validateSub2ApiAccountSelector(account, "--account");
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
return { ...disclosure, confirm, accounts: uniqueAccounts };
|
|
}
|
|
|
|
function parseSentinelReportOptions(args: string[]): SentinelReportOptions {
|
|
let events = 20;
|
|
let explicitEvents = false;
|
|
const disclosureArgs: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) {
|
|
disclosureArgs.push(arg);
|
|
if (arg === "--target") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
|
disclosureArgs.push(value);
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg === "--events") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--events requires a positive integer");
|
|
events = readReportEventLimit(value, "--events");
|
|
explicitEvents = true;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--events=")) {
|
|
events = readReportEventLimit(arg.slice("--events=".length), "--events");
|
|
explicitEvents = true;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
if (disclosure.full && !explicitEvents) events = 80;
|
|
return { ...disclosure, events };
|
|
}
|
|
|
|
function parseTraceOptions(args: string[]): TraceOptions {
|
|
let requestId: string | null = null;
|
|
let since = "24h";
|
|
let tail = 20_000;
|
|
let contextSeconds = 300;
|
|
let showLines = false;
|
|
const disclosureArgs: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) {
|
|
disclosureArgs.push(arg);
|
|
if (arg === "--target") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
|
disclosureArgs.push(value);
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg === "--show-lines") {
|
|
showLines = true;
|
|
continue;
|
|
}
|
|
if (arg === "--request-id" || arg === "--id") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a request id`);
|
|
requestId = value.trim();
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--request-id=")) {
|
|
requestId = arg.slice("--request-id=".length).trim();
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--id=")) {
|
|
requestId = arg.slice("--id=".length).trim();
|
|
continue;
|
|
}
|
|
if (arg === "--since") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--since requires a kubectl duration such as 24h or 90m");
|
|
since = parseKubectlDuration(value);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--since=")) {
|
|
since = parseKubectlDuration(arg.slice("--since=".length));
|
|
continue;
|
|
}
|
|
if (arg === "--tail") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--tail requires an integer");
|
|
tail = parseTraceLimit(value, "--tail", 100, 200_000);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--tail=")) {
|
|
tail = parseTraceLimit(arg.slice("--tail=".length), "--tail", 100, 200_000);
|
|
continue;
|
|
}
|
|
if (arg === "--context-seconds") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--context-seconds requires an integer");
|
|
contextSeconds = parseTraceLimit(value, "--context-seconds", 0, 3600);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--context-seconds=")) {
|
|
contextSeconds = parseTraceLimit(arg.slice("--context-seconds=".length), "--context-seconds", 0, 3600);
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
if (requestId === null || requestId.length === 0) throw new Error("trace requires --request-id <requestId>");
|
|
if (!/^[A-Za-z0-9_.:-]{8,128}$/u.test(requestId)) throw new Error("--request-id has an unsupported format");
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
return { ...disclosure, requestId, since, tail, contextSeconds, showLines };
|
|
}
|
|
|
|
function parseKubectlDuration(raw: string): string {
|
|
const value = raw.trim();
|
|
if (!/^[1-9][0-9]*(?:s|m|h)$/u.test(value)) throw new Error("--since must be a kubectl duration such as 24h, 90m, or 300s");
|
|
return value;
|
|
}
|
|
|
|
function parseTraceLimit(raw: string, option: string, min: number, max: number): number {
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < min || value > max) throw new Error(`${option} must be an integer from ${min} to ${max}`);
|
|
return value;
|
|
}
|
|
|
|
function readReportEventLimit(raw: string, option: string): number {
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 1 || value > 200) throw new Error(`${option} must be an integer from 1 to 200`);
|
|
return value;
|
|
}
|
|
|
|
function parseDisclosureOptions(args: string[]): DisclosureOptions {
|
|
validateOptions(args, new Set(["--full", "--raw", "--target"]));
|
|
const raw = args.includes("--raw");
|
|
return { full: raw || args.includes("--full"), raw, targetId: parseTargetId(args) };
|
|
}
|
|
|
|
function parseTargetId(args: string[]): string {
|
|
let targetId: string | null = null;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--target") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
|
targetId = value;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length);
|
|
}
|
|
const resolvedTargetId = targetId ?? defaultCodexPoolRuntimeTargetId();
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(resolvedTargetId)) throw new Error("--target must be a simple target id");
|
|
return resolvedTargetId;
|
|
}
|
|
|
|
function splitAccountNames(value: string): string[] {
|
|
return value.split(",").map((item) => item.trim()).filter(Boolean);
|
|
}
|
|
|
|
function validateSub2ApiAccountSelector(value: string, option: string): void {
|
|
if (value.length === 0 || value.length > 256) throw new Error(`${option} must be a non-empty account name up to 256 characters`);
|
|
if (/[\r\n]/u.test(value)) throw new Error(`${option} must not contain newlines`);
|
|
if (!/^[^<>"'`\\]+$/u.test(value)) throw new Error(`${option} contains unsupported characters`);
|
|
}
|
|
|
|
function validateOptions(args: string[], booleanOptions: Set<string>): void {
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--target") {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--target=") && booleanOptions.has("--target")) continue;
|
|
if (booleanOptions.has(arg)) continue;
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function stripBooleanOptions(args: string[], stripped: Set<string>): string[] {
|
|
return args.filter((arg) => !stripped.has(arg));
|
|
}
|
|
|
|
interface Sub2ApiRuntimeConfig {
|
|
defaultTargetId: string;
|
|
appSecretName: string;
|
|
secretsRoot: string;
|
|
targets: Record<string, unknown>[];
|
|
}
|
|
|
|
function readSub2ApiRuntimeConfig(): Sub2ApiRuntimeConfig {
|
|
const parsed = Bun.YAML.parse(readFileSync(sub2apiConfigPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) throw new Error(`${sub2apiConfigPath} must contain a YAML object`);
|
|
const defaults = isRecord(parsed.defaults) ? parsed.defaults : null;
|
|
const defaultTargetId = defaults === null ? null : stringValue(defaults.targetId);
|
|
if (defaultTargetId === null || !/^[A-Za-z0-9._-]+$/u.test(defaultTargetId)) throw new Error(`${sub2apiConfigPath}.defaults.targetId must be a simple target id`);
|
|
const runtime = isRecord(parsed.runtime) ? parsed.runtime : null;
|
|
const database = runtime !== null && isRecord(runtime.database) ? runtime.database : null;
|
|
const appSecretName = database === null ? null : stringValue(database.secretName);
|
|
if (appSecretName === null) throw new Error(`${sub2apiConfigPath}.runtime.database.secretName is required`);
|
|
validateKubernetesName(appSecretName, `${sub2apiConfigPath}.runtime.database.secretName`, true);
|
|
const secrets = runtime !== null && isRecord(runtime.secrets) ? runtime.secrets : null;
|
|
const secretsRoot = secrets === null ? null : stringValue(secrets.root);
|
|
if (secretsRoot === null || !secretsRoot.startsWith("/")) throw new Error(`${sub2apiConfigPath}.runtime.secrets.root must be an absolute path`);
|
|
if (!Array.isArray(parsed.targets) || !parsed.targets.every(isRecord)) throw new Error(`${sub2apiConfigPath}.targets must be a list`);
|
|
return {
|
|
defaultTargetId,
|
|
appSecretName,
|
|
secretsRoot,
|
|
targets: parsed.targets,
|
|
};
|
|
}
|
|
|
|
function defaultCodexPoolRuntimeTargetId(): string {
|
|
return readSub2ApiRuntimeConfig().defaultTargetId;
|
|
}
|
|
|
|
function codexPoolRuntimeTarget(targetId?: string): CodexPoolRuntimeTarget {
|
|
const runtimeConfig = readSub2ApiRuntimeConfig();
|
|
const resolvedTargetId = targetId ?? runtimeConfig.defaultTargetId;
|
|
const raw = runtimeConfig.targets.find((item) => String(item.id ?? "").toLowerCase() === resolvedTargetId.toLowerCase());
|
|
if (!isRecord(raw)) throw new Error(`${sub2apiConfigPath}.targets does not contain target ${targetId}`);
|
|
const id = stringValue(raw.id) ?? resolvedTargetId;
|
|
const route = stringValue(raw.route) ?? "";
|
|
const targetNamespace = stringValue(raw.namespace);
|
|
if (route.length === 0) throw new Error(`${sub2apiConfigPath}.targets[${id}].route is required`);
|
|
if (targetNamespace === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].namespace is required`);
|
|
validateKubernetesName(targetNamespace, `${sub2apiConfigPath}.targets[${id}].namespace`, true);
|
|
const sentinelImageBuild = readTargetSentinelImageBuild(raw, id);
|
|
|
|
let egressProxy: CodexPoolRuntimeTarget["egressProxy"] = null;
|
|
if (isRecord(raw.egressProxy) && raw.egressProxy.enabled === true) {
|
|
const proxyServiceName = stringValue(raw.egressProxy.serviceName);
|
|
const listenPort = numberValue(raw.egressProxy.listenPort);
|
|
if (proxyServiceName === null || listenPort === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].egressProxy.serviceName/listenPort are required`);
|
|
validateKubernetesName(proxyServiceName, `${sub2apiConfigPath}.targets[${id}].egressProxy.serviceName`, true);
|
|
if (!Number.isInteger(listenPort) || listenPort < 1 || listenPort > 65535) throw new Error(`${sub2apiConfigPath}.targets[${id}].egressProxy.listenPort must be a TCP port`);
|
|
const noProxyRaw = Array.isArray(raw.egressProxy.noProxy) ? raw.egressProxy.noProxy : [];
|
|
const noProxy = noProxyRaw.map((entry) => stringValue(entry)).filter((entry): entry is string => entry !== null && entry.length > 0).join(",");
|
|
egressProxy = {
|
|
enabled: true,
|
|
applyToSentinel: raw.egressProxy.applyToSentinel === undefined ? true : raw.egressProxy.applyToSentinel === true,
|
|
serviceName: proxyServiceName,
|
|
listenPort,
|
|
httpProxy: `http://${proxyServiceName}.${targetNamespace}.svc.cluster.local:${listenPort}`,
|
|
noProxy,
|
|
};
|
|
}
|
|
|
|
let publicBaseUrl: string | null = null;
|
|
const publicExposure = readTargetPublicExposure(raw, id);
|
|
if (publicExposure !== null && publicExposure.enabled) publicBaseUrl = publicExposure.publicBaseUrl;
|
|
|
|
return {
|
|
id,
|
|
route,
|
|
namespace: targetNamespace,
|
|
serviceName,
|
|
serviceDns: `${serviceName}.${targetNamespace}.svc.cluster.local:8080`,
|
|
publicBaseUrl,
|
|
publicExposure,
|
|
appSecretName: runtimeConfig.appSecretName,
|
|
secretsRoot: runtimeConfig.secretsRoot,
|
|
sentinelImageBuild,
|
|
egressProxy,
|
|
};
|
|
}
|
|
|
|
function readTargetSentinelImageBuild(raw: Record<string, unknown>, targetId: string): CodexPoolRuntimeTarget["sentinelImageBuild"] {
|
|
const codexPool = isRecord(raw.codexPool) ? raw.codexPool : null;
|
|
const imageBuild = codexPool !== null && isRecord(codexPool.sentinelImageBuild) ? codexPool.sentinelImageBuild : null;
|
|
if (imageBuild === null) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild must be a YAML object`);
|
|
const policy = stringValue(imageBuild.baseImageCachePolicy);
|
|
if (policy !== "pull" && policy !== "local-if-present") {
|
|
throw new Error(`${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild.baseImageCachePolicy must be pull or local-if-present`);
|
|
}
|
|
return {
|
|
baseImageCachePolicy: policy,
|
|
noProxy: readTargetNoProxy(imageBuild.noProxy, `${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild.noProxy`),
|
|
};
|
|
}
|
|
|
|
function readTargetPublicExposure(raw: Record<string, unknown>, targetId: string): PublicServiceExposure | null {
|
|
if (raw.publicExposure === undefined || raw.publicExposure === null) return null;
|
|
if (!isRecord(raw.publicExposure)) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].publicExposure must be a YAML object`);
|
|
const exposure = raw.publicExposure;
|
|
const enabled = readRequiredTargetBoolean(exposure.enabled, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.enabled`);
|
|
const dns = readRequiredTargetRecord(exposure.dns, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns`);
|
|
const frpc = readRequiredTargetRecord(exposure.frpc, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc`);
|
|
const pk01 = readRequiredTargetRecord(exposure.pk01, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01`);
|
|
const publicBaseUrl = readTargetBaseUrl(exposure.publicBaseUrl, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.publicBaseUrl`, "https:");
|
|
const hostname = readRequiredTargetString(dns.hostname, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.hostname`);
|
|
if (new URL(publicBaseUrl).hostname !== hostname) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].publicExposure publicBaseUrl hostname must match dns.hostname`);
|
|
const config: PublicServiceExposure = {
|
|
enabled,
|
|
publicBaseUrl,
|
|
dns: {
|
|
hostname,
|
|
expectedA: readRequiredTargetString(dns.expectedA, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.expectedA`),
|
|
resolvers: readTargetStringArray(dns.resolvers, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.resolvers`),
|
|
},
|
|
frpc: {
|
|
deploymentName: readRequiredTargetString(frpc.deploymentName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.deploymentName`),
|
|
secretName: readRequiredTargetString(frpc.secretName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretName`),
|
|
secretKey: readRequiredTargetString(frpc.secretKey, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretKey`),
|
|
image: readRequiredTargetString(frpc.image, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.image`),
|
|
serverAddr: readRequiredTargetString(frpc.serverAddr, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverAddr`),
|
|
serverPort: readTargetPort(frpc.serverPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverPort`),
|
|
proxyName: readRequiredTargetString(frpc.proxyName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.proxyName`),
|
|
remotePort: readTargetPort(frpc.remotePort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.remotePort`),
|
|
localIP: readRequiredTargetString(frpc.localIP, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localIP`),
|
|
localPort: readTargetPort(frpc.localPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localPort`),
|
|
tokenSourceRef: readRequiredTargetString(frpc.tokenSourceRef, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.tokenSourceRef`),
|
|
tokenSourceKey: readRequiredTargetString(frpc.tokenSourceKey, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.tokenSourceKey`),
|
|
},
|
|
pk01: {
|
|
route: readRequiredTargetString(pk01.route, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.route`),
|
|
caddyConfigPath: readAbsoluteTargetPath(pk01.caddyConfigPath, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.caddyConfigPath`),
|
|
caddyServiceName: readRequiredTargetString(pk01.caddyServiceName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.caddyServiceName`),
|
|
responseHeaderTimeoutSeconds: readPositiveTargetInteger(pk01.responseHeaderTimeoutSeconds, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.responseHeaderTimeoutSeconds`),
|
|
},
|
|
};
|
|
validateKubernetesName(config.frpc.deploymentName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.deploymentName`, true);
|
|
validateKubernetesName(config.frpc.secretName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretName`, true);
|
|
validateProxyName(config.frpc.proxyName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.proxyName`);
|
|
validatePort(config.frpc.serverPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverPort`);
|
|
validatePort(config.frpc.remotePort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.remotePort`);
|
|
validatePort(config.frpc.localPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localPort`);
|
|
return config;
|
|
}
|
|
|
|
function readRequiredTargetRecord(value: unknown, path: string): Record<string, unknown> {
|
|
if (!isRecord(value)) throw new Error(`${path} must be a YAML object`);
|
|
return value;
|
|
}
|
|
|
|
function readRequiredTargetString(value: unknown, path: string): string {
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${path} must be a non-empty string`);
|
|
if (/[\r\n]/u.test(text)) throw new Error(`${path} must not contain newlines`);
|
|
return text;
|
|
}
|
|
|
|
function readRequiredTargetBoolean(value: unknown, path: string): boolean {
|
|
const parsed = booleanValue(value);
|
|
if (parsed === null) throw new Error(`${path} must be a boolean`);
|
|
return parsed;
|
|
}
|
|
|
|
function readTargetBaseUrl(value: unknown, path: string, protocol: "http:" | "https:" | null = null): string {
|
|
const baseUrl = normalizeBaseUrl(stringValue(value));
|
|
if (baseUrl === null) throw new Error(`${path} must be a valid http(s) URL`);
|
|
if (protocol !== null && new URL(baseUrl).protocol !== protocol) throw new Error(`${path} must use ${protocol}//`);
|
|
return baseUrl;
|
|
}
|
|
|
|
function readTargetPort(value: unknown, path: string): number {
|
|
const port = numberValue(value);
|
|
if (port === null || !Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path} must be an integer TCP port`);
|
|
return port;
|
|
}
|
|
|
|
function readPositiveTargetInteger(value: unknown, path: string): number {
|
|
const number = numberValue(value);
|
|
if (number === null || !Number.isInteger(number) || number < 1) throw new Error(`${path} must be a positive integer`);
|
|
return number;
|
|
}
|
|
|
|
function readAbsoluteTargetPath(value: unknown, path: string): string {
|
|
const text = readRequiredTargetString(value, path);
|
|
if (!text.startsWith("/")) throw new Error(`${path} must be an absolute path`);
|
|
return text;
|
|
}
|
|
|
|
function readTargetStringArray(value: unknown, path: string): string[] {
|
|
if (!Array.isArray(value)) throw new Error(`${path} must be a YAML array`);
|
|
const result = value.map((item, index) => readRequiredTargetString(item, `${path}[${index}]`));
|
|
if (result.length === 0) throw new Error(`${path} must not be empty`);
|
|
return result;
|
|
}
|
|
|
|
function readTargetNoProxy(value: unknown, path: string): string {
|
|
return readTargetStringArray(value, path).join(",");
|
|
}
|
|
|
|
function targetFlag(target: CodexPoolRuntimeTarget): string {
|
|
return target.id === defaultCodexPoolRuntimeTargetId() ? "" : ` --target ${target.id}`;
|
|
}
|
|
|
|
function codexPoolPlan(options?: DisclosureOptions): Record<string, unknown> {
|
|
const resolvedOptions = options ?? { full: false, raw: false, targetId: defaultCodexPoolRuntimeTargetId() };
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(resolvedOptions.targetId);
|
|
const profiles = collectCodexProfiles();
|
|
const ok = profiles.length > 0 && profiles.every((profile) => profile.ok);
|
|
const consumerBaseUrl = runtimeTarget.publicBaseUrl === null ? null : codexConsumerBaseUrl(pool, runtimeTarget);
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-plan",
|
|
source: {
|
|
directory: join(homedir(), ".codex"),
|
|
configPattern: "YAML-selected config files under ~/.codex",
|
|
authPattern: "YAML-selected auth files under ~/.codex",
|
|
valuesPrinted: false,
|
|
},
|
|
target: poolTarget(pool, runtimeTarget),
|
|
config: {
|
|
path: codexPoolConfigPath,
|
|
pool: options.full ? pool : codexPoolConfigSummary(pool, runtimeTarget),
|
|
},
|
|
profiles: options.full ? profiles.map(redactProfile) : profiles.map(compactProfile),
|
|
decision: {
|
|
accountType: "openai/apikey",
|
|
grouping: `All discovered Codex profiles are bound to one Sub2API group named ${pool.groupName}.`,
|
|
unifiedApiKey: `The client-facing API_KEY is controlled by k3s Secret ${runtimeTarget.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}.`,
|
|
sentinel: pool.sentinel.monitor.enabled
|
|
? `Account sentinel is enabled as k8s CronJob ${runtimeTarget.namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.`
|
|
: "Account sentinel monitoring is disabled by YAML.",
|
|
publicExposure: runtimeTarget.publicBaseUrl === null
|
|
? `Target-level public exposure is disabled or absent in ${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.`
|
|
: `Codex consumers for target ${runtimeTarget.id} use target-level public exposure ${consumerBaseUrl}.`,
|
|
idempotency: "sync reuses the group, account names, and k3s Secret when they already exist; credentials are updated from the current local Codex files for YAML-managed profiles only; managed accounts missing from YAML are preserved unless --prune-removed is explicitly provided.",
|
|
configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.",
|
|
manualAccountProtection: pool.manualAccounts.protected.length === 0
|
|
? "No manual Sub2API accounts are protected by YAML."
|
|
: `${pool.manualAccounts.protected.length} manual Sub2API account(s) are protected from UniDesk-managed credentials, prune, sentinel probe, and sentinel freeze paths; only explicitly declared proxy/group bindings are reconciled.`,
|
|
},
|
|
next: ok
|
|
? { sync: `bun scripts/cli.ts platform-infra sub2api codex-pool sync${targetFlag(runtimeTarget)} --confirm` }
|
|
: { fix: "Ensure every discovered config.toml profile has a base_url and either auth.json OPENAI_API_KEY or the configured env_key present in this shell." },
|
|
};
|
|
}
|
|
|
|
async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const profiles = collectCodexProfiles();
|
|
const planOk = profiles.length > 0 && profiles.every((profile) => profile.ok);
|
|
if (!options.confirm || !planOk) {
|
|
return {
|
|
...codexPoolPlan(options),
|
|
ok: !options.confirm ? planOk : false,
|
|
mode: options.confirm ? "blocked-invalid-local-profile" : "dry-run",
|
|
next: options.confirm
|
|
? { fix: "Repair invalid local Codex profiles, then rerun sync --confirm." }
|
|
: { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" },
|
|
};
|
|
}
|
|
|
|
const sentinelImage = pool.sentinel.monitor.enabled
|
|
? await runCodexPoolSentinelImage(config, pool, { action: "build", confirm: true, dryRun: false, full: options.full, raw: false, targetId: options.targetId })
|
|
: { ok: true, mode: "skipped-monitor-disabled" };
|
|
if (sentinelImage.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-sync",
|
|
mode: "blocked-sentinel-image",
|
|
sentinelImage,
|
|
next: {
|
|
image: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build${targetFlag(runtimeTarget)} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
const payload = {
|
|
pruneRemoved: options.pruneRemoved,
|
|
sentinel: {
|
|
manifest: renderCodexPoolSentinelManifest(pool.sentinel, sentinelProfileSecrets(profiles), {
|
|
namespace: runtimeTarget.namespace,
|
|
serviceName: runtimeTarget.serviceName,
|
|
serviceDns: runtimeTarget.serviceDns,
|
|
appSecretName: runtimeTarget.appSecretName,
|
|
adminEmailDefault: pool.adminEmailDefault,
|
|
proxy: runtimeTarget.egressProxy?.applyToSentinel ? {
|
|
httpProxy: runtimeTarget.egressProxy.httpProxy,
|
|
noProxy: runtimeTarget.egressProxy.noProxy,
|
|
} : null,
|
|
}),
|
|
summary: codexPoolSentinelSummary(pool.sentinel),
|
|
},
|
|
pool: {
|
|
groupName: pool.groupName,
|
|
groupDescription: pool.groupDescription,
|
|
apiKeyName: pool.apiKeyName,
|
|
apiKeySecretName: pool.apiKeySecretName,
|
|
apiKeySecretKey: pool.apiKeySecretKey,
|
|
adminEmailDefault: pool.adminEmailDefault,
|
|
minOwnerBalanceUsd: pool.minOwnerBalanceUsd,
|
|
minOwnerConcurrency: pool.minOwnerConcurrency,
|
|
defaultAccountPriority: pool.defaultAccountPriority,
|
|
defaultAccountCapacity: pool.defaultAccountCapacity,
|
|
defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
|
|
defaultSentinelProtect: pool.defaultSentinelProtect,
|
|
},
|
|
manualAccounts: {
|
|
bindingSources: pool.manualAccounts.bindingSources.items.map(manualBindingSourcePlan),
|
|
protected: resolvedManualAccountProtections(pool, runtimeTarget),
|
|
},
|
|
profiles: profiles.map((profile) => ({
|
|
profile: profile.profile,
|
|
accountName: profile.accountName,
|
|
configFile: profile.configFile,
|
|
authFile: profile.authFile,
|
|
provider: profile.provider,
|
|
baseUrl: profile.baseUrl,
|
|
wireApi: profile.wireApi,
|
|
model: profile.model,
|
|
apiKey: profile.apiKey,
|
|
apiKeySource: profile.apiKeySource,
|
|
apiKeyFingerprint: fingerprint(profile.apiKey ?? ""),
|
|
sentinelProbeConfigFingerprint: codexPoolSentinelProbeConfigFingerprint({
|
|
accountName: profile.accountName,
|
|
profile: profile.profile,
|
|
baseUrl: profile.baseUrl,
|
|
apiKeyFingerprint: fingerprint(profile.apiKey ?? ""),
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
|
trustUpstream: profile.trustUpstream,
|
|
sentinelProtect: profile.sentinelProtect,
|
|
}),
|
|
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
trustUpstream: profile.trustUpstream,
|
|
sentinelProtect: profile.sentinelProtect,
|
|
priority: profile.priority,
|
|
capacity: profile.capacity,
|
|
loadFactor: profile.loadFactor,
|
|
tempUnschedulable: profile.tempUnschedulable,
|
|
tempUnschedulableCredentials: renderSub2ApiTempUnschedulableCredentials(profile.tempUnschedulable),
|
|
})),
|
|
};
|
|
const result = await runRemoteCodexPoolScript(config, "sync", syncScript(payload, pool, runtimeTarget), runtimeTarget);
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sync",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sync",
|
|
local: {
|
|
profileCount: profiles.length,
|
|
pruneRemoved: options.pruneRemoved,
|
|
invalidProfiles: profiles.filter((profile) => !profile.ok).map(compactProfile),
|
|
profiles: options.full ? profiles.map(redactProfile) : undefined,
|
|
valuesPrinted: false,
|
|
},
|
|
sentinelImage,
|
|
remote: parsed === null
|
|
? compactCapture(result, { full: options.full || result.exitCode !== 0 })
|
|
: options.full ? parsed : codexPoolSyncSummary(parsed),
|
|
next: {
|
|
validate: `bun scripts/cli.ts platform-infra sub2api codex-pool validate${targetFlag(runtimeTarget)}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function codexPoolSentinelImage(config: UniDeskConfig, options: SentinelImageOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
return await runCodexPoolSentinelImage(config, pool, options);
|
|
}
|
|
|
|
async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolConfig, options: SentinelImageOptions): Promise<Record<string, unknown>> {
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const target = codexPoolSentinelRuntimeImage(pool.sentinel);
|
|
if (options.action === "build" && options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-image",
|
|
mode: "dry-run",
|
|
image: target,
|
|
dockerfile: sentinelImageDockerfilePath,
|
|
mutation: false,
|
|
next: {
|
|
confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build${targetFlag(runtimeTarget)} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
const mode: RemoteCodexPoolMode = options.action === "status" ? "sentinel-image-status" : "sentinel-image-build";
|
|
const script = options.action === "status" ? sentinelImageStatusScript(pool, runtimeTarget) : sentinelImageBuildScript(pool, runtimeTarget);
|
|
const result = await runRemoteCodexPoolScript(config, mode, script, runtimeTarget);
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-image",
|
|
mode: options.action,
|
|
image: target,
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-image",
|
|
mode: options.action,
|
|
image: target,
|
|
summary: parsed,
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolValidate(config: UniDeskConfig, options: DisclosureOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const result = await runRemoteCodexPoolScript(config, "validate", validateScript(pool, runtimeTarget), runtimeTarget);
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-validate",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-validate",
|
|
summary: options.full ? parsed : codexPoolValidationSummary(parsed),
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolTrace(config: UniDeskConfig, options: TraceOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const result = await capture(config, runtimeTarget.route, ["sh"], traceScript(pool, options, runtimeTarget));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
|
|
if (options.raw) {
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-trace",
|
|
remote: compactCapture(result, { full: true }),
|
|
trace: parsed,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const text = renderTraceReport(parsed, {
|
|
requestId: options.requestId ?? "",
|
|
showLines: options.showLines,
|
|
remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }),
|
|
});
|
|
return renderedCliResult(ok, "platform-infra sub2api codex-pool trace", text);
|
|
}
|
|
|
|
async function codexPoolSentinelReport(config: UniDeskConfig, options: SentinelReportOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const result = await capture(config, runtimeTarget.route, ["sh"], sentinelReportScript(pool, options.events, runtimeTarget));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
|
|
if (options.raw) {
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-report",
|
|
remote: compactCapture(result, { full: true }),
|
|
report: parsed,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const text = renderSentinelReport(parsed, {
|
|
events: options.events,
|
|
full: options.full,
|
|
remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }),
|
|
});
|
|
return renderedCliResult(ok, "platform-infra sub2api codex-pool sentinel-report", text);
|
|
}
|
|
|
|
async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelProbeOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const protectedNames = new Set(pool.manualAccounts.protected.map((account) => account.accountName.toLowerCase()));
|
|
const protectedRequested = options.accounts.filter((account) => protectedNames.has(account.toLowerCase()));
|
|
if (protectedRequested.length > 0) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
error: "account-protected-manual",
|
|
protected: protectedRequested,
|
|
message: "Protected manual Sub2API accounts are not YAML-managed and must not be probed by the UniDesk sentinel.",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const configuredAccounts = desiredAccountNames(pool);
|
|
const missing = options.accounts.filter((account) => !configuredAccounts.includes(account));
|
|
if (missing.length > 0) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
error: "account-not-in-yaml",
|
|
missing,
|
|
configuredAccounts,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
mode: "dry-run",
|
|
target: poolTarget(pool, runtimeTarget),
|
|
accounts: options.accounts,
|
|
effect: "Would create one Kubernetes Job from the managed sentinel CronJob and force an immediate marker probe for the requested account(s).",
|
|
next: {
|
|
confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe${targetFlag(runtimeTarget)} --account ${options.accounts.join(",")} --confirm`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const payload = {
|
|
accounts: options.accounts,
|
|
};
|
|
const result = await runRemoteCodexPoolScript(config, "sentinel-probe", sentinelProbeScript(payload, pool, runtimeTarget), runtimeTarget);
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
summary: options.full ? parsed : compactSentinelProbeResult(parsed),
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolCleanupProbes(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-cleanup-probes",
|
|
mode: "dry-run",
|
|
target: poolTarget(readCodexPoolConfig(), runtimeTarget),
|
|
scope: "Only deletes temporary resources whose names start with unidesk-probe-.",
|
|
next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes${targetFlag(runtimeTarget)} --confirm` },
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const pool = readCodexPoolConfig();
|
|
const result = await capture(config, runtimeTarget.route, ["sh"], cleanupProbesScript(pool, runtimeTarget));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-cleanup-probes",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-cleanup-probes",
|
|
summary: parsed,
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
if (runtimeTarget.publicExposure === null || !runtimeTarget.publicExposure.enabled) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
mode: "disabled-by-yaml",
|
|
target: poolTarget(pool, runtimeTarget),
|
|
source: `${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure`,
|
|
};
|
|
}
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
mode: "dry-run",
|
|
target: poolTarget(pool, runtimeTarget),
|
|
publicExposure: targetPublicExposureSummary(runtimeTarget),
|
|
next: {
|
|
confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool expose${targetFlag(runtimeTarget)} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
const secretMaterial = prepareTargetPublicExposureSecret(runtimeTarget);
|
|
const caddyResult = await applyPk01CaddyBlock(config, serviceName, runtimeTarget.publicExposure);
|
|
const remoteResult = await capture(config, runtimeTarget.route, ["sh"], targetPublicExposureApplyScript(runtimeTarget, secretMaterial));
|
|
const parsed = parseJsonOutput(remoteResult.stdout);
|
|
const publicProbe = await probePublicModels(pool, "without-api-key", undefined, runtimeTarget);
|
|
const ok = caddyResult.ok === true && remoteResult.exitCode === 0 && boolField(parsed, "ok", false) && publicProbe.httpStatus === 401;
|
|
if (options.raw) {
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
frpcSecret: secretMaterialSummary(secretMaterial),
|
|
pk01Caddy: caddyResult,
|
|
remote: compactCapture(remoteResult, { full: true }),
|
|
parsed,
|
|
publicProbe,
|
|
};
|
|
}
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
mode: "confirmed",
|
|
publicExposure: targetPublicExposureSummary(runtimeTarget),
|
|
frpcSecret: secretMaterialSummary(secretMaterial),
|
|
pk01Caddy: caddyResult,
|
|
remote: parsed ?? compactCapture(remoteResult, { full: options.full || remoteResult.exitCode !== 0 }),
|
|
publicProbe,
|
|
next: {
|
|
configureLocal: `bun scripts/cli.ts platform-infra sub2api codex-pool configure-local${targetFlag(runtimeTarget)} --confirm`,
|
|
validate: `bun scripts/cli.ts platform-infra sub2api codex-pool validate${targetFlag(runtimeTarget)}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const runtimeTarget = codexPoolRuntimeTarget(options.targetId);
|
|
const consumerExposureAvailable = runtimeTarget.publicBaseUrl !== null;
|
|
const codexDir = join(homedir(), ".codex");
|
|
const configPath = join(codexDir, "config.toml");
|
|
const authPath = join(codexDir, "auth.json");
|
|
const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`);
|
|
const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`);
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-configure-local",
|
|
mode: "dry-run",
|
|
target: {
|
|
codexDir,
|
|
configPath,
|
|
authPath,
|
|
backupConfigPath,
|
|
backupAuthPath,
|
|
baseUrl: consumerExposureAvailable ? codexConsumerBaseUrl(pool, runtimeTarget) : null,
|
|
providerName: pool.localCodex.providerName,
|
|
wireApi: pool.localCodex.wireApi,
|
|
modelContextWindow: pool.localCodex.modelContextWindow,
|
|
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
|
|
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
|
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
|
responsesSmokeModel: pool.localCodex.responsesSmokeModel,
|
|
valuesPrinted: false,
|
|
},
|
|
next: {
|
|
confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool configure-local${targetFlag(runtimeTarget)} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
if (!consumerExposureAvailable) throw new Error(`${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.enabled must be true; configure-local needs one consumer URL`);
|
|
const keyResult = await fetchPoolApiKey(config, pool, runtimeTarget);
|
|
if (keyResult.apiKey === null) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-configure-local",
|
|
error: keyResult.error ?? "pool API key missing",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const writeResult = writeLocalCodexConfig(pool, keyResult.apiKey, runtimeTarget);
|
|
const validateResult = await validatePublicGatewayWithKey(pool, keyResult.apiKey, runtimeTarget);
|
|
return {
|
|
ok: writeResult.ok && validateResult.ok,
|
|
action: "platform-infra-sub2api-codex-pool-configure-local",
|
|
mode: "confirmed",
|
|
local: writeResult,
|
|
validation: validateResult,
|
|
apiKey: {
|
|
secret: `${runtimeTarget.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`,
|
|
keyPreview: apiKeyPreview(keyResult.apiKey),
|
|
apiKeyFingerprint: fingerprint(keyResult.apiKey),
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function collectCodexProfiles(): CodexProfile[] {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const pool = readCodexPoolConfig();
|
|
if (!existsSync(codexDir)) return [];
|
|
const seenAccountNames = new Set<string>();
|
|
return pool.profiles.map((entry) => {
|
|
const resolved = resolveProfileFiles(codexDir, entry);
|
|
const profile = entry.profile;
|
|
const accountName = entry.accountName ?? uniqueAccountName(profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
const configFile = resolved.configFile;
|
|
const authFile = resolved.authFile;
|
|
const configPath = join(codexDir, configFile);
|
|
const authPath = join(codexDir, authFile);
|
|
const base: CodexProfile = {
|
|
profile,
|
|
accountName,
|
|
configFile,
|
|
authFile,
|
|
provider: "",
|
|
baseUrl: "",
|
|
wireApi: null,
|
|
model: null,
|
|
envKey: null,
|
|
apiKey: null,
|
|
apiKeySource: null,
|
|
openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent: entry.upstreamUserAgent,
|
|
trustUpstream: entry.trustUpstream,
|
|
sentinelProtect: entry.sentinelProtect,
|
|
priority: entry.priority,
|
|
capacity: entry.capacity ?? pool.defaultAccountCapacity,
|
|
loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor,
|
|
tempUnschedulable: entry.tempUnschedulable,
|
|
authOpenAIKeyShape: existsSync(authPath) ? "unknown" : "missing",
|
|
ok: false,
|
|
error: null,
|
|
};
|
|
|
|
try {
|
|
if (!existsSync(configPath)) throw new Error(`config file ${configFile} is missing`);
|
|
const parsed = Bun.TOML.parse(readFileSync(configPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) throw new Error("config is not a TOML object");
|
|
const providers = parsed.model_providers;
|
|
if (!isRecord(providers)) throw new Error("model_providers is missing");
|
|
const provider = stringValue(parsed.model_provider) ?? Object.keys(providers)[0] ?? "";
|
|
if (provider === "") throw new Error("model_provider is missing");
|
|
const providerConfig = providers[provider];
|
|
if (!isRecord(providerConfig)) throw new Error(`model provider ${provider} is missing`);
|
|
const baseUrl = normalizeBaseUrl(stringValue(providerConfig.base_url));
|
|
if (baseUrl === null) throw new Error(`model provider ${provider} base_url is missing or invalid`);
|
|
base.provider = provider;
|
|
base.baseUrl = baseUrl;
|
|
base.envKey = stringValue(providerConfig.env_key);
|
|
base.wireApi = stringValue(providerConfig.wire_api);
|
|
base.model = stringValue(parsed.model);
|
|
|
|
const auth = readAuthAPIKey(authPath);
|
|
base.authOpenAIKeyShape = auth.shape;
|
|
if (auth.apiKey !== null) {
|
|
base.apiKey = auth.apiKey;
|
|
base.apiKeySource = "auth-json";
|
|
} else if (base.envKey !== null && typeof process.env[base.envKey] === "string" && process.env[base.envKey]!.length > 0) {
|
|
base.apiKey = process.env[base.envKey]!;
|
|
base.apiKeySource = "env";
|
|
}
|
|
if (base.apiKey === null || base.apiKey.length === 0) {
|
|
throw new Error(base.envKey === null ? "auth OPENAI_API_KEY is missing or empty" : `auth OPENAI_API_KEY is missing and env ${base.envKey} is not present`);
|
|
}
|
|
base.ok = true;
|
|
return base;
|
|
} catch (error) {
|
|
base.error = error instanceof Error ? error.message : String(error);
|
|
return base;
|
|
}
|
|
});
|
|
}
|
|
|
|
function resolveProfileFiles(codexDir: string, profile: CodexPoolProfileConfig): { configFile: string; authFile: string } {
|
|
const configFile = existsSync(join(codexDir, profile.configFile)) || profile.fallbackConfigFile === null
|
|
? profile.configFile
|
|
: profile.fallbackConfigFile;
|
|
const authFile = existsSync(join(codexDir, profile.authFile)) || profile.fallbackAuthFile === null
|
|
? profile.authFile
|
|
: profile.fallbackAuthFile;
|
|
return { configFile, authFile };
|
|
}
|
|
|
|
function readCodexPoolConfig(): CodexPoolConfig {
|
|
if (!existsSync(codexPoolConfigPath)) throw new Error(`${codexPoolConfigPath} is required`);
|
|
const parsed = Bun.YAML.parse(readFileSync(codexPoolConfigPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) throw new Error(`${codexPoolConfigPath} must contain a YAML object`);
|
|
const version = integerConfigField(parsed, "version", "");
|
|
if (version !== 1) throw new Error(`${codexPoolConfigPath}.version must be 1`);
|
|
const kind = requiredStringConfigField(parsed, "kind", "");
|
|
if (kind !== "platform-infra-sub2api-codex-pool") throw new Error(`${codexPoolConfigPath}.kind must be platform-infra-sub2api-codex-pool`);
|
|
const metadata = requiredRecordConfigField(parsed, "metadata", "");
|
|
const pool = parsed.pool;
|
|
if (!isRecord(pool)) throw new Error(`${codexPoolConfigPath}.pool must be a YAML object`);
|
|
if (!isRecord(parsed.profiles)) throw new Error(`${codexPoolConfigPath}.profiles must be a YAML object`);
|
|
rejectSchedulableYamlField(pool, "pool");
|
|
const defaultTempUnschedulable = readTempUnschedulablePolicy(pool.defaultTempUnschedulable, "pool.defaultTempUnschedulable");
|
|
const defaultAccountPriorityValue = readAccountPriority(pool.defaultAccountPriority, "pool.defaultAccountPriority");
|
|
const defaultAccountCapacityValue = readAccountCapacity(pool.defaultAccountCapacity, "pool.defaultAccountCapacity");
|
|
const defaultAccountLoadFactorValue = readAccountLoadFactor(pool.defaultAccountLoadFactor, "pool.defaultAccountLoadFactor");
|
|
const defaultSentinelProtect = readSentinelProtectPolicy(pool.defaultSentinelProtect, "pool.defaultSentinelProtect");
|
|
const profiles = readProfileConfig(
|
|
parsed.profiles,
|
|
defaultTempUnschedulable,
|
|
defaultSentinelProtect,
|
|
defaultAccountPriorityValue,
|
|
);
|
|
const manualAccounts = readManualAccountsConfig(parsed.manualAccounts);
|
|
assertProtectedManualAccountsNotManaged(profiles, manualAccounts);
|
|
const declaredAccountCapacity = desiredProfileCapacityTotal(profiles, defaultAccountCapacityValue);
|
|
const minOwnerConcurrencySource = pool.minOwnerConcurrency === undefined || pool.minOwnerConcurrency === null ? "auto" : "yaml";
|
|
const minOwnerConcurrency = minOwnerConcurrencySource === "auto"
|
|
? Math.max(1, declaredAccountCapacity)
|
|
: readOwnerConcurrency(pool.minOwnerConcurrency, "pool.minOwnerConcurrency");
|
|
const config: CodexPoolConfig = {
|
|
version,
|
|
kind,
|
|
metadata: {
|
|
id: requiredStringConfigField(metadata, "id", "metadata"),
|
|
owner: requiredStringConfigField(metadata, "owner", "metadata"),
|
|
relatedIssues: integerArrayConfigField(metadata, "relatedIssues", "metadata"),
|
|
},
|
|
groupName: requiredStringConfigField(pool, "groupName", "pool"),
|
|
groupDescription: readRequiredDescription(pool.groupDescription, "pool.groupDescription", 300),
|
|
apiKeyName: requiredStringConfigField(pool, "apiKeyName", "pool"),
|
|
apiKeySecretName: requiredStringConfigField(pool, "apiKeySecretName", "pool"),
|
|
apiKeySecretKey: requiredStringConfigField(pool, "apiKeySecretKey", "pool"),
|
|
adminEmailDefault: readRequiredEmail(pool.adminEmailDefault, "pool.adminEmailDefault"),
|
|
minOwnerBalanceUsd: readRequiredPositiveNumber(pool.minOwnerBalanceUsd, "pool.minOwnerBalanceUsd"),
|
|
minOwnerConcurrency,
|
|
minOwnerConcurrencySource,
|
|
defaultAccountPriority: defaultAccountPriorityValue,
|
|
defaultAccountCapacity: defaultAccountCapacityValue,
|
|
defaultAccountLoadFactor: defaultAccountLoadFactorValue,
|
|
defaultTempUnschedulable,
|
|
defaultSentinelProtect,
|
|
profiles,
|
|
manualAccounts,
|
|
publicExposure: readPublicExposureConfig(parsed.publicExposure),
|
|
localCodex: readLocalCodexConfig(parsed.localCodex),
|
|
sentinel: {
|
|
...readCodexPoolSentinelConfig(parsed.sentinel, codexPoolConfigPath),
|
|
protectedManualAccounts: manualAccounts.protected.map((account) => account.accountName),
|
|
},
|
|
};
|
|
validateKubernetesName(config.groupName, "pool.groupName", false);
|
|
validateKubernetesName(config.apiKeySecretName, "pool.apiKeySecretName", true);
|
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(config.apiKeySecretKey)) {
|
|
throw new Error(`${codexPoolConfigPath}.pool.apiKeySecretKey must be a valid secret key name`);
|
|
}
|
|
if (config.minOwnerBalanceUsd <= 0) throw new Error(`${codexPoolConfigPath}.pool.minOwnerBalanceUsd must be > 0`);
|
|
if (declaredAccountCapacity > 0 && config.minOwnerConcurrency < declaredAccountCapacity) {
|
|
throw new Error(`${codexPoolConfigPath}.pool.minOwnerConcurrency must be >= the sum of declared account capacities (${declaredAccountCapacity})`);
|
|
}
|
|
return config;
|
|
}
|
|
|
|
function readProfileConfig(
|
|
value: unknown,
|
|
defaultTempUnschedulable: CodexTempUnschedulablePolicy,
|
|
defaultSentinelProtect: CodexSentinelProtectPolicy,
|
|
defaultPriority: number,
|
|
): CodexPoolProfileConfig[] {
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.profiles must be a YAML object`);
|
|
const entries = value.entries;
|
|
if (!Array.isArray(entries)) throw new Error(`${codexPoolConfigPath}.profiles.entries must be a YAML array`);
|
|
return entries.map((entry, index) => {
|
|
if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] must be an object`);
|
|
rejectSchedulableYamlField(entry, `profiles.entries[${index}]`);
|
|
const profile = stringValue(entry.profile);
|
|
const configFile = stringValue(entry.configFile);
|
|
const authFile = stringValue(entry.authFile);
|
|
if (profile === null || configFile === null || authFile === null) {
|
|
throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] requires profile, configFile, and authFile`);
|
|
}
|
|
const accountName = stringValue(entry.accountName);
|
|
if (accountName !== null) validateKubernetesName(accountName, `profiles.entries[${index}].accountName`, false);
|
|
validateCodexFileName(configFile, `profiles.entries[${index}].configFile`);
|
|
validateCodexFileName(authFile, `profiles.entries[${index}].authFile`);
|
|
const fallbackConfigFile = stringValue(entry.fallbackConfigFile);
|
|
const fallbackAuthFile = stringValue(entry.fallbackAuthFile);
|
|
if (fallbackConfigFile !== null) validateCodexFileName(fallbackConfigFile, `profiles.entries[${index}].fallbackConfigFile`);
|
|
if (fallbackAuthFile !== null) validateCodexFileName(fallbackAuthFile, `profiles.entries[${index}].fallbackAuthFile`);
|
|
const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`);
|
|
const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`);
|
|
const trustUpstream = readTrustUpstream(entry.trustUpstream, `profiles.entries[${index}].trustUpstream`);
|
|
const sentinelProtect = readSentinelProtectPolicy(entry.sentinelProtect, `profiles.entries[${index}].sentinelProtect`, defaultSentinelProtect);
|
|
const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority);
|
|
const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`);
|
|
const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`);
|
|
const tempUnschedulable = readTempUnschedulablePolicy(entry.tempUnschedulable, `profiles.entries[${index}].tempUnschedulable`, defaultTempUnschedulable);
|
|
return {
|
|
profile,
|
|
accountName,
|
|
configFile,
|
|
authFile,
|
|
fallbackConfigFile,
|
|
fallbackAuthFile,
|
|
openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent,
|
|
trustUpstream,
|
|
sentinelProtect,
|
|
priority,
|
|
capacity,
|
|
loadFactor,
|
|
tempUnschedulable,
|
|
};
|
|
});
|
|
}
|
|
|
|
function desiredProfileCapacityTotal(profiles: CodexPoolProfileConfig[], defaultCapacity: number): number {
|
|
return profiles.reduce((total, profile) => total + (profile.capacity ?? defaultCapacity), 0);
|
|
}
|
|
|
|
function readManualAccountsConfig(value: unknown): CodexPoolManualAccountsConfig {
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.manualAccounts must be a YAML object`);
|
|
const bindingSources = readManualBindingSources(value.bindingSources, "manualAccounts.bindingSources");
|
|
const protectedRaw = value.protected;
|
|
if (!Array.isArray(protectedRaw)) throw new Error(`${codexPoolConfigPath}.manualAccounts.protected must be a YAML array`);
|
|
const seen = new Set<string>();
|
|
const protectedAccounts = protectedRaw.map((entry, index): CodexPoolManualAccountProtection => {
|
|
const key = `manualAccounts.protected[${index}]`;
|
|
const accountName = typeof entry === "string"
|
|
? readManualAccountName(entry, key)
|
|
: isRecord(entry)
|
|
? readManualAccountName(entry.accountName, `${key}.accountName`)
|
|
: null;
|
|
if (accountName === null) throw new Error(`${codexPoolConfigPath}.${key} must be an account name string or object with accountName`);
|
|
const normalized = accountName.toLowerCase();
|
|
if (seen.has(normalized)) throw new Error(`${codexPoolConfigPath}.${key}.accountName is duplicated in manualAccounts.protected`);
|
|
seen.add(normalized);
|
|
const reason = isRecord(entry) ? readManualAccountReason(entry.reason, `${key}.reason`) : null;
|
|
const proxyBinding = isRecord(entry) ? readManualAccountProxyBinding(entry.proxyBinding, `${key}.proxyBinding`, bindingSources) : null;
|
|
const groupBinding = isRecord(entry) ? readManualAccountGroupBinding(entry.groupBinding, `${key}.groupBinding`, bindingSources) : null;
|
|
return { accountName, reason, proxyBinding, groupBinding };
|
|
});
|
|
return { bindingSources, protected: protectedAccounts };
|
|
}
|
|
|
|
function readManualBindingSources(value: unknown, key: string): CodexPoolManualBindingSourcesConfig {
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const items: CodexPoolManualBindingSource[] = [];
|
|
const byId: Record<string, CodexPoolManualBindingSource> = {};
|
|
for (const [id, rawSource] of Object.entries(value)) {
|
|
const path = `${key}.${id}`;
|
|
validateManualBindingSourceId(id, path);
|
|
if (!isRecord(rawSource)) throw new Error(`${codexPoolConfigPath}.${path} must be a YAML object`);
|
|
const source: CodexPoolManualBindingSource = {
|
|
id,
|
|
enabled: readBooleanConfig(rawSource.enabled, `${path}.enabled`),
|
|
kind: readManualBindingKind(rawSource.kind, `${path}.kind`),
|
|
provider: readManualBindingProvider(rawSource.provider, `${path}.provider`),
|
|
description: readManualAccountReason(rawSource.description, `${path}.description`),
|
|
};
|
|
if (source.kind === "proxy" && source.provider !== "target-egress-proxy") {
|
|
throw new Error(`${codexPoolConfigPath}.${path}.provider must be target-egress-proxy for kind=proxy`);
|
|
}
|
|
if (source.kind === "group" && source.provider !== "pool-group") {
|
|
throw new Error(`${codexPoolConfigPath}.${path}.provider must be pool-group for kind=group`);
|
|
}
|
|
byId[id] = source;
|
|
items.push(source);
|
|
}
|
|
if (items.length === 0) throw new Error(`${codexPoolConfigPath}.${key} must declare at least one source`);
|
|
return { items, byId };
|
|
}
|
|
|
|
function readManualAccountProxyBinding(value: unknown, key: string, bindingSources: CodexPoolManualBindingSourcesConfig): CodexPoolManualAccountProxyBinding | null {
|
|
if (value === undefined || value === null) return null;
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`);
|
|
const source = readManualBindingSourceRef(value.source, `${key}.source`, bindingSources, "proxy", enabled);
|
|
const proxyName = stringValue(value.proxyName);
|
|
if (proxyName === null || proxyName.trim().length === 0) throw new Error(`${codexPoolConfigPath}.${key}.proxyName is required`);
|
|
validateProxyName(proxyName, `${key}.proxyName`);
|
|
return {
|
|
enabled,
|
|
source,
|
|
proxyName,
|
|
};
|
|
}
|
|
|
|
function readManualAccountGroupBinding(value: unknown, key: string, bindingSources: CodexPoolManualBindingSourcesConfig): CodexPoolManualAccountGroupBinding | null {
|
|
if (value === undefined || value === null) return null;
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`);
|
|
const source = readManualBindingSourceRef(value.source, `${key}.source`, bindingSources, "group", enabled);
|
|
return {
|
|
enabled,
|
|
source,
|
|
};
|
|
}
|
|
|
|
function readManualBindingSourceRef(value: unknown, key: string, bindingSources: CodexPoolManualBindingSourcesConfig, expectedKind: CodexPoolManualBindingKind, bindingEnabled: boolean): string {
|
|
const sourceId = stringValue(value);
|
|
if (sourceId === null) throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
validateManualBindingSourceId(sourceId, key);
|
|
const source = bindingSources.byId[sourceId];
|
|
if (source === undefined) throw new Error(`${codexPoolConfigPath}.${key} references unknown manualAccounts.bindingSources id ${sourceId}`);
|
|
if (source.kind !== expectedKind) throw new Error(`${codexPoolConfigPath}.${key} references ${sourceId} with kind=${source.kind}; expected ${expectedKind}`);
|
|
if (bindingEnabled && !source.enabled) throw new Error(`${codexPoolConfigPath}.${key} references disabled manualAccounts.bindingSources.${sourceId}`);
|
|
return sourceId;
|
|
}
|
|
|
|
function readManualBindingKind(value: unknown, key: string): CodexPoolManualBindingKind {
|
|
const text = stringValue(value);
|
|
if (text !== "proxy" && text !== "group") throw new Error(`${codexPoolConfigPath}.${key} must be proxy or group`);
|
|
return text;
|
|
}
|
|
|
|
function readManualBindingProvider(value: unknown, key: string): CodexPoolManualBindingProvider {
|
|
const text = stringValue(value);
|
|
if (text !== "target-egress-proxy" && text !== "pool-group") throw new Error(`${codexPoolConfigPath}.${key} must be target-egress-proxy or pool-group`);
|
|
return text;
|
|
}
|
|
|
|
function validateManualBindingSourceId(value: string, key: string): void {
|
|
if (!/^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported source id format`);
|
|
}
|
|
|
|
function readManualAccountName(value: unknown, key: string): string | null {
|
|
const text = stringValue(value)?.trim() ?? null;
|
|
if (text === null || text.length === 0) return null;
|
|
if (text.length > 256) throw new Error(`${codexPoolConfigPath}.${key} must be at most 256 characters`);
|
|
if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`);
|
|
if (!/^[^<>"'`\\]+$/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} contains unsupported characters`);
|
|
return text;
|
|
}
|
|
|
|
function readManualAccountReason(value: unknown, key: string): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
const text = stringValue(value)?.trim() ?? null;
|
|
if (text === null || text.length === 0) return null;
|
|
if (text.length > 240) throw new Error(`${codexPoolConfigPath}.${key} must be at most 240 characters`);
|
|
if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`);
|
|
return text;
|
|
}
|
|
|
|
function assertProtectedManualAccountsNotManaged(profiles: CodexPoolProfileConfig[], manualAccounts: CodexPoolManualAccountsConfig): void {
|
|
const protectedNames = new Set(manualAccounts.protected.map((account) => account.accountName.toLowerCase()));
|
|
for (const [index, profile] of profiles.entries()) {
|
|
const accountName = profile.accountName;
|
|
if (accountName !== null && protectedNames.has(accountName.toLowerCase())) {
|
|
throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}].accountName is listed in manualAccounts.protected; protected manual accounts must not be YAML-managed`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function rejectSchedulableYamlField(value: Record<string, unknown>, key: string): void {
|
|
if (Object.prototype.hasOwnProperty.call(value, "schedulable")) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}.schedulable is process control, not durable YAML config; use codex-pool sync actions instead`);
|
|
}
|
|
}
|
|
|
|
function readOpenAIResponsesWebSocketsV2Mode(value: unknown, key: string): OpenAIResponsesWebSocketsV2Mode | null {
|
|
if (value === undefined || value === null) return null;
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`);
|
|
if (text === "off" || text === "ctx_pool" || text === "passthrough") return text;
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be one of off, ctx_pool, passthrough`);
|
|
}
|
|
|
|
function readUpstreamUserAgent(value: unknown, key: string): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`);
|
|
if (text.length > 512) throw new Error(`${codexPoolConfigPath}.${key} must be at most 512 characters`);
|
|
if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`);
|
|
return text;
|
|
}
|
|
|
|
function readTrustUpstream(value: unknown, key: string): boolean {
|
|
if (value === undefined || value === null) return false;
|
|
const parsed = booleanValue(value);
|
|
if (parsed === null) throw new Error(`${codexPoolConfigPath}.${key} must be a boolean`);
|
|
return parsed;
|
|
}
|
|
|
|
function readSentinelProtectPolicy(value: unknown, key: string, fallback?: CodexSentinelProtectPolicy): CodexSentinelProtectPolicy {
|
|
if (value === undefined || value === null) {
|
|
if (fallback !== undefined) return fallback;
|
|
throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
}
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback?.enabled);
|
|
const required = ["consecutiveFailures", "initialRetryDelaySeconds", "maxRetryDelaySeconds", "backoffMultiplier"];
|
|
for (const field of required) {
|
|
if (fallback === undefined && (value[field] === undefined || value[field] === null)) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}.${field} is required`);
|
|
}
|
|
if (enabled && value[field] === undefined && fallback === undefined) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}.${field} is required when enabled=true`);
|
|
}
|
|
}
|
|
const consecutiveFailures = value.consecutiveFailures === undefined || value.consecutiveFailures === null
|
|
? fallback?.consecutiveFailures
|
|
: readBoundedInteger(value.consecutiveFailures, `${key}.consecutiveFailures`, 1, 20);
|
|
const initialRetryDelaySeconds = value.initialRetryDelaySeconds === undefined || value.initialRetryDelaySeconds === null
|
|
? fallback?.initialRetryDelaySeconds
|
|
: readBoundedInteger(value.initialRetryDelaySeconds, `${key}.initialRetryDelaySeconds`, 1, 3600);
|
|
const maxRetryDelaySeconds = value.maxRetryDelaySeconds === undefined || value.maxRetryDelaySeconds === null
|
|
? fallback?.maxRetryDelaySeconds
|
|
: readBoundedInteger(value.maxRetryDelaySeconds, `${key}.maxRetryDelaySeconds`, 1, 3600);
|
|
const backoffMultiplier = value.backoffMultiplier === undefined || value.backoffMultiplier === null
|
|
? fallback?.backoffMultiplier
|
|
: readBoundedInteger(value.backoffMultiplier, `${key}.backoffMultiplier`, 1, 10);
|
|
if (consecutiveFailures === undefined || initialRetryDelaySeconds === undefined || maxRetryDelaySeconds === undefined || backoffMultiplier === undefined) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} is missing required sentinel protection fields`);
|
|
}
|
|
if (maxRetryDelaySeconds < initialRetryDelaySeconds) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}.maxRetryDelaySeconds must be >= initialRetryDelaySeconds`);
|
|
}
|
|
return {
|
|
enabled,
|
|
consecutiveFailures,
|
|
initialRetryDelaySeconds,
|
|
maxRetryDelaySeconds,
|
|
backoffMultiplier,
|
|
};
|
|
}
|
|
|
|
function readBoundedInteger(value: unknown, key: string, min: number, max: number): number {
|
|
const parsed = numberValue(value);
|
|
if (parsed === null || !Number.isInteger(parsed) || parsed < min || parsed > max) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from ${min} to ${max}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function readAccountPriority(value: unknown, key: string, fallback?: number): number {
|
|
if (value === undefined || value === null) {
|
|
if (fallback !== undefined) return fallback;
|
|
throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
}
|
|
const priority = numberValue(value);
|
|
if (priority === null || !Number.isInteger(priority) || priority < 0 || priority > 1000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 0 to 1000`);
|
|
}
|
|
return priority;
|
|
}
|
|
|
|
function readAccountCapacity(value: unknown, key: string): number {
|
|
const capacity = numberValue(value);
|
|
if (capacity === null || !Number.isInteger(capacity) || capacity < 1 || capacity > 1000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`);
|
|
}
|
|
return capacity;
|
|
}
|
|
|
|
function readOwnerConcurrency(value: unknown, key: string): number {
|
|
const concurrency = numberValue(value);
|
|
if (concurrency === null || !Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 100000`);
|
|
}
|
|
return concurrency;
|
|
}
|
|
|
|
function readAccountLoadFactor(value: unknown, key: string): number {
|
|
const loadFactor = numberValue(value);
|
|
if (loadFactor === null || !Number.isInteger(loadFactor) || loadFactor < 1 || loadFactor > 1000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`);
|
|
}
|
|
return loadFactor;
|
|
}
|
|
|
|
function readCaddyTimeoutSeconds(value: unknown, key: string): number {
|
|
if (value === undefined || value === null) throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
const seconds = numberValue(value);
|
|
if (seconds === null || !Number.isInteger(seconds) || seconds < 30 || seconds > 900) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 30 to 900`);
|
|
}
|
|
return seconds;
|
|
}
|
|
|
|
function readCaddyEdgeRetryConfig(value: unknown, key: string): CodexPoolCaddyEdgeRetryConfig {
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`);
|
|
|
|
const tryDurationSeconds = readPositiveInteger(value.tryDurationSeconds, `${key}.tryDurationSeconds`);
|
|
const tryIntervalMilliseconds = readPositiveInteger(value.tryIntervalMilliseconds, `${key}.tryIntervalMilliseconds`);
|
|
const retryMatchValue = isRecord(value.retryMatch) ? value.retryMatch : null;
|
|
if (retryMatchValue === null) throw new Error(`${codexPoolConfigPath}.${key}.retryMatch must be a YAML object`);
|
|
const retryMatch = {
|
|
methods: readCaddyRetryMethods(retryMatchValue.methods, `${key}.retryMatch.methods`),
|
|
paths: readCaddyRetryPaths(retryMatchValue.paths, `${key}.retryMatch.paths`),
|
|
};
|
|
if (retryMatch.methods.length === 0 && retryMatch.paths.length === 0) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}.retryMatch must include at least one method or path matcher when enabled=true`);
|
|
}
|
|
return { enabled, tryDurationSeconds, tryIntervalMilliseconds, retryMatch };
|
|
}
|
|
|
|
function readPositiveInteger(value: unknown, key: string): number {
|
|
const parsed = numberValue(value);
|
|
if (parsed === null || !Number.isInteger(parsed) || parsed < 1) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be a positive integer`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function readPositiveIntegerConfig(value: unknown, key: string): number {
|
|
if (value === undefined || value === null) throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
return readPositiveInteger(value, key);
|
|
}
|
|
|
|
function readCaddyRetryMethods(value: unknown, key: string): string[] {
|
|
if (value === undefined || value === null) return [];
|
|
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
|
const seen = new Set<string>();
|
|
const methods: string[] = [];
|
|
for (const item of value) {
|
|
const method = stringValue(item)?.toUpperCase() ?? null;
|
|
if (method === null || !/^[A-Z][A-Z0-9_-]*$/u.test(method)) throw new Error(`${codexPoolConfigPath}.${key} entries must be HTTP method tokens`);
|
|
if (seen.has(method)) continue;
|
|
seen.add(method);
|
|
methods.push(method);
|
|
}
|
|
return methods;
|
|
}
|
|
|
|
function readCaddyRetryPaths(value: unknown, key: string): string[] {
|
|
if (value === undefined || value === null) return [];
|
|
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
|
const seen = new Set<string>();
|
|
const paths: string[] = [];
|
|
for (const item of value) {
|
|
const path = stringValue(item);
|
|
if (path === null || !path.startsWith("/") || /[\r\n]/u.test(path)) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} entries must be Caddy path matchers starting with / and without newlines`);
|
|
}
|
|
if (seen.has(path)) continue;
|
|
seen.add(path);
|
|
paths.push(path);
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
function readTempUnschedulablePolicy(value: unknown, key: string, fallback?: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
|
|
if (value === undefined || value === null) {
|
|
if (fallback !== undefined) return cloneTempUnschedulablePolicy(fallback);
|
|
throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
}
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback?.enabled);
|
|
let rules: CodexTempUnschedulableRule[];
|
|
if (value.rules === undefined || value.rules === null) {
|
|
if (fallback === undefined) throw new Error(`${codexPoolConfigPath}.${key}.rules is required`);
|
|
rules = cloneTempUnschedulablePolicy(fallback).rules;
|
|
} else {
|
|
rules = readTempUnschedulableRules(value.rules, `${key}.rules`);
|
|
}
|
|
if (enabled && rules.length === 0) throw new Error(`${codexPoolConfigPath}.${key}.rules must not be empty when enabled=true`);
|
|
return { enabled, rules };
|
|
}
|
|
|
|
function readTempUnschedulableRules(value: unknown, key: string): CodexTempUnschedulableRule[] {
|
|
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
|
return value.map((entry, index) => {
|
|
if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.${key}[${index}] must be an object`);
|
|
const statusCode = numberValue(entry.statusCode ?? entry.errorCode);
|
|
if (statusCode === null || !Number.isInteger(statusCode) || statusCode < 100 || statusCode > 599) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}[${index}].statusCode must be an HTTP status code from 100 to 599`);
|
|
}
|
|
const durationMinutes = numberValue(entry.durationMinutes);
|
|
if (durationMinutes === null || !Number.isInteger(durationMinutes) || durationMinutes < 1 || durationMinutes > 1440) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}[${index}].durationMinutes must be an integer from 1 to 1440`);
|
|
}
|
|
const keywords = readTempUnschedulableKeywords(entry.keywords, `${key}[${index}].keywords`);
|
|
const description = stringValue(entry.description);
|
|
if (description !== null && description.length > 240) throw new Error(`${codexPoolConfigPath}.${key}[${index}].description must be at most 240 characters`);
|
|
return { statusCode, keywords, durationMinutes, description };
|
|
});
|
|
}
|
|
|
|
function readTempUnschedulableKeywords(value: unknown, key: string): string[] {
|
|
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
|
const seen = new Set<string>();
|
|
const keywords: string[] = [];
|
|
for (const item of value) {
|
|
const keyword = stringValue(item);
|
|
if (keyword === null) throw new Error(`${codexPoolConfigPath}.${key} entries must be non-empty strings`);
|
|
if (keyword.length > 120) throw new Error(`${codexPoolConfigPath}.${key} entries must be at most 120 characters`);
|
|
if (/[\r\n]/u.test(keyword)) throw new Error(`${codexPoolConfigPath}.${key} entries must not contain newlines`);
|
|
const normalized = keyword.toLowerCase();
|
|
if (seen.has(normalized)) continue;
|
|
seen.add(normalized);
|
|
keywords.push(keyword);
|
|
}
|
|
if (keywords.length === 0) throw new Error(`${codexPoolConfigPath}.${key} must not be empty`);
|
|
return keywords;
|
|
}
|
|
|
|
function cloneTempUnschedulablePolicy(policy: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
|
|
return {
|
|
enabled: policy.enabled,
|
|
rules: policy.rules.map((rule) => ({
|
|
statusCode: rule.statusCode,
|
|
keywords: [...rule.keywords],
|
|
durationMinutes: rule.durationMinutes,
|
|
description: rule.description,
|
|
})),
|
|
};
|
|
}
|
|
|
|
function readBooleanConfig(value: unknown, key: string, fallback?: boolean): boolean {
|
|
if (value === undefined || value === null) {
|
|
if (fallback !== undefined) return fallback;
|
|
throw new Error(`${codexPoolConfigPath}.${key} is required`);
|
|
}
|
|
const parsed = booleanValue(value);
|
|
if (parsed === null) throw new Error(`${codexPoolConfigPath}.${key} must be a boolean`);
|
|
return parsed;
|
|
}
|
|
|
|
function readPublicExposureConfig(value: unknown): CodexPoolPublicExposureConfig {
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.publicExposure must be a YAML object`);
|
|
const masterFrpsValue = requiredRecordConfigField(value, "masterFrps", "publicExposure");
|
|
const masterCaddyValue = requiredRecordConfigField(value, "masterCaddy", "publicExposure");
|
|
const config: CodexPoolPublicExposureConfig = {
|
|
enabled: readBooleanConfig(value.enabled, "publicExposure.enabled"),
|
|
proxyName: requiredStringConfigField(value, "proxyName", "publicExposure"),
|
|
configMapName: requiredStringConfigField(value, "configMapName", "publicExposure"),
|
|
deploymentName: requiredStringConfigField(value, "deploymentName", "publicExposure"),
|
|
frpcImage: requiredStringConfigField(value, "frpcImage", "publicExposure"),
|
|
serverAddr: requiredStringConfigField(value, "serverAddr", "publicExposure"),
|
|
serverPort: readRequiredPort(value.serverPort, "publicExposure.serverPort"),
|
|
remotePort: readRequiredPort(value.remotePort, "publicExposure.remotePort"),
|
|
localIP: requiredStringConfigField(value, "localIP", "publicExposure"),
|
|
localPort: readRequiredPort(value.localPort, "publicExposure.localPort"),
|
|
publicBaseUrl: readRequiredBaseUrl(value.publicBaseUrl, "publicExposure.publicBaseUrl"),
|
|
masterBaseUrl: readRequiredBaseUrl(value.masterBaseUrl, "publicExposure.masterBaseUrl"),
|
|
masterFrps: {
|
|
configPath: requiredStringConfigField(masterFrpsValue, "configPath", "publicExposure.masterFrps"),
|
|
containerName: requiredStringConfigField(masterFrpsValue, "containerName", "publicExposure.masterFrps"),
|
|
},
|
|
masterCaddy: {
|
|
enabled: readBooleanConfig(masterCaddyValue.enabled, "publicExposure.masterCaddy.enabled"),
|
|
domain: requiredStringConfigField(masterCaddyValue, "domain", "publicExposure.masterCaddy"),
|
|
configPath: requiredStringConfigField(masterCaddyValue, "configPath", "publicExposure.masterCaddy"),
|
|
serviceName: requiredStringConfigField(masterCaddyValue, "serviceName", "publicExposure.masterCaddy"),
|
|
upstreamBaseUrl: readRequiredBaseUrl(masterCaddyValue.upstreamBaseUrl, "publicExposure.masterCaddy.upstreamBaseUrl"),
|
|
responseHeaderTimeoutSeconds: readCaddyTimeoutSeconds(
|
|
masterCaddyValue.responseHeaderTimeoutSeconds,
|
|
"publicExposure.masterCaddy.responseHeaderTimeoutSeconds",
|
|
),
|
|
edgeRetry: readCaddyEdgeRetryConfig(
|
|
masterCaddyValue.edgeRetry,
|
|
"publicExposure.masterCaddy.edgeRetry",
|
|
),
|
|
},
|
|
};
|
|
validateKubernetesName(config.configMapName, "publicExposure.configMapName", true);
|
|
validateKubernetesName(config.deploymentName, "publicExposure.deploymentName", true);
|
|
validateProxyName(config.proxyName, "publicExposure.proxyName");
|
|
validatePort(config.serverPort, "publicExposure.serverPort");
|
|
validatePort(config.remotePort, "publicExposure.remotePort");
|
|
validatePort(config.localPort, "publicExposure.localPort");
|
|
if (!/^[A-Za-z0-9._:/-]+$/u.test(config.frpcImage)) throw new Error(`${codexPoolConfigPath}.publicExposure.frpcImage has an unsupported format`);
|
|
if (!/^[A-Za-z0-9._:-]+$/u.test(config.serverAddr)) throw new Error(`${codexPoolConfigPath}.publicExposure.serverAddr has an unsupported format`);
|
|
if (!/^[A-Za-z0-9._:-]+$/u.test(config.localIP)) throw new Error(`${codexPoolConfigPath}.publicExposure.localIP has an unsupported format`);
|
|
if (!config.masterFrps.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterFrps.configPath must be absolute`);
|
|
validateProxyName(config.masterFrps.containerName, "publicExposure.masterFrps.containerName");
|
|
validatePublicHostname(config.masterCaddy.domain, "publicExposure.masterCaddy.domain");
|
|
if (!config.masterCaddy.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.configPath must be absolute`);
|
|
validateProxyName(config.masterCaddy.serviceName, "publicExposure.masterCaddy.serviceName");
|
|
const upstream = new URL(config.masterCaddy.upstreamBaseUrl);
|
|
if (upstream.protocol !== "http:") throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.upstreamBaseUrl must use http:// for the local upstream`);
|
|
return config;
|
|
}
|
|
|
|
function readLocalCodexConfig(value: unknown): CodexPoolLocalCodexConfig {
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.localCodex must be a YAML object`);
|
|
const config = {
|
|
backupSuffix: requiredStringConfigField(value, "backupSuffix", "localCodex"),
|
|
providerName: requiredStringConfigField(value, "providerName", "localCodex"),
|
|
wireApi: requiredStringConfigField(value, "wireApi", "localCodex"),
|
|
modelContextWindow: readPositiveIntegerConfig(value.modelContextWindow, "localCodex.modelContextWindow"),
|
|
modelAutoCompactTokenLimit: readPositiveIntegerConfig(value.modelAutoCompactTokenLimit, "localCodex.modelAutoCompactTokenLimit"),
|
|
supportsWebSockets: readBooleanConfig(value.supportsWebSockets, "localCodex.supportsWebSockets"),
|
|
responsesWebSocketsV2: readBooleanConfig(value.responsesWebSocketsV2, "localCodex.responsesWebSocketsV2"),
|
|
responsesSmokeModel: requiredStringConfigField(value, "responsesSmokeModel", "localCodex"),
|
|
};
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(config.backupSuffix)) throw new Error(`${codexPoolConfigPath}.localCodex.backupSuffix has an unsupported format`);
|
|
validateProxyName(config.providerName, "localCodex.providerName");
|
|
validateProxyName(config.wireApi, "localCodex.wireApi");
|
|
validateModelName(config.responsesSmokeModel, "localCodex.responsesSmokeModel");
|
|
return config;
|
|
}
|
|
|
|
function validateCodexFileName(value: string, key: string): void {
|
|
if (!/^(config\.toml|config\.toml\.[A-Za-z0-9._-]+|auth\.json|auth\.json\.[A-Za-z0-9._-]+)$/u.test(value)) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} has an unsupported file name`);
|
|
}
|
|
}
|
|
|
|
function validateProxyName(value: string, key: string): void {
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`);
|
|
}
|
|
|
|
function validateModelName(value: string, key: string): void {
|
|
if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported model name`);
|
|
}
|
|
|
|
function validatePublicHostname(value: string, key: string): void {
|
|
if (value.length > 253 || !/^[A-Za-z0-9.-]+$/u.test(value) || value.startsWith(".") || value.endsWith(".")) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} has an unsupported hostname format`);
|
|
}
|
|
}
|
|
|
|
function validatePort(value: number, key: string): void {
|
|
if (!Number.isInteger(value) || value < 1 || value > 65535) throw new Error(`${codexPoolConfigPath}.${key} must be an integer port`);
|
|
}
|
|
|
|
function readAuthAPIKey(authPath: string): { apiKey: string | null; shape: string } {
|
|
if (!existsSync(authPath)) return { apiKey: null, shape: "missing" };
|
|
const parsed = JSON.parse(readFileSync(authPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) return { apiKey: null, shape: "non-object" };
|
|
const value = parsed.OPENAI_API_KEY;
|
|
const shape = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
|
|
if (typeof value === "string" && value.length > 0) return { apiKey: value, shape };
|
|
return { apiKey: null, shape };
|
|
}
|
|
|
|
function uniqueAccountName(profile: string, seen: Set<string>): string {
|
|
const normalized = profile
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9._-]+/gu, "-")
|
|
.replace(/^-+|-+$/gu, "") || "default";
|
|
let candidate = `unidesk-codex-${normalized}`;
|
|
let counter = 2;
|
|
while (seen.has(candidate)) {
|
|
candidate = `unidesk-codex-${normalized}-${counter}`;
|
|
counter += 1;
|
|
}
|
|
seen.add(candidate);
|
|
return candidate;
|
|
}
|
|
|
|
function redactProfile(profile: CodexProfile): Record<string, unknown> {
|
|
return {
|
|
profile: profile.profile,
|
|
accountName: profile.accountName,
|
|
configFile: profile.configFile,
|
|
authFile: profile.authFile,
|
|
provider: profile.provider || null,
|
|
baseUrl: profile.baseUrl || null,
|
|
wireApi: profile.wireApi,
|
|
model: profile.model,
|
|
envKey: profile.envKey,
|
|
apiKeySource: profile.apiKeySource,
|
|
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
trustUpstream: profile.trustUpstream,
|
|
sentinelProtect: profile.sentinelProtect,
|
|
priority: profile.priority,
|
|
capacity: profile.capacity,
|
|
loadFactor: profile.loadFactor,
|
|
tempUnschedulable: tempUnschedulableSummary(profile.tempUnschedulable),
|
|
apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0,
|
|
apiKeyBytes: profile.apiKey === null ? 0 : Buffer.byteLength(profile.apiKey, "utf8"),
|
|
apiKeyFingerprint: profile.apiKey === null ? null : fingerprint(profile.apiKey),
|
|
authOpenAIKeyShape: profile.authOpenAIKeyShape,
|
|
ok: profile.ok,
|
|
error: profile.error,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactProfile(profile: CodexProfile): Record<string, unknown> {
|
|
return {
|
|
profile: profile.profile,
|
|
accountName: profile.accountName,
|
|
provider: profile.provider || null,
|
|
model: profile.model,
|
|
priority: profile.priority,
|
|
trustUpstream: profile.trustUpstream,
|
|
sentinelProtectEnabled: profile.sentinelProtect.enabled,
|
|
sentinelProtectConsecutiveFailures: profile.sentinelProtect.enabled ? profile.sentinelProtect.consecutiveFailures : undefined,
|
|
capacity: profile.capacity,
|
|
loadFactor: profile.loadFactor,
|
|
tempUnschedulableEnabled: profile.tempUnschedulable.enabled,
|
|
tempUnschedulableRuleCount: profile.tempUnschedulable.rules.length,
|
|
apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0,
|
|
ok: profile.ok,
|
|
error: profile.error,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function renderSub2ApiTempUnschedulableCredentials(policy: CodexTempUnschedulablePolicy): Record<string, unknown> {
|
|
if (!policy.enabled) return {};
|
|
return {
|
|
temp_unschedulable_enabled: policy.enabled,
|
|
temp_unschedulable_rules: policy.rules.map((rule) => ({
|
|
error_code: rule.statusCode,
|
|
keywords: [...rule.keywords],
|
|
duration_minutes: rule.durationMinutes,
|
|
description: rule.description ?? "",
|
|
})),
|
|
};
|
|
}
|
|
|
|
function tempUnschedulableSummary(policy: CodexTempUnschedulablePolicy): Record<string, unknown> {
|
|
return {
|
|
enabled: policy.enabled,
|
|
ruleCount: policy.rules.length,
|
|
statusCodes: policy.rules.map((rule) => rule.statusCode),
|
|
};
|
|
}
|
|
|
|
function codexPoolConfigSummary(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record<string, unknown> {
|
|
const accountCapacityTotal = desiredAccountCapacityTotal(pool);
|
|
return {
|
|
version: pool.version,
|
|
kind: pool.kind,
|
|
metadata: pool.metadata,
|
|
groupName: pool.groupName,
|
|
groupDescription: pool.groupDescription,
|
|
apiKeyName: pool.apiKeyName,
|
|
apiKeySecretName: pool.apiKeySecretName,
|
|
apiKeySecretKey: pool.apiKeySecretKey,
|
|
adminEmailDefault: pool.adminEmailDefault,
|
|
minOwnerBalanceUsd: pool.minOwnerBalanceUsd,
|
|
minOwnerConcurrency: pool.minOwnerConcurrency,
|
|
minOwnerConcurrencySource: pool.minOwnerConcurrencySource,
|
|
accountCapacityTotal,
|
|
defaultAccountPriority: pool.defaultAccountPriority,
|
|
defaultAccountCapacity: pool.defaultAccountCapacity,
|
|
defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
|
|
defaultTempUnschedulable: tempUnschedulableSummary(pool.defaultTempUnschedulable),
|
|
defaultSentinelProtect: pool.defaultSentinelProtect,
|
|
profileCount: pool.profiles.length,
|
|
manualAccounts: {
|
|
bindingSources: pool.manualAccounts.bindingSources.items.map(manualBindingSourcePlan),
|
|
protectedCount: pool.manualAccounts.protected.length,
|
|
protected: pool.manualAccounts.protected,
|
|
controlPolicy: "manual accounts are not created, credential-updated, pruned, probed, or frozen by UniDesk codex-pool sync/sentinel; optional proxy_id and pool group membership bindings are narrow YAML-controlled exceptions",
|
|
},
|
|
publicExposure: targetPublicExposureSummary(target),
|
|
localCodex: pool.localCodex,
|
|
sentinel: codexPoolSentinelSummary(pool.sentinel),
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool plan --full",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactText(value: unknown, limit = 260): string {
|
|
if (typeof value !== "string") return "";
|
|
return value.length > limit ? `${value.slice(0, limit)}…` : value;
|
|
}
|
|
|
|
function recordArray(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.filter(isRecord) : [];
|
|
}
|
|
|
|
function pickSummaryFields(item: Record<string, unknown>, keys: string[]): Record<string, unknown> {
|
|
const result: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
if (item[key] !== undefined) result[key] = item[key];
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function compactStatusBlock(block: unknown, keys: string[]): Record<string, unknown> | null {
|
|
if (!isRecord(block)) return null;
|
|
const items = recordArray(block.items);
|
|
const attentionItems = items
|
|
.filter((item) => item.ok === false)
|
|
.map((item) => pickSummaryFields(item, keys));
|
|
return {
|
|
ok: block.ok,
|
|
desired: block.desired,
|
|
missing: block.missing,
|
|
mismatched: block.mismatched,
|
|
totals: block.totals,
|
|
itemCount: items.length,
|
|
attentionItems,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactTempUnschedulableStatus(block: unknown): Record<string, unknown> | null {
|
|
if (!isRecord(block)) return null;
|
|
const items = recordArray(block.items);
|
|
const compactItems = items.map((item) => {
|
|
const result = pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"status",
|
|
"schedulable",
|
|
"tempUnschedulableUntil",
|
|
"tempUnschedulableSet",
|
|
"ok",
|
|
]);
|
|
if (item.tempUnschedulableReasonPreview !== undefined) {
|
|
result.tempUnschedulableReason = tempUnschedulableReasonSummary(item.tempUnschedulableReasonPreview);
|
|
}
|
|
return result;
|
|
});
|
|
const frozenItems = compactItems.filter((item) => item.tempUnschedulableSet === true);
|
|
const focusedFrozenItems = uniqueByAccountName([
|
|
...frozenItems.filter((item) => isRecord(item.tempUnschedulableReason) && item.tempUnschedulableReason.statusCode === 400),
|
|
...frozenItems.filter((item) => item.schedulable === false),
|
|
...frozenItems,
|
|
]).slice(0, 6);
|
|
return {
|
|
ok: block.ok,
|
|
desired: block.desired,
|
|
enabledCount: Array.isArray(block.enabled) ? block.enabled.length : undefined,
|
|
missing: block.missing,
|
|
mismatched: block.mismatched,
|
|
itemCount: compactItems.length,
|
|
frozenCount: frozenItems.length,
|
|
frozenShown: focusedFrozenItems.length,
|
|
frozen: focusedFrozenItems.map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"schedulable",
|
|
"tempUnschedulableUntil",
|
|
"tempUnschedulableReason",
|
|
])),
|
|
manuallyUnschedulable: compactItems
|
|
.filter((item) => item.schedulable === false && item.tempUnschedulableSet !== true)
|
|
.map((item) => pickSummaryFields(item, ["accountName", "accountId", "status", "schedulable"])),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactManualAccounts(block: unknown): Record<string, unknown> | null {
|
|
if (!isRecord(block)) return null;
|
|
const items = recordArray(block.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"reason",
|
|
"ok",
|
|
"exists",
|
|
"accountId",
|
|
"status",
|
|
"schedulable",
|
|
"inYamlProfiles",
|
|
"runtimeMarkedUnideskManaged",
|
|
"proxyBinding",
|
|
"groupBinding",
|
|
"controlPolicy",
|
|
]));
|
|
const proxySync = isRecord(block.proxySync)
|
|
? {
|
|
ok: block.proxySync.ok,
|
|
itemCount: block.proxySync.itemCount,
|
|
items: recordArray(block.proxySync.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"enabled",
|
|
"ok",
|
|
"action",
|
|
"proxyAction",
|
|
"expectedProxyName",
|
|
"proxyId",
|
|
"runtimeProxyId",
|
|
"runtimeProxyName",
|
|
"bindingAligned",
|
|
"controlPolicy",
|
|
])),
|
|
valuesPrinted: false,
|
|
}
|
|
: undefined;
|
|
const groupSync = isRecord(block.groupSync)
|
|
? {
|
|
ok: block.groupSync.ok,
|
|
itemCount: block.groupSync.itemCount,
|
|
items: recordArray(block.groupSync.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"enabled",
|
|
"ok",
|
|
"action",
|
|
"source",
|
|
"poolGroupName",
|
|
"poolGroupId",
|
|
"bindingAligned",
|
|
"controlPolicy",
|
|
])),
|
|
valuesPrinted: false,
|
|
}
|
|
: undefined;
|
|
return {
|
|
ok: block.ok,
|
|
protectedCount: block.protectedCount,
|
|
items,
|
|
proxySync,
|
|
groupSync,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function tempUnschedulableReasonSummary(value: unknown): Record<string, unknown> {
|
|
const reason = compactText(value, 180);
|
|
const statusCodeMatch = reason.match(/"status_code":(\d{3})/u) ?? reason.match(/OpenAI\s+(\d{3})/u) ?? reason.match(/\((\d{3})\)/u);
|
|
const keywordMatch = reason.match(/"matched_keyword":"([^"]+)"/u);
|
|
const summary: Record<string, unknown> = {
|
|
statusCode: statusCodeMatch ? Number(statusCodeMatch[1]) : undefined,
|
|
matchedKeyword: keywordMatch ? keywordMatch[1] : undefined,
|
|
};
|
|
if (!statusCodeMatch || !keywordMatch) summary.preview = compactText(reason, 120);
|
|
return summary;
|
|
}
|
|
|
|
function uniqueByAccountName(items: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
const seen = new Set<unknown>();
|
|
const result: Record<string, unknown>[] = [];
|
|
for (const item of items) {
|
|
const key = item.accountName ?? item.accountId;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
result.push(item);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function compactRuntimeCapability(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
const probe = isRecord(block.probe) ? block.probe : {};
|
|
const logEvidence = isRecord(probe.logEvidence) ? probe.logEvidence : {};
|
|
const accountState = isRecord(probe.accountState) ? probe.accountState : (isRecord(probe.badAccountState) ? probe.badAccountState : {});
|
|
const resources = isRecord(block.resources) ? block.resources : {};
|
|
const requirement = isRecord(block.requirement) ? block.requirement : {};
|
|
return {
|
|
ok: block.ok,
|
|
required: block.required,
|
|
capability: block.capability,
|
|
outcome: block.outcome,
|
|
requirement: Object.keys(requirement).length === 0 ? undefined : {
|
|
statusCode: requirement.statusCode,
|
|
representativeKeyword: requirement.representativeKeyword,
|
|
durationMinutes: requirement.durationMinutes,
|
|
sourceAccountName: requirement.sourceAccountName,
|
|
},
|
|
requestEvidence: Object.keys(probe).length === 0 ? undefined : {
|
|
requestId: probe.requestId,
|
|
durationMs: probe.durationMs,
|
|
httpStatus: probe.httpStatus,
|
|
responseOk: probe.responseOk,
|
|
accountState: Object.keys(accountState).length === 0 ? undefined : {
|
|
accountId: accountState.accountId,
|
|
status: accountState.status,
|
|
schedulable: accountState.schedulable,
|
|
tempUnschedulableUntil: accountState.tempUnschedulableUntil,
|
|
tempUnschedulableSet: accountState.tempUnschedulableSet,
|
|
tempUnschedulableReasonPreview: compactText(accountState.tempUnschedulableReasonPreview, 180),
|
|
},
|
|
logEvidence: Object.keys(logEvidence).length === 0 ? undefined : {
|
|
matchedLogLineCount: logEvidence.matchedLogLineCount,
|
|
failovers: logEvidence.failovers,
|
|
final: logEvidence.final,
|
|
},
|
|
bodyPreview: probe.responseOk === false ? compactText(probe.bodyPreview, 240) : undefined,
|
|
},
|
|
resources: Object.keys(resources).length === 0 ? undefined : {
|
|
groupId: resources.groupId,
|
|
accountId: resources.accountId,
|
|
badAccountId: resources.badAccountId,
|
|
goodAccountId: resources.goodAccountId,
|
|
apiKeyId: resources.apiKeyId,
|
|
valuesPrinted: false,
|
|
},
|
|
message: block.message,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactGatewayModels(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
return pickSummaryFields(block, [
|
|
"ok",
|
|
"httpStatus",
|
|
"modelCount",
|
|
"method",
|
|
"serviceDns",
|
|
"valuesPrinted",
|
|
]);
|
|
}
|
|
|
|
function compactGatewayResponses(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
const evidence = isRecord(block.evidence) ? block.evidence : {};
|
|
return {
|
|
ok: block.ok,
|
|
degraded: block.degraded,
|
|
outcome: block.outcome,
|
|
httpStatus: block.httpStatus,
|
|
method: block.method,
|
|
model: block.model,
|
|
requestId: block.requestId,
|
|
durationMs: block.durationMs,
|
|
outputTextPreview: block.outputTextPreview,
|
|
evidence: Object.keys(evidence).length === 0 ? undefined : {
|
|
matchedLogLineCount: evidence.matchedLogLineCount,
|
|
failoverCount: Array.isArray(evidence.failovers) ? evidence.failovers.length : undefined,
|
|
failovers: Array.isArray(evidence.failovers) ? evidence.failovers.slice(-5) : undefined,
|
|
final: evidence.final,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactGatewayResponsesRecent(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
return {
|
|
ok: block.ok,
|
|
degraded: block.degraded,
|
|
window: block.window,
|
|
tailLines: block.tailLines,
|
|
failoverCount: block.failoverCount,
|
|
forwardFailureCount: block.forwardFailureCount,
|
|
finalErrorCount: block.finalErrorCount,
|
|
slowFinalErrorCount: block.slowFinalErrorCount,
|
|
contextCanceledCount: block.contextCanceledCount,
|
|
ignoredProbeNoiseCount: block.ignoredProbeNoiseCount,
|
|
failoverBudgetExhaustedCount: Array.isArray(block.failoverBudgetExhausted) ? block.failoverBudgetExhausted.length : undefined,
|
|
failoverBudgetExhausted: Array.isArray(block.failoverBudgetExhausted) ? block.failoverBudgetExhausted.slice(-3).reverse() : undefined,
|
|
recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined,
|
|
recentForwardFailures: Array.isArray(block.recentForwardFailures) ? block.recentForwardFailures.slice(-4).reverse().map((item) => ({
|
|
...item,
|
|
errorPreview: compactText(item.errorPreview, 220),
|
|
})) : undefined,
|
|
recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-3).reverse() : undefined,
|
|
recentSlowFinalErrors: Array.isArray(block.recentSlowFinalErrors) ? block.recentSlowFinalErrors.slice(-3).reverse() : undefined,
|
|
recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-3).reverse() : undefined,
|
|
logsExitCode: block.logsExitCode,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactGatewayCompactRecent(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
return {
|
|
ok: block.ok,
|
|
degraded: block.degraded,
|
|
window: block.window,
|
|
tailLines: block.tailLines,
|
|
failureCount: block.failureCount,
|
|
successCount: block.successCount,
|
|
failoverCount: block.failoverCount,
|
|
finalErrorCount: block.finalErrorCount,
|
|
contextCanceledCount: block.contextCanceledCount,
|
|
recentFailures: Array.isArray(block.recentFailures) ? block.recentFailures.slice(-2).reverse() : undefined,
|
|
recentSuccesses: Array.isArray(block.recentSuccesses) ? block.recentSuccesses.slice(-2).reverse() : undefined,
|
|
recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined,
|
|
recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-2).reverse() : undefined,
|
|
recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-2).reverse() : undefined,
|
|
logsExitCode: block.logsExitCode,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactSentinelErrorDetails(value: unknown): Record<string, unknown> | undefined {
|
|
if (!isRecord(value)) return undefined;
|
|
const body = isRecord(value.body) ? value.body : {};
|
|
const openaiError = isRecord(value.openaiError) ? value.openaiError : {};
|
|
return {
|
|
kind: value.kind,
|
|
statusCode: value.statusCode,
|
|
code: value.code ?? body.code ?? openaiError.code,
|
|
type: value.type ?? body.type ?? openaiError.type,
|
|
bodyHash: value.bodyHash,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactSentinelQuarantine(item: unknown): Record<string, unknown> {
|
|
if (!isRecord(item)) return {};
|
|
const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {};
|
|
return {
|
|
accountName: item.accountName,
|
|
until: item.until,
|
|
applied: item.applied,
|
|
reason: item.reason,
|
|
failureKind: item.failureKind,
|
|
intervalMinutes: item.intervalMinutes,
|
|
sentinelProtect: Object.keys(protect).length > 0 ? {
|
|
enabled: protect.enabled,
|
|
decision: protect.decision,
|
|
failureCount: protect.failureCount,
|
|
threshold: protect.threshold,
|
|
} : undefined,
|
|
error: compactSentinelErrorDetails(item.errorDetails),
|
|
};
|
|
}
|
|
|
|
function compactSentinelRecentAccount(item: unknown): Record<string, unknown> {
|
|
if (!isRecord(item)) return {};
|
|
const action = isRecord(item.action) ? item.action : {};
|
|
const error = compactSentinelErrorDetails(item.errorDetails);
|
|
const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {};
|
|
return {
|
|
accountName: item.accountName,
|
|
lastProbeAt: item.lastProbeAt,
|
|
lastStatus: item.lastStatus,
|
|
nextProbeAfter: item.nextProbeAfter,
|
|
ok: item.ok,
|
|
purpose: item.purpose,
|
|
httpStatus: item.httpStatus,
|
|
durationMs: item.durationMs,
|
|
markerMatched: item.markerMatched,
|
|
sentinelProtect: Object.keys(protect).length > 0 ? {
|
|
enabled: protect.enabled,
|
|
decision: protect.decision,
|
|
protected: protect.protected,
|
|
failureCount: protect.failureCount,
|
|
threshold: protect.threshold,
|
|
} : undefined,
|
|
failureKind: item.failureKind,
|
|
requestShape: item.requestShape,
|
|
action: Object.keys(action).length > 0 ? {
|
|
taken: action.taken,
|
|
type: action.type,
|
|
} : undefined,
|
|
errorStatusCode: error?.statusCode,
|
|
errorCode: error?.code,
|
|
errorBodyHash: error?.bodyHash,
|
|
};
|
|
}
|
|
|
|
function compactSentinelLastRun(value: unknown): Record<string, unknown> | undefined {
|
|
if (!isRecord(value)) return undefined;
|
|
return {
|
|
at: value.at,
|
|
monitorEnabled: value.monitorEnabled,
|
|
actionsEnabled: value.actionsEnabled,
|
|
profileCount: value.profileCount,
|
|
selected: value.selected,
|
|
okCount: value.okCount,
|
|
mismatchCount: value.mismatchCount,
|
|
markerMismatchCount: value.markerMismatchCount,
|
|
transportFailureCount: value.transportFailureCount,
|
|
actionsTaken: value.actionsTaken,
|
|
gatewayFailureMonitor: value.gatewayFailureMonitor,
|
|
selection: value.selection,
|
|
reconcileCount: Array.isArray(value.reconcile) ? value.reconcile.length : undefined,
|
|
};
|
|
}
|
|
|
|
function compactSentinelStatus(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
const runtime = isRecord(block.runtime) ? block.runtime : block;
|
|
const desired = isRecord(runtime.desired) ? runtime.desired : {};
|
|
const cronJob = isRecord(runtime.cronJob) ? runtime.cronJob : {};
|
|
const secret = isRecord(runtime.secret) ? runtime.secret : {};
|
|
const configMap = isRecord(runtime.configMap) ? runtime.configMap : {};
|
|
const state = isRecord(runtime.state) ? runtime.state : {};
|
|
const freezeReassert = isRecord(block.freezeReassert) ? block.freezeReassert : {};
|
|
const qualityGatePrepare = isRecord(block.qualityGatePrepare) ? block.qualityGatePrepare : {};
|
|
const qualityGate = isRecord(block.qualityGate) ? block.qualityGate : {};
|
|
const quarantined = recordArray(state.quarantined).map(compactSentinelQuarantine);
|
|
const recentAccounts = recordArray(state.recentAccounts).map(compactSentinelRecentAccount);
|
|
const recentAttention = recentAccounts.filter((item) => item.ok === false || isRecord(item.action) && item.action.taken === true);
|
|
const recentHealthy = recentAccounts
|
|
.filter((item) => item.ok === true)
|
|
.slice(-3)
|
|
.map((item) => pickSummaryFields(item, ["accountName", "lastProbeAt", "lastStatus", "nextProbeAfter", "ok", "httpStatus", "markerMatched"]));
|
|
return {
|
|
ok: block.ok,
|
|
action: block.action,
|
|
desired: {
|
|
monitorEnabled: desired.monitorEnabled,
|
|
actionsEnabled: desired.actionsEnabled,
|
|
schedule: desired.schedule,
|
|
cronJobName: desired.cronJobName,
|
|
configMapName: desired.configMapName,
|
|
credentialsSecretName: desired.credentialsSecretName,
|
|
stateConfigMapName: desired.stateConfigMapName,
|
|
},
|
|
cronJob: {
|
|
exists: cronJob.exists,
|
|
schedule: cronJob.schedule,
|
|
suspend: cronJob.suspend,
|
|
lastScheduleTime: cronJob.lastScheduleTime,
|
|
active: cronJob.active,
|
|
error: cronJob.error,
|
|
},
|
|
secret: {
|
|
exists: secret.exists,
|
|
profileSecretPresent: secret.profileSecretPresent,
|
|
valuesPrinted: false,
|
|
error: secret.error,
|
|
},
|
|
configMap: {
|
|
exists: configMap.exists,
|
|
configPresent: configMap.configPresent,
|
|
runnerPresent: configMap.runnerPresent,
|
|
error: configMap.error,
|
|
},
|
|
state: {
|
|
exists: state.exists,
|
|
accountCount: state.accountCount,
|
|
quarantinedCount: state.quarantinedCount,
|
|
quarantinedShown: Math.min(quarantined.length, 3),
|
|
quarantined: quarantined.slice(-3),
|
|
recentAccountCount: recentAccounts.length,
|
|
recentAttention: uniqueByAccountName(recentAttention.slice(-3)),
|
|
recentHealthy,
|
|
lastRun: compactSentinelLastRun(state.lastRun),
|
|
error: state.error,
|
|
},
|
|
freezeReassert: Object.keys(freezeReassert).length > 0 ? {
|
|
ok: freezeReassert.ok,
|
|
skipped: freezeReassert.skipped,
|
|
reason: freezeReassert.reason,
|
|
itemCount: freezeReassert.itemCount,
|
|
attentionItems: freezeReassert.attentionItems,
|
|
} : undefined,
|
|
pendingProbePrepare: Object.keys(qualityGatePrepare).length > 0 ? {
|
|
ok: qualityGatePrepare.ok,
|
|
skipped: qualityGatePrepare.skipped,
|
|
reason: qualityGatePrepare.reason,
|
|
changedCount: qualityGatePrepare.changedCount,
|
|
fingerprintOnlyCount: qualityGatePrepare.fingerprintOnlyCount,
|
|
pendingOnly: qualityGatePrepare.pendingOnly,
|
|
items: qualityGatePrepare.items,
|
|
} : undefined,
|
|
pendingProbe: Object.keys(qualityGate).length > 0 ? {
|
|
ok: qualityGate.ok,
|
|
skipped: qualityGate.skipped,
|
|
reason: qualityGate.reason,
|
|
changedCount: qualityGate.changedCount,
|
|
fingerprintOnlyCount: qualityGate.fingerprintOnlyCount,
|
|
clampedCount: qualityGate.clampedCount,
|
|
items: qualityGate.items,
|
|
clampedItems: qualityGate.clampedItems,
|
|
} : undefined,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactSentinelProbeResult(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (parsed === null) return null;
|
|
const probe = isRecord(parsed.probe) ? parsed.probe : {};
|
|
const summary = isRecord(probe.summary) ? probe.summary : {};
|
|
const state = isRecord(parsed.sentinelState) ? parsed.sentinelState : {};
|
|
return {
|
|
ok: parsed.ok,
|
|
mode: parsed.mode,
|
|
namespace: parsed.namespace,
|
|
job: parsed.job,
|
|
requestedAccounts: parsed.requestedAccounts,
|
|
summary: {
|
|
at: summary.at,
|
|
monitorEnabled: summary.monitorEnabled,
|
|
actionsEnabled: summary.actionsEnabled,
|
|
selected: summary.selected,
|
|
okCount: summary.okCount,
|
|
mismatchCount: summary.mismatchCount,
|
|
markerMismatchCount: summary.markerMismatchCount,
|
|
transportFailureCount: summary.transportFailureCount,
|
|
actionsTaken: summary.actionsTaken,
|
|
gatewayFailureMonitor: summary.gatewayFailureMonitor,
|
|
selection: summary.selection,
|
|
},
|
|
results: recordArray(probe.results).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"purpose",
|
|
"ok",
|
|
"markerMatched",
|
|
"sentinelProtect",
|
|
"httpStatus",
|
|
"durationMs",
|
|
"usage",
|
|
"outputHash",
|
|
"outputPreview",
|
|
"responseBodyHash",
|
|
"responseBodyPreview",
|
|
"error",
|
|
"errorDetails",
|
|
"failureKind",
|
|
"sdk",
|
|
"requestShape",
|
|
])),
|
|
actions: recordArray(probe.actions).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"taken",
|
|
"type",
|
|
"error",
|
|
])),
|
|
sentinelState: {
|
|
quarantined: state.quarantined,
|
|
recentAccounts: state.recentAccounts,
|
|
lastRun: state.lastRun,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function renderedCliResult(ok: boolean, command: string, renderedText: string): RenderedCliResult {
|
|
return { ok, command, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
function renderSentinelReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { events: number; full: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
"SUB2API SENTINEL REPORT unavailable",
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const metadata = isRecord(parsed.metadata) ? parsed.metadata : {};
|
|
const cronJob = isRecord(parsed.cronJob) ? parsed.cronJob : {};
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const accounts = recordArray(parsed.accounts);
|
|
const runs = recordArray(parsed.runs);
|
|
const globalLedger = isRecord(parsed.globalLedger) ? parsed.globalLedger : {};
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API SENTINEL",
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`accounts=${summary.accountCount ?? accounts.length}`,
|
|
`quarantined=${summary.quarantinedCount ?? "?"}`,
|
|
`schedulable=${summary.runtimeSchedulableCount ?? "?"}`,
|
|
`unschedulable=${summary.runtimeUnschedulableCount ?? "?"}`,
|
|
`history=${summary.historyCount ?? runs.length}`,
|
|
`window=${formatWindow(summary.historyFrom, summary.historyTo)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"CRON",
|
|
`schedule=${cronJob.schedule ?? "-"}`,
|
|
`last=${shortIso(cronJob.lastScheduleTime)}`,
|
|
`active=${cronJob.active ?? "-"}`,
|
|
`state=${metadata.namespace ?? "-"}/${metadata.stateConfigMapName ?? "-"}`,
|
|
`ledger=req:${globalLedger.requestCount ?? 0} tok:${formatNumber(globalLedger.totalTokens)} cost:$${formatCost(globalLedger.estimatedCostUsd)}`,
|
|
].join(" "));
|
|
lines.push("");
|
|
lines.push("ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "STATE", "Q", "SCH", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
|
...accounts.map((account) => [
|
|
stringValue(account.account) ?? "-",
|
|
stringValue(account.status) ?? "-",
|
|
account.quarantineActive === true ? "Y" : "-",
|
|
account.runtimeSchedulable === true ? "Y" : account.runtimeSchedulable === false ? "N" : "-",
|
|
account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-",
|
|
account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-",
|
|
account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-",
|
|
textValue(account.freezeIntervalMin),
|
|
textValue(account.successIntervalMin),
|
|
textValue(account.successMaxIntervalMin),
|
|
textValue(account.probeCount),
|
|
shortIso(account.lastEventAt ?? account.lastProbeAt),
|
|
textValue(account.lastHttp),
|
|
account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-",
|
|
shorten(stringValue(account.lastFailureKind) ?? "-", 20),
|
|
shorten(stringValue(account.lastAction) ?? "-", 16),
|
|
shortIso(account.nextProbeAfter),
|
|
textValue(account.observedLastToNextMin),
|
|
]),
|
|
]));
|
|
if (runs.length > 0 || context.full) {
|
|
lines.push("");
|
|
lines.push(`RUNS last=${Math.min(context.events, runs.length)}`);
|
|
lines.push(renderTable([
|
|
["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "GF", "GACT", "REASSERT"],
|
|
...runs.slice(-context.events).map((run) => [
|
|
shortIso(run.at),
|
|
textValue(run.selected),
|
|
textValue(run.due),
|
|
textValue(run.ok),
|
|
textValue(run.mismatch),
|
|
textValue(run.transportFailures),
|
|
textValue(run.actionsTaken),
|
|
textValue(run.gatewayFailures),
|
|
textValue(run.gatewayActions),
|
|
textValue(run.reasserts),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("LEGEND Q=quarantined SCH=Sub2API runtime schedulable T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function renderTraceReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { requestId: string; showLines: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
`SUB2API TRACE ${context.requestId} unavailable`,
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const request = isRecord(parsed.request) ? parsed.request : {};
|
|
const final = isRecord(parsed.final) ? parsed.final : {};
|
|
const window = isRecord(parsed.window) ? parsed.window : {};
|
|
const events = recordArray(parsed.events);
|
|
const failovers = recordArray(parsed.failovers);
|
|
const selectFailures = recordArray(parsed.selectFailures);
|
|
const upstreamErrors = recordArray(parsed.upstreamErrors);
|
|
const tempUnschedulable = recordArray(parsed.tempUnschedulable);
|
|
const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {};
|
|
const accountSnapshot = recordArray(parsed.accountSnapshot);
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API TRACE",
|
|
stringValue(parsed.requestId) ?? context.requestId,
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`outcome=${stringValue(summary.outcome) ?? "-"}`,
|
|
`reason=${stringValue(summary.reason) ?? "-"}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"REQUEST",
|
|
`path=${request.path ?? final.path ?? "-"}`,
|
|
`model=${request.model ?? final.model ?? "-"}`,
|
|
`stream=${textValue(request.stream)}`,
|
|
`body=${textValue(request.bodyBytes)}`,
|
|
`first=${shortIso(summary.firstAt)}`,
|
|
`last=${shortIso(summary.lastAt)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"FINAL",
|
|
`status=${textValue(final.statusCode)}`,
|
|
`account=${formatAccountRef(final)}`,
|
|
`latency_ms=${textValue(final.latencyMs)}`,
|
|
`events=${textValue(summary.eventCount)}`,
|
|
`window=${window.beforeSeconds ?? "?"}s/${window.afterSeconds ?? "?"}s`,
|
|
].join(" "));
|
|
if (failovers.length > 0) {
|
|
lines.push("");
|
|
lines.push("FAILOVER");
|
|
lines.push(renderTable([
|
|
["AT", "ACCOUNT", "UPSTREAM", "SWITCH", "MAX"],
|
|
...failovers.map((item) => [
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.upstreamStatus),
|
|
textValue(item.switchCount),
|
|
textValue(item.maxSwitches),
|
|
]),
|
|
]));
|
|
}
|
|
if (selectFailures.length > 0) {
|
|
lines.push("");
|
|
lines.push("SELECT-FAILED");
|
|
lines.push(renderTable([
|
|
["AT", "ERROR", "EXCLUDED"],
|
|
...selectFailures.map((item) => [
|
|
shortIso(item.at),
|
|
shorten(stringValue(item.error) ?? "-", 56),
|
|
textValue(item.excludedAccountCount),
|
|
]),
|
|
]));
|
|
}
|
|
if (upstreamErrors.length > 0 || tempUnschedulable.length > 0) {
|
|
lines.push("");
|
|
lines.push("ACCOUNT SIGNALS");
|
|
lines.push(renderTable([
|
|
["TYPE", "PHASE", "AT", "ACCOUNT", "STATUS", "RULE", "UNTIL", "DETAIL"],
|
|
...upstreamErrors.map((item) => [
|
|
"upstream-error",
|
|
textValue(item.phase),
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
"-",
|
|
"-",
|
|
shorten(stringValue(item.error) ?? "-", 36),
|
|
]),
|
|
...tempUnschedulable.map((item) => [
|
|
"temp-unsched",
|
|
textValue(item.phase),
|
|
shortIso(item.at),
|
|
formatAccountRef(item),
|
|
textValue(item.statusCode),
|
|
textValue(item.ruleIndex),
|
|
shortIso(item.until),
|
|
shorten(stringValue(item.reason) ?? stringValue(item.matchedKeyword) ?? "-", 36),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("WINDOW STATS");
|
|
lines.push(renderTable([
|
|
["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"],
|
|
[
|
|
textValue(windowStats.matchedLines),
|
|
textValue(windowStats.eventCount),
|
|
textValue(windowStats.finalErrorCount),
|
|
textValue(windowStats.failoverCount),
|
|
textValue(windowStats.selectFailedCount),
|
|
textValue(windowStats.tempUnschedulableCount),
|
|
textValue(windowStats.adminSchedulableCount),
|
|
],
|
|
]));
|
|
if (accountSnapshot.length > 0) {
|
|
lines.push("");
|
|
lines.push("CURRENT ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ID", "ACCOUNT", "SCHED", "STATUS", "CONC", "TEMP_UNTIL"],
|
|
...accountSnapshot.slice(0, 20).map((item) => [
|
|
textValue(item.accountId),
|
|
shorten(stringValue(item.accountName) ?? "-", 32),
|
|
textValue(item.schedulable),
|
|
textValue(item.status),
|
|
textValue(item.concurrency),
|
|
shortIso(item.tempUnschedulableUntil),
|
|
]),
|
|
]));
|
|
}
|
|
if (context.showLines || parsed.showLines === true) {
|
|
const rawLines = recordArray(parsed.rawLines);
|
|
if (rawLines.length > 0) {
|
|
lines.push("");
|
|
lines.push("RAW LINES");
|
|
for (const item of rawLines) {
|
|
lines.push(shorten(stringValue(item.line) ?? "", 1000));
|
|
}
|
|
}
|
|
}
|
|
lines.push("");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <id> --raw");
|
|
lines.push("Lines: add --show-lines for bounded matched log lines.");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function formatAccountRef(item: Record<string, unknown>): string {
|
|
const name = stringValue(item.accountName);
|
|
const id = textValue(item.accountId);
|
|
if (name !== null && id !== "-") return `${name}#${id}`;
|
|
if (name !== null) return name;
|
|
return id;
|
|
}
|
|
|
|
function renderTable(rows: string[][]): string {
|
|
if (rows.length === 0) return "";
|
|
const widths: number[] = [];
|
|
for (const row of rows) {
|
|
row.forEach((cell, index) => {
|
|
widths[index] = Math.max(widths[index] ?? 0, displayWidth(cell));
|
|
});
|
|
}
|
|
return rows.map((row) => row.map((cell, index) => padRight(cell, widths[index] ?? 0)).join(" ").trimEnd()).join("\n");
|
|
}
|
|
|
|
function padRight(value: string, width: number): string {
|
|
const pad = width - displayWidth(value);
|
|
return pad <= 0 ? value : `${value}${" ".repeat(pad)}`;
|
|
}
|
|
|
|
function displayWidth(value: string): number {
|
|
return [...value].reduce((width, char) => width + (char.charCodeAt(0) > 0x7f ? 2 : 1), 0);
|
|
}
|
|
|
|
function formatWindow(from: unknown, to: unknown): string {
|
|
const left = shortIso(from);
|
|
const right = shortIso(to);
|
|
return left === "-" && right === "-" ? "-" : `${left}..${right}`;
|
|
}
|
|
|
|
function shortIso(value: unknown): string {
|
|
const text = stringValue(value);
|
|
if (text === null) return "-";
|
|
return text.replace(/^\d{4}-/u, "").replace(/:00Z$/u, "Z").replace("T", " ");
|
|
}
|
|
|
|
function textValue(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Math.round(value * 10) / 10);
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
return String(value);
|
|
}
|
|
|
|
function shorten(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : `${value.slice(0, Math.max(0, maxChars - 1))}…`;
|
|
}
|
|
|
|
function formatNumber(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0";
|
|
if (Math.abs(num) >= 1_000_000) return `${(num / 1_000_000).toFixed(1)}M`;
|
|
if (Math.abs(num) >= 1_000) return `${(num / 1_000).toFixed(1)}K`;
|
|
return String(Math.round(num));
|
|
}
|
|
|
|
function formatCost(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0.0000";
|
|
return num.toFixed(4);
|
|
}
|
|
|
|
function codexPoolValidationSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (parsed === null) return null;
|
|
const validation = isRecord(parsed.validation) ? parsed.validation : {};
|
|
const runtimeCapabilities = isRecord(parsed.runtimeCapabilities) ? parsed.runtimeCapabilities : {};
|
|
return {
|
|
ok: parsed.ok,
|
|
degraded: parsed.degraded,
|
|
mode: parsed.mode,
|
|
namespace: parsed.namespace,
|
|
serviceDns: parsed.serviceDns,
|
|
appPod: parsed.appPod,
|
|
admin: parsed.admin,
|
|
apiKey: parsed.apiKey,
|
|
ownerBalance: parsed.ownerBalance,
|
|
ownerConcurrency: parsed.ownerConcurrency,
|
|
capacity: compactStatusBlock(parsed.capacity, ["accountName", "accountId", "expectedCapacity", "runtimeConcurrency", "priority", "status", "schedulable", "ok"]),
|
|
loadFactor: compactStatusBlock(parsed.loadFactor, ["accountName", "accountId", "expectedLoadFactor", "runtimeLoadFactor", "priority", "status", "schedulable", "ok"]),
|
|
webSocketsV2: compactStatusBlock(parsed.webSocketsV2, ["accountName", "accountId", "expectedMode", "runtimeMode", "runtimeEnabled", "status", "schedulable", "ok"]),
|
|
tempUnschedulable: compactTempUnschedulableStatus(parsed.tempUnschedulable),
|
|
manualAccounts: compactManualAccounts(parsed.manualAccounts),
|
|
sentinel: compactSentinelStatus(parsed.sentinel),
|
|
runtimeCapabilities: {
|
|
ok: runtimeCapabilities.ok,
|
|
runtimeImage: runtimeCapabilities.runtimeImage,
|
|
successBodyReclassification: compactRuntimeCapability(runtimeCapabilities.successBodyReclassification),
|
|
modelRouting400Failover: compactRuntimeCapability(runtimeCapabilities.modelRouting400Failover),
|
|
valuesPrinted: false,
|
|
},
|
|
validation: {
|
|
gatewayModels: compactGatewayModels(validation.gatewayModels),
|
|
gatewayResponses: compactGatewayResponses(validation.gatewayResponses),
|
|
gatewayResponsesRecent: compactGatewayResponsesRecent(validation.gatewayResponsesRecent),
|
|
gatewayCompactRecent: compactGatewayCompactRecent(validation.gatewayCompactRecent),
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function codexPoolSyncSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
const base = codexPoolValidationSummary(parsed);
|
|
if (parsed === null || base === null) return base;
|
|
const accounts = isRecord(parsed.accounts) ? parsed.accounts : {};
|
|
const compactAccounts = recordArray(accounts.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"action",
|
|
"status",
|
|
"runtimeSchedulable",
|
|
"priority",
|
|
"capacity",
|
|
"loadFactor",
|
|
"tempUnschedulableConfigured",
|
|
"tempUnschedulableRuleCount",
|
|
"ok",
|
|
]));
|
|
return {
|
|
...base,
|
|
pool: parsed.pool,
|
|
accounts: {
|
|
desired: accounts.desired,
|
|
created: accounts.created,
|
|
updated: accounts.updated,
|
|
pruned: accounts.pruned,
|
|
pruneMode: accounts.pruneMode,
|
|
itemCount: compactAccounts.length,
|
|
attentionItems: compactAccounts.filter((item) => item.ok === false || item.runtimeSchedulable === false),
|
|
prunedItems: accounts.prunedItems,
|
|
processControl: accounts.processControl,
|
|
valuesPrinted: false,
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw",
|
|
pruneRemoved: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --prune-removed",
|
|
},
|
|
};
|
|
}
|
|
|
|
function poolTarget(pool = readCodexPoolConfig(), target = codexPoolRuntimeTarget()): Record<string, unknown> {
|
|
return {
|
|
id: target.id,
|
|
route: target.route,
|
|
namespace: target.namespace,
|
|
service: serviceName,
|
|
serviceDns: target.serviceDns,
|
|
publicBaseUrl: target.publicBaseUrl,
|
|
consumerBaseUrl: target.publicBaseUrl === null ? null : codexConsumerBaseUrl(pool, target),
|
|
configPath: codexPoolConfigPath,
|
|
groupName: pool.groupName,
|
|
apiKeyName: pool.apiKeyName,
|
|
apiKeySecret: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`,
|
|
publicExposure: targetPublicExposureSummary(target),
|
|
sentinelImageBuild: {
|
|
source: `${sub2apiConfigPath}.targets[${target.id}].codexPool.sentinelImageBuild`,
|
|
baseImageCachePolicy: target.sentinelImageBuild.baseImageCachePolicy,
|
|
noProxy: target.sentinelImageBuild.noProxy,
|
|
},
|
|
minOwnerConcurrency: pool.minOwnerConcurrency,
|
|
minOwnerConcurrencySource: pool.minOwnerConcurrencySource,
|
|
accountCapacityTotal: desiredAccountCapacityTotal(pool),
|
|
defaultAccountPriority: pool.defaultAccountPriority,
|
|
defaultAccountCapacity: pool.defaultAccountCapacity,
|
|
defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
|
|
defaultSentinelProtect: pool.defaultSentinelProtect,
|
|
sentinel: {
|
|
monitorEnabled: pool.sentinel.monitor.enabled,
|
|
actionsEnabled: pool.sentinel.actions.enabled,
|
|
cronJobName: pool.sentinel.cronJobName,
|
|
stateConfigMapName: pool.sentinel.stateConfigMapName,
|
|
},
|
|
egressProxy: target.egressProxy === null ? null : {
|
|
enabled: target.egressProxy.enabled,
|
|
applyToSentinel: target.egressProxy.applyToSentinel,
|
|
serviceName: target.egressProxy.serviceName,
|
|
listenPort: target.egressProxy.listenPort,
|
|
httpProxy: target.egressProxy.httpProxy,
|
|
noProxy: target.egressProxy.noProxy,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProfileSecret[] {
|
|
return profiles
|
|
.filter((profile) => profile.ok && profile.apiKey !== null && profile.apiKey.length > 0)
|
|
.map((profile) => ({
|
|
accountName: profile.accountName,
|
|
profile: profile.profile,
|
|
baseUrl: profile.baseUrl,
|
|
apiKey: profile.apiKey ?? "",
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
trustUpstream: profile.trustUpstream,
|
|
sentinelProtect: profile.sentinelProtect,
|
|
}));
|
|
}
|
|
|
|
function resolvedManualAccountProtections(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record<string, unknown>[] {
|
|
return pool.manualAccounts.protected.map((account) => ({
|
|
accountName: account.accountName,
|
|
reason: account.reason,
|
|
proxyBinding: resolveManualProxyBinding(account.proxyBinding, pool, target),
|
|
groupBinding: resolveManualGroupBinding(account.groupBinding, pool),
|
|
}));
|
|
}
|
|
|
|
function resolveManualProxyBinding(binding: CodexPoolManualAccountProxyBinding | null, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record<string, unknown> | null {
|
|
if (binding === null) return null;
|
|
const source = manualBindingSource(pool, binding.source);
|
|
if (source.kind !== "proxy") throw new Error(`${codexPoolConfigPath}.manualAccounts binding source ${source.id} must have kind=proxy`);
|
|
if (binding.enabled && source.provider === "target-egress-proxy" && (target.egressProxy === null || target.egressProxy.enabled !== true)) {
|
|
throw new Error(`${codexPoolConfigPath}.manualAccounts.bindingSources.${source.id} requires ${sub2apiConfigPath}.targets[${target.id}].egressProxy.enabled=true`);
|
|
}
|
|
return {
|
|
enabled: binding.enabled,
|
|
source: source.id,
|
|
proxyName: binding.proxyName,
|
|
sourcePlan: {
|
|
...manualBindingSourcePlan(source),
|
|
targetEgressProxy: source.provider === "target-egress-proxy" && target.egressProxy !== null
|
|
? {
|
|
targetId: target.id,
|
|
namespace: target.namespace,
|
|
serviceName: target.egressProxy.serviceName,
|
|
listenPort: target.egressProxy.listenPort,
|
|
httpProxy: target.egressProxy.httpProxy,
|
|
noProxy: target.egressProxy.noProxy,
|
|
}
|
|
: null,
|
|
},
|
|
};
|
|
}
|
|
|
|
function resolveManualGroupBinding(binding: CodexPoolManualAccountGroupBinding | null, pool: CodexPoolConfig): Record<string, unknown> | null {
|
|
if (binding === null) return null;
|
|
const source = manualBindingSource(pool, binding.source);
|
|
if (source.kind !== "group") throw new Error(`${codexPoolConfigPath}.manualAccounts binding source ${source.id} must have kind=group`);
|
|
return {
|
|
enabled: binding.enabled,
|
|
source: source.id,
|
|
sourcePlan: {
|
|
...manualBindingSourcePlan(source),
|
|
poolGroupName: source.provider === "pool-group" ? pool.groupName : null,
|
|
},
|
|
};
|
|
}
|
|
|
|
function manualBindingSource(pool: CodexPoolConfig, sourceId: string): CodexPoolManualBindingSource {
|
|
const source = pool.manualAccounts.bindingSources.byId[sourceId];
|
|
if (source === undefined) throw new Error(`${codexPoolConfigPath}.manualAccounts binding source ${sourceId} is not declared`);
|
|
return source;
|
|
}
|
|
|
|
function manualBindingSourcePlan(source: CodexPoolManualBindingSource): Record<string, unknown> {
|
|
return {
|
|
id: source.id,
|
|
enabled: source.enabled,
|
|
kind: source.kind,
|
|
provider: source.provider,
|
|
description: source.description,
|
|
};
|
|
}
|
|
|
|
function targetPublicExposureSummary(target: CodexPoolRuntimeTarget): Record<string, unknown> | null {
|
|
const exposure = target.publicExposure;
|
|
if (exposure === null) return null;
|
|
return {
|
|
source: `${sub2apiConfigPath}.targets[${target.id}].publicExposure`,
|
|
enabled: exposure.enabled,
|
|
target: {
|
|
id: target.id,
|
|
route: target.route,
|
|
namespace: target.namespace,
|
|
serviceDns: target.serviceDns,
|
|
},
|
|
urls: {
|
|
publicBaseUrl: exposure.publicBaseUrl,
|
|
consumerBaseUrl: target.publicBaseUrl === null ? null : `${target.publicBaseUrl.replace(/\/+$/u, "")}/`,
|
|
},
|
|
dns: exposure.dns,
|
|
frpc: {
|
|
deploymentName: exposure.frpc.deploymentName,
|
|
secretName: exposure.frpc.secretName,
|
|
secretKey: exposure.frpc.secretKey,
|
|
image: exposure.frpc.image,
|
|
serverAddr: exposure.frpc.serverAddr,
|
|
serverPort: exposure.frpc.serverPort,
|
|
proxyName: exposure.frpc.proxyName,
|
|
remotePort: exposure.frpc.remotePort,
|
|
localIP: exposure.frpc.localIP,
|
|
localPort: exposure.frpc.localPort,
|
|
tokenSourceRef: exposure.frpc.tokenSourceRef,
|
|
tokenSourceKey: exposure.frpc.tokenSourceKey,
|
|
},
|
|
pk01: exposure.pk01,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function prepareTargetPublicExposureSecret(target: CodexPoolRuntimeTarget): ReturnType<typeof prepareFrpcSecret> {
|
|
if (target.publicExposure === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure is required`);
|
|
return prepareFrpcSecret({
|
|
secretRoot: target.secretsRoot,
|
|
exposure: target.publicExposure,
|
|
sourcePathRedactor: redactRepoPath,
|
|
parseEnvFile,
|
|
requiredEnvValue,
|
|
readTextFile: (sourcePath) => {
|
|
if (!existsSync(sourcePath)) throw new Error(`publicExposure requires ${redactRepoPath(sourcePath)} with ${target.publicExposure?.frpc.tokenSourceKey}; materialize the PK01 frps token source first`);
|
|
return readTextFile(sourcePath);
|
|
},
|
|
});
|
|
}
|
|
|
|
function secretMaterialSummary(secret: ReturnType<typeof prepareFrpcSecret>): Record<string, unknown> {
|
|
return {
|
|
sourceRef: secret.sourceRef,
|
|
sourcePath: secret.sourcePath,
|
|
secretName: secret.secretName,
|
|
secretKey: secret.secretKey,
|
|
fingerprint: secret.fingerprint,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function targetPublicExposureApplyScript(target: CodexPoolRuntimeTarget, secret: ReturnType<typeof prepareFrpcSecret>): string {
|
|
if (target.publicExposure === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure is required`);
|
|
const manifest = renderFrpcManifest({
|
|
id: target.id,
|
|
route: target.route,
|
|
namespace: target.namespace,
|
|
replicas: 1,
|
|
publicExposure: target.publicExposure,
|
|
} satisfies PublicServiceTarget);
|
|
const manifestB64 = Buffer.from(manifest, "utf8").toString("base64");
|
|
const frpcTomlB64 = Buffer.from(secret.frpcToml, "utf8").toString("base64");
|
|
return `
|
|
set -u
|
|
tmp="$(mktemp -d)"
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
manifest="$tmp/sub2api-public-exposure.yaml"
|
|
frpc_toml="$tmp/frpc.toml"
|
|
printf '%s' '${manifestB64}' | base64 -d > "$manifest"
|
|
printf '%s' '${frpcTomlB64}' | base64 -d > "$frpc_toml"
|
|
ns_out="$tmp/ns.out"
|
|
ns_err="$tmp/ns.err"
|
|
secret_out="$tmp/secret.out"
|
|
secret_err="$tmp/secret.err"
|
|
apply_out="$tmp/apply.out"
|
|
apply_err="$tmp/apply.err"
|
|
rollout_out="$tmp/rollout.out"
|
|
rollout_err="$tmp/rollout.err"
|
|
pods_json="$tmp/pods.json"
|
|
pods_err="$tmp/pods.err"
|
|
logs_out="$tmp/logs.out"
|
|
logs_err="$tmp/logs.err"
|
|
kubectl create namespace ${target.namespace} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$ns_out" 2>"$ns_err"
|
|
ns_rc=$?
|
|
secret_rc=1
|
|
apply_rc=1
|
|
rollout_rc=1
|
|
if [ "$ns_rc" -eq 0 ]; then
|
|
kubectl -n ${target.namespace} create secret generic ${secret.secretName} \\
|
|
--from-file=${secret.secretKey}="$frpc_toml" \\
|
|
--dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$secret_out" 2>"$secret_err"
|
|
secret_rc=$?
|
|
else
|
|
: >"$secret_out"
|
|
printf '%s\\n' 'skipped because namespace apply failed' >"$secret_err"
|
|
fi
|
|
if [ "$ns_rc" -eq 0 ] && [ "$secret_rc" -eq 0 ]; then
|
|
kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$apply_out" 2>"$apply_err"
|
|
apply_rc=$?
|
|
if [ "$apply_rc" -eq 0 ]; then
|
|
kubectl -n ${target.namespace} rollout status deployment/${target.publicExposure.frpc.deploymentName} --timeout=60s >"$rollout_out" 2>"$rollout_err"
|
|
rollout_rc=$?
|
|
else
|
|
printf '%s\\n' 'skipped because apply failed' >"$rollout_err"
|
|
fi
|
|
else
|
|
: >"$apply_out"
|
|
printf '%s\\n' 'skipped because namespace or secret step failed' >"$apply_err"
|
|
: >"$rollout_out"
|
|
printf '%s\\n' 'skipped because namespace or secret step failed' >"$rollout_err"
|
|
fi
|
|
kubectl -n ${target.namespace} get pods -l app.kubernetes.io/name=${target.publicExposure.frpc.deploymentName} -o json >"$pods_json" 2>"$pods_err"
|
|
pods_rc=$?
|
|
kubectl -n ${target.namespace} logs deployment/${target.publicExposure.frpc.deploymentName} --tail=80 >"$logs_out" 2>"$logs_err"
|
|
logs_rc=$?
|
|
python3 - "$tmp" "$ns_rc" "$secret_rc" "$apply_rc" "$rollout_rc" "$pods_rc" "$logs_rc" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
tmp = sys.argv[1]
|
|
ns_rc, secret_rc, apply_rc, rollout_rc, pods_rc, logs_rc = [int(value) for value in sys.argv[2:]]
|
|
|
|
def text(name, limit=4000):
|
|
path = os.path.join(tmp, name)
|
|
try:
|
|
return open(path, encoding="utf-8", errors="replace").read()[-limit:]
|
|
except FileNotFoundError:
|
|
return ""
|
|
|
|
def load_json(name):
|
|
path = os.path.join(tmp, name)
|
|
try:
|
|
return json.load(open(path, encoding="utf-8"))
|
|
except Exception:
|
|
return None
|
|
|
|
pods_obj = load_json("pods.json")
|
|
pods = []
|
|
if isinstance(pods_obj, dict):
|
|
for item in pods_obj.get("items") or []:
|
|
status = item.get("status") or {}
|
|
statuses = status.get("containerStatuses") or []
|
|
pods.append({
|
|
"name": item.get("metadata", {}).get("name"),
|
|
"phase": status.get("phase"),
|
|
"ready": all(cs.get("ready") is True for cs in statuses) if statuses else False,
|
|
"restarts": sum(int(cs.get("restartCount") or 0) for cs in statuses),
|
|
})
|
|
|
|
logs = text("logs.out", 4000)
|
|
payload = {
|
|
"ok": ns_rc == 0 and secret_rc == 0 and apply_rc == 0 and rollout_rc == 0 and pods_rc == 0 and len(pods) > 0 and all(p["ready"] for p in pods),
|
|
"target": "${target.id}",
|
|
"namespace": "${target.namespace}",
|
|
"source": "${sub2apiConfigPath}.targets[${target.id}].publicExposure",
|
|
"proxy": {
|
|
"name": "${target.publicExposure.frpc.proxyName}",
|
|
"remotePort": ${JSON.stringify(target.publicExposure.frpc.remotePort)},
|
|
"publicBaseUrl": "${target.publicExposure.publicBaseUrl}",
|
|
"localIP": "${target.publicExposure.frpc.localIP}",
|
|
"localPort": ${JSON.stringify(target.publicExposure.frpc.localPort)},
|
|
},
|
|
"resources": {
|
|
"secret": "${secret.secretName}",
|
|
"secretKey": "${secret.secretKey}",
|
|
"deployment": "${target.publicExposure.frpc.deploymentName}",
|
|
"image": "${target.publicExposure.frpc.image}",
|
|
},
|
|
"steps": {
|
|
"namespace": {"exitCode": ns_rc, "stdout": text("ns.out"), "stderr": text("ns.err")},
|
|
"secret": {"exitCode": secret_rc, "stdout": text("secret.out"), "stderr": text("secret.err"), "valuesPrinted": False},
|
|
"apply": {"exitCode": apply_rc, "stdout": text("apply.out"), "stderr": text("apply.err")},
|
|
"rollout": {"exitCode": rollout_rc, "stdout": text("rollout.out"), "stderr": text("rollout.err")},
|
|
"pods": {"exitCode": pods_rc, "items": pods, "stderr": text("pods.err")},
|
|
"logs": {
|
|
"exitCode": logs_rc,
|
|
"startProxySuccess": "start proxy success" in logs,
|
|
"stdoutTail": logs,
|
|
"stderrTail": text("logs.err"),
|
|
},
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
sys.exit(0 if payload["ok"] else 1)
|
|
PY
|
|
`;
|
|
}
|
|
|
|
async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): Promise<{ apiKey: string | null; error: string | null }> {
|
|
const result = await capture(config, target.route, ["sh"], `
|
|
set -u
|
|
kubectl -n ${target.namespace} get secret ${pool.apiKeySecretName} -o json
|
|
`);
|
|
if (result.exitCode !== 0) {
|
|
return { apiKey: null, error: `read pool API key secret failed: ${result.stderr.slice(-1000)}` };
|
|
}
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
const data = isRecord(parsed?.data) ? parsed.data : null;
|
|
const encoded = typeof data?.[pool.apiKeySecretKey] === "string" ? data[pool.apiKeySecretKey] : null;
|
|
if (encoded === null) return { apiKey: null, error: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey} missing` };
|
|
try {
|
|
const apiKey = Buffer.from(encoded, "base64").toString("utf8");
|
|
return apiKey.length > 0 ? { apiKey, error: null } : { apiKey: null, error: "decoded API key is empty" };
|
|
} catch (error) {
|
|
return { apiKey: null, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget()): Record<string, unknown> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
mkdirSync(codexDir, { recursive: true, mode: 0o700 });
|
|
const configPath = join(codexDir, "config.toml");
|
|
const authPath = join(codexDir, "auth.json");
|
|
const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`);
|
|
const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`);
|
|
if (!existsSync(configPath)) throw new Error(`${configPath} missing`);
|
|
if (!existsSync(authPath)) throw new Error(`${authPath} missing`);
|
|
const configBackupAction = copyIfMissing(configPath, backupConfigPath);
|
|
const authBackupAction = copyIfMissing(authPath, backupAuthPath);
|
|
const currentToml = readFileSync(configPath, "utf8");
|
|
const nextToml = updateCodexConfigToml(currentToml, pool, target);
|
|
writeFileSync(configPath, nextToml, { encoding: "utf8", mode: 0o600 });
|
|
chmodSync(configPath, 0o600);
|
|
writeFileSync(authPath, `${JSON.stringify({ OPENAI_API_KEY: apiKey }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
chmodSync(authPath, 0o600);
|
|
return {
|
|
ok: true,
|
|
codexDir,
|
|
config: {
|
|
path: configPath,
|
|
bytes: statSync(configPath).size,
|
|
backupPath: backupConfigPath,
|
|
backupAction: configBackupAction,
|
|
backupBytes: statSync(backupConfigPath).size,
|
|
},
|
|
auth: {
|
|
path: authPath,
|
|
bytes: statSync(authPath).size,
|
|
backupPath: backupAuthPath,
|
|
backupAction: authBackupAction,
|
|
backupBytes: statSync(backupAuthPath).size,
|
|
openaiApiKeyShape: "string",
|
|
openaiApiKeyBytes: Buffer.byteLength(apiKey, "utf8"),
|
|
openaiApiKeyFingerprint: fingerprint(apiKey),
|
|
valuesPrinted: false,
|
|
},
|
|
provider: {
|
|
name: pool.localCodex.providerName,
|
|
baseUrl: codexConsumerBaseUrl(pool, target),
|
|
wireApi: pool.localCodex.wireApi,
|
|
modelContextWindow: pool.localCodex.modelContextWindow,
|
|
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
|
|
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
|
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function copyIfMissing(source: string, target: string): "created" | "kept-existing" {
|
|
if (existsSync(target)) return "kept-existing";
|
|
copyFileSync(source, target);
|
|
chmodSync(target, 0o600);
|
|
return "created";
|
|
}
|
|
|
|
function updateCodexConfigToml(current: string, pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): string {
|
|
return renderCodexLocalConsumerToml(current, {
|
|
providerName: pool.localCodex.providerName,
|
|
baseUrl: codexConsumerBaseUrl(pool, target),
|
|
wireApi: pool.localCodex.wireApi,
|
|
modelContextWindow: pool.localCodex.modelContextWindow,
|
|
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
|
|
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
|
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
|
});
|
|
}
|
|
|
|
function codexConsumerBaseUrl(pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): string {
|
|
void pool;
|
|
const baseUrl = target.publicBaseUrl;
|
|
if (baseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure.enabled must be true before resolving the Codex consumer base URL`);
|
|
return `${baseUrl.replace(/\/+$/u, "")}/`;
|
|
}
|
|
|
|
export function renderCodexLocalConsumerToml(current: string, options: CodexLocalConsumerTomlOptions): string {
|
|
let next = upsertTopLevelTomlString(current, "model_provider", options.providerName);
|
|
next = upsertTopLevelTomlInteger(next, "model_context_window", options.modelContextWindow);
|
|
next = upsertTopLevelTomlInteger(next, "model_auto_compact_token_limit", options.modelAutoCompactTokenLimit);
|
|
next = upsertProviderSection(next, options);
|
|
next = upsertTomlSectionBoolean(next, "features", "responses_websockets_v2", options.responsesWebSocketsV2);
|
|
return next.endsWith("\n") ? next : `${next}\n`;
|
|
}
|
|
|
|
function upsertTopLevelTomlString(text: string, key: string, value: string): string {
|
|
const lines = text.split(/\r?\n/u);
|
|
const firstSection = lines.findIndex((line) => /^\s*\[/.test(line));
|
|
const end = firstSection === -1 ? lines.length : firstSection;
|
|
for (let index = 0; index < end; index += 1) {
|
|
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
|
|
lines[index] = `${key} = ${tomlString(value)}`;
|
|
return lines.join("\n");
|
|
}
|
|
}
|
|
lines.splice(0, 0, `${key} = ${tomlString(value)}`);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function upsertTopLevelTomlInteger(text: string, key: string, value: number): string {
|
|
const lines = text.split(/\r?\n/u);
|
|
const firstSection = lines.findIndex((line) => /^\s*\[/.test(line));
|
|
const end = firstSection === -1 ? lines.length : firstSection;
|
|
const assignment = `${key} = ${value}`;
|
|
for (let index = 0; index < end; index += 1) {
|
|
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
|
|
lines[index] = assignment;
|
|
return lines.join("\n");
|
|
}
|
|
}
|
|
lines.splice(end, 0, assignment);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function upsertProviderSection(text: string, options: CodexLocalConsumerTomlOptions): string {
|
|
const { providerName, baseUrl, wireApi, supportsWebSockets } = options;
|
|
const header = `[model_providers.${providerName}]`;
|
|
const lines = text.split(/\r?\n/u);
|
|
const start = lines.findIndex((line) => line.trim() === header);
|
|
const canonical = [
|
|
`name = ${tomlString(providerName)}`,
|
|
`base_url = ${tomlString(baseUrl)}`,
|
|
`wire_api = ${tomlString(wireApi)}`,
|
|
"requires_openai_auth = true",
|
|
`supports_websockets = ${tomlBoolean(supportsWebSockets)}`,
|
|
];
|
|
if (start === -1) {
|
|
const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== "";
|
|
return [...lines, ...(needsBlank ? [""] : []), header, ...canonical].join("\n");
|
|
}
|
|
let end = lines.length;
|
|
for (let index = start + 1; index < lines.length; index += 1) {
|
|
if (/^\s*\[/.test(lines[index])) {
|
|
end = index;
|
|
break;
|
|
}
|
|
}
|
|
const preserved = lines.slice(start + 1, end).filter((line) => {
|
|
return !/^\s*(name|base_url|wire_api|requires_openai_auth|supports_websockets|env_key)\s*=/u.test(line);
|
|
});
|
|
const replacement = [header, ...canonical, ...preserved.filter((line) => line.trim() !== "")];
|
|
return [...lines.slice(0, start), ...replacement, ...lines.slice(end)].join("\n");
|
|
}
|
|
|
|
function upsertTomlSectionBoolean(text: string, sectionName: string, key: string, value: boolean): string {
|
|
const header = `[${sectionName}]`;
|
|
const lines = text.split(/\r?\n/u);
|
|
const start = lines.findIndex((line) => line.trim() === header);
|
|
const assignment = `${key} = ${tomlBoolean(value)}`;
|
|
if (start === -1) {
|
|
const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== "";
|
|
return [...lines, ...(needsBlank ? [""] : []), header, assignment].join("\n");
|
|
}
|
|
let end = lines.length;
|
|
for (let index = start + 1; index < lines.length; index += 1) {
|
|
if (/^\s*\[/.test(lines[index])) {
|
|
end = index;
|
|
break;
|
|
}
|
|
}
|
|
for (let index = start + 1; index < end; index += 1) {
|
|
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
|
|
lines[index] = assignment;
|
|
return lines.join("\n");
|
|
}
|
|
}
|
|
return [...lines.slice(0, start + 1), assignment, ...lines.slice(start + 1)].join("\n");
|
|
}
|
|
|
|
async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget()): Promise<Record<string, unknown>> {
|
|
const probe = await probePublicModels(pool, "with-api-key", apiKey, target);
|
|
return {
|
|
...probe,
|
|
ok: probe.ok === true,
|
|
apiKeyFingerprint: fingerprint(apiKey),
|
|
keyPreview: apiKeyPreview(apiKey),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, target = codexPoolRuntimeTarget()): Promise<Record<string, unknown>> {
|
|
void pool;
|
|
if (target.publicBaseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure.enabled must be true before probing the public gateway`);
|
|
const baseUrl = target.publicBaseUrl;
|
|
const url = `${baseUrl.replace(/\/+$/u, "")}/v1/models`;
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), 8000);
|
|
try {
|
|
const response = await fetch(url, {
|
|
method: "GET",
|
|
headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
|
|
signal: controller.signal,
|
|
});
|
|
const body = await response.text();
|
|
let parsed: unknown = null;
|
|
try {
|
|
parsed = body.trim().length > 0 ? JSON.parse(body) : null;
|
|
} catch {
|
|
parsed = null;
|
|
}
|
|
const modelCount = isRecord(parsed) && Array.isArray(parsed.data) ? parsed.data.length : null;
|
|
return {
|
|
ok: response.ok,
|
|
mode,
|
|
method: "GET /v1/models",
|
|
baseUrl,
|
|
httpStatus: response.status,
|
|
modelCount,
|
|
bodyPreview: response.ok ? "" : body.slice(0, 500),
|
|
valuesPrinted: false,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
mode,
|
|
method: "GET /v1/models",
|
|
baseUrl,
|
|
httpStatus: 0,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
valuesPrinted: false,
|
|
};
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
function apiKeyPreview(apiKey: string): string {
|
|
if (apiKey.length <= 14) return "***";
|
|
return `${apiKey.slice(0, 10)}...${apiKey.slice(-4)}`;
|
|
}
|
|
|
|
function tomlString(value: string): string {
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function tomlBoolean(value: boolean): string {
|
|
return value ? "true" : "false";
|
|
}
|
|
|
|
function escapeRegExp(value: string): string {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
}
|
|
|
|
function normalizeBaseUrl(value: string | null): string | null {
|
|
if (value === null) return null;
|
|
const trimmed = value.trim().replace(/\/+$/u, "");
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = new URL(trimmed);
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
return parsed.toString().replace(/\/+$/u, "");
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function stringValue(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
function requiredStringConfigField(obj: Record<string, unknown>, key: string, path: string): string {
|
|
const value = stringValue(obj[key]);
|
|
const prefix = path.length > 0 ? `${path}.` : "";
|
|
if (value === null) throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
function readRequiredDescription(value: unknown, key: string, maxChars: number): string {
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a non-empty string`);
|
|
if (text.length > maxChars) throw new Error(`${codexPoolConfigPath}.${key} must be at most ${maxChars} characters`);
|
|
if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`);
|
|
return text;
|
|
}
|
|
|
|
function readRequiredEmail(value: unknown, key: string): string {
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a non-empty string`);
|
|
if (!/^[^@\s]+@[^@\s]+[.][^@\s]+$/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must be an email address`);
|
|
return text;
|
|
}
|
|
|
|
function readRequiredBaseUrl(value: unknown, key: string): string {
|
|
const baseUrl = normalizeBaseUrl(stringValue(value));
|
|
if (baseUrl === null) throw new Error(`${codexPoolConfigPath}.${key} must be a valid http(s) URL`);
|
|
return baseUrl;
|
|
}
|
|
|
|
function readRequiredPort(value: unknown, key: string): number {
|
|
const port = numberValue(value);
|
|
if (port === null || !Number.isInteger(port) || port < 1 || port > 65535) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer port`);
|
|
}
|
|
return port;
|
|
}
|
|
|
|
function readRequiredPositiveNumber(value: unknown, key: string): number {
|
|
const number = numberValue(value);
|
|
if (number === null || number <= 0) throw new Error(`${codexPoolConfigPath}.${key} must be > 0`);
|
|
return number;
|
|
}
|
|
|
|
function requiredRecordConfigField(obj: Record<string, unknown>, key: string, path: string): Record<string, unknown> {
|
|
const value = obj[key];
|
|
const prefix = path.length > 0 ? `${path}.` : "";
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be a YAML object`);
|
|
return value;
|
|
}
|
|
|
|
function integerConfigField(obj: Record<string, unknown>, key: string, path: string): number {
|
|
const value = obj[key];
|
|
const prefix = path.length > 0 ? `${path}.` : "";
|
|
if (typeof value !== "number" || !Number.isInteger(value)) throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be an integer`);
|
|
return value;
|
|
}
|
|
|
|
function integerArrayConfigField(obj: Record<string, unknown>, key: string, path: string): number[] {
|
|
const value = obj[key];
|
|
const prefix = path.length > 0 ? `${path}.` : "";
|
|
if (!Array.isArray(value) || !value.every((item) => typeof item === "number" && Number.isInteger(item))) {
|
|
throw new Error(`${codexPoolConfigPath}.${prefix}${key} must be an array of integers`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function numberValue(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function booleanValue(value: unknown): boolean | null {
|
|
if (typeof value === "boolean") return value;
|
|
if (typeof value === "string") {
|
|
if (value.trim() === "true") return true;
|
|
if (value.trim() === "false") return false;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function validateKubernetesName(value: string, key: string, strictDns: boolean): void {
|
|
const pattern = strictDns ? /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u : /^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u;
|
|
if (!pattern.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`);
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function fingerprint(value: string): string {
|
|
return shortSha256Fingerprint(value);
|
|
}
|
|
|
|
export function codexPoolSentinelProbeConfigFingerprint(input: {
|
|
accountName: string;
|
|
profile: string;
|
|
baseUrl: string;
|
|
apiKeyFingerprint: string | null;
|
|
upstreamUserAgent: string | null;
|
|
openaiResponsesWebSocketsV2Mode: string | null;
|
|
trustUpstream: boolean;
|
|
sentinelProtect: CodexSentinelProtectPolicy;
|
|
}): string {
|
|
return fingerprint(JSON.stringify({
|
|
accountName: input.accountName,
|
|
profile: input.profile,
|
|
baseUrl: normalizeBaseUrl(input.baseUrl) ?? input.baseUrl,
|
|
apiKeyFingerprint: input.apiKeyFingerprint,
|
|
upstreamUserAgent: input.upstreamUserAgent,
|
|
openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode,
|
|
trustUpstream: input.trustUpstream,
|
|
sentinelProtect: input.sentinelProtect,
|
|
}));
|
|
}
|
|
|
|
function syncScript(payload: unknown, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string {
|
|
const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
|
return remotePythonScript("sync", encoded, pool, target);
|
|
}
|
|
|
|
function validateScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string {
|
|
return remotePythonScript("validate", "", pool, target);
|
|
}
|
|
|
|
function traceScript(pool: CodexPoolConfig, options: TraceOptions, target: CodexPoolRuntimeTarget): string {
|
|
const encoded = Buffer.from(JSON.stringify({
|
|
requestId: options.requestId,
|
|
since: options.since,
|
|
tail: options.tail,
|
|
contextSeconds: options.contextSeconds,
|
|
showLines: options.showLines,
|
|
}), "utf8").toString("base64");
|
|
return remotePythonScript("trace", encoded, pool, target);
|
|
}
|
|
|
|
function sentinelProbeScript(payload: unknown, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string {
|
|
const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
|
return remotePythonScript("sentinel-probe", encoded, pool, target);
|
|
}
|
|
|
|
function sentinelReportScript(pool: CodexPoolConfig, events: number, target: CodexPoolRuntimeTarget): string {
|
|
const stateName = pool.sentinel.stateConfigMapName;
|
|
const cronJobName = pool.sentinel.cronJobName;
|
|
return `
|
|
set -eu
|
|
python3 - <<'PY'
|
|
import json
|
|
import subprocess
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
NAMESPACE = ${JSON.stringify(target.namespace)}
|
|
STATE_NAME = ${JSON.stringify(stateName)}
|
|
CRONJOB_NAME = ${JSON.stringify(cronJobName)}
|
|
EVENT_LIMIT = ${JSON.stringify(events)}
|
|
|
|
def run(cmd):
|
|
return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
def text(data, limit=2000):
|
|
if isinstance(data, bytes):
|
|
data = data.decode("utf-8", errors="replace")
|
|
return data[-limit:]
|
|
|
|
def kube_json(args):
|
|
proc = run(["kubectl", *args, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return None, text(proc.stderr)
|
|
try:
|
|
return json.loads(proc.stdout.decode("utf-8")), None
|
|
except Exception as exc:
|
|
return None, str(exc)
|
|
|
|
def parse_iso(value):
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except Exception:
|
|
return None
|
|
|
|
def minutes_between(a, b):
|
|
left = parse_iso(a)
|
|
right = parse_iso(b)
|
|
if left is None or right is None:
|
|
return None
|
|
return round((right - left).total_seconds() / 60, 1)
|
|
|
|
def day_ledgers(state):
|
|
ledger = {}
|
|
raw = state.get("ledger")
|
|
if isinstance(raw, dict):
|
|
for item in raw.values():
|
|
if not isinstance(item, dict):
|
|
continue
|
|
add_ledger(ledger, item)
|
|
return ledger
|
|
|
|
def add_ledger(target, source):
|
|
target["inputTokens"] = target.get("inputTokens", 0) + int(source.get("inputTokens") or 0)
|
|
target["outputTokens"] = target.get("outputTokens", 0) + int(source.get("outputTokens") or 0)
|
|
target["totalTokens"] = target.get("totalTokens", 0) + int(source.get("totalTokens") or 0)
|
|
target["estimatedCostUsd"] = target.get("estimatedCostUsd", 0.0) + float(source.get("estimatedCostUsd") or 0)
|
|
target["requestCount"] = target.get("requestCount", 0) + int(source.get("requestCount") or 0)
|
|
|
|
def account_ledger(account_state):
|
|
total = {}
|
|
raw = account_state.get("daily")
|
|
if isinstance(raw, dict):
|
|
for item in raw.values():
|
|
if isinstance(item, dict):
|
|
add_ledger(total, item)
|
|
return total
|
|
|
|
def action_type(probe):
|
|
action = probe.get("action") if isinstance(probe, dict) else None
|
|
if isinstance(action, dict):
|
|
value = action.get("type")
|
|
if value:
|
|
return value
|
|
return "taken" if action.get("taken") is True else ""
|
|
return ""
|
|
|
|
def error_code(probe):
|
|
details = probe.get("errorDetails") if isinstance(probe, dict) else None
|
|
if not isinstance(details, dict):
|
|
return ""
|
|
openai_error = details.get("openaiError")
|
|
if isinstance(openai_error, dict):
|
|
return openai_error.get("code") or openai_error.get("type") or ""
|
|
return details.get("kind") or ""
|
|
|
|
def sentinel_protect_summary(account_state, probe):
|
|
probe_protect = probe.get("sentinelProtect") if isinstance(probe.get("sentinelProtect"), dict) else {}
|
|
config_protect = account_state.get("sentinelProtectConfig") if isinstance(account_state.get("sentinelProtectConfig"), dict) else {}
|
|
source = probe_protect if probe_protect else config_protect
|
|
if not isinstance(source, dict) or source.get("enabled") is not True:
|
|
return {
|
|
"enabled": False,
|
|
"threshold": None,
|
|
"decision": None,
|
|
"failureCount": None,
|
|
}
|
|
return {
|
|
"enabled": True,
|
|
"threshold": source.get("threshold") or source.get("consecutiveFailures"),
|
|
"decision": probe_protect.get("decision"),
|
|
"failureCount": probe_protect.get("failureCount"),
|
|
}
|
|
|
|
def report():
|
|
cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME])
|
|
state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME])
|
|
state = {}
|
|
parse_error = None
|
|
if isinstance(state_cm, dict):
|
|
raw_state = (state_cm.get("data") or {}).get("state.json")
|
|
if isinstance(raw_state, str) and raw_state.strip():
|
|
try:
|
|
state = json.loads(raw_state)
|
|
except Exception as exc:
|
|
parse_error = str(exc)
|
|
accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
history = state.get("history") if isinstance(state.get("history"), list) else []
|
|
account_rows = []
|
|
for name, account_state in sorted(accounts.items()):
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
probe = account_state.get("lastProbe") if isinstance(account_state.get("lastProbe"), dict) else {}
|
|
quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else {}
|
|
gateway_failure = account_state.get("lastGatewayFailure") if isinstance(account_state.get("lastGatewayFailure"), dict) else {}
|
|
last_event_at = account_state.get("lastGatewayFailureAt") or account_state.get("lastProbeAt")
|
|
last_failure_kind = quarantine.get("failureKind") if quarantine.get("active") is True and quarantine.get("failureKind") else probe.get("failureKind")
|
|
last_action = action_type(probe)
|
|
if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")):
|
|
last_action = action_type(gateway_failure)
|
|
ledger = account_ledger(account_state)
|
|
protect = sentinel_protect_summary(account_state, probe)
|
|
account_rows.append({
|
|
"account": name,
|
|
"status": account_state.get("lastStatus"),
|
|
"quarantineActive": quarantine.get("active") is True,
|
|
"quarantineApplied": quarantine.get("applied") if isinstance(quarantine, dict) else None,
|
|
"freezeIntervalMin": quarantine.get("intervalMinutes") if isinstance(quarantine, dict) else None,
|
|
"freezeUntil": quarantine.get("until") if isinstance(quarantine, dict) else None,
|
|
"runtimeAccountId": account_state.get("runtimeAccountId"),
|
|
"runtimeStatus": account_state.get("runtimeStatus"),
|
|
"runtimeSchedulable": account_state.get("runtimeSchedulable"),
|
|
"runtimeMissing": account_state.get("runtimeMissing"),
|
|
"runtimeTempUnschedulableSet": account_state.get("runtimeTempUnschedulableSet"),
|
|
"runtimeTempUnschedulableUntil": account_state.get("runtimeTempUnschedulableUntil"),
|
|
"runtimeSyncedAt": account_state.get("runtimeSyncedAt"),
|
|
"runtimeSyncError": account_state.get("runtimeSyncError"),
|
|
"trustUpstream": account_state.get("trustUpstream") if account_state.get("trustUpstream") is not None else probe.get("trustUpstream"),
|
|
"successStreak": account_state.get("successStreak") or 0,
|
|
"successIntervalMin": account_state.get("successIntervalMinutes") or 0,
|
|
"successMaxIntervalMin": account_state.get("successMaxIntervalMinutes") or probe.get("successMaxIntervalMinutes"),
|
|
"probeCount": ledger.get("requestCount", 0),
|
|
"inputTokens": ledger.get("inputTokens", 0),
|
|
"outputTokens": ledger.get("outputTokens", 0),
|
|
"totalTokens": ledger.get("totalTokens", 0),
|
|
"estimatedCostUsd": round(float(ledger.get("estimatedCostUsd", 0)), 6),
|
|
"lastProbeAt": account_state.get("lastProbeAt"),
|
|
"lastEventAt": last_event_at,
|
|
"lastGatewayFailureAt": account_state.get("lastGatewayFailureAt"),
|
|
"lastPurpose": probe.get("purpose"),
|
|
"lastHttp": probe.get("httpStatus"),
|
|
"lastMarker": probe.get("markerMatched"),
|
|
"sentinelProtectEnabled": protect.get("enabled"),
|
|
"sentinelProtectDecision": protect.get("decision"),
|
|
"sentinelProtectFailureCount": protect.get("failureCount"),
|
|
"sentinelProtectThreshold": protect.get("threshold"),
|
|
"lastFailureKind": last_failure_kind,
|
|
"lastErrorCode": error_code(probe),
|
|
"lastAction": last_action,
|
|
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
|
"observedLastToNextMin": minutes_between(last_event_at, account_state.get("nextProbeAfter")),
|
|
"requestShape": probe.get("requestShape"),
|
|
})
|
|
run_rows = []
|
|
for item in history:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
selection = item.get("selection") if isinstance(item.get("selection"), dict) else {}
|
|
gateway = item.get("gatewayFailureMonitor") if isinstance(item.get("gatewayFailureMonitor"), dict) else {}
|
|
run_rows.append({
|
|
"at": item.get("at"),
|
|
"selected": item.get("selected"),
|
|
"due": selection.get("due"),
|
|
"ok": item.get("okCount"),
|
|
"mismatch": item.get("mismatchCount") if item.get("mismatchCount") is not None else item.get("markerMismatchCount"),
|
|
"transportFailures": item.get("transportFailureCount"),
|
|
"actionsTaken": item.get("actionsTaken"),
|
|
"gatewayFailures": gateway.get("newFailures"),
|
|
"gatewayActions": gateway.get("actionsTaken"),
|
|
"reasserts": len(item.get("reconcile") or []),
|
|
})
|
|
quarantined = [item for item in account_rows if item.get("quarantineActive") is True]
|
|
runtime_schedulable = [item for item in account_rows if item.get("runtimeSchedulable") is True]
|
|
runtime_unschedulable = [item for item in account_rows if item.get("runtimeSchedulable") is False]
|
|
cron_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
|
|
cron_status = cronjob.get("status") if isinstance(cronjob, dict) else {}
|
|
global_ledger = day_ledgers(state)
|
|
result = {
|
|
"ok": state_error is None and parse_error is None,
|
|
"metadata": {
|
|
"namespace": NAMESPACE,
|
|
"stateConfigMapName": STATE_NAME,
|
|
"cronJobName": CRONJOB_NAME,
|
|
"generatedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"valuesPrinted": False,
|
|
},
|
|
"cronJob": {
|
|
"exists": isinstance(cronjob, dict),
|
|
"schedule": cron_spec.get("schedule") if isinstance(cron_spec, dict) else None,
|
|
"suspend": cron_spec.get("suspend") if isinstance(cron_spec, dict) else None,
|
|
"lastScheduleTime": cron_status.get("lastScheduleTime") if isinstance(cron_status, dict) else None,
|
|
"active": len(cron_status.get("active") or []) if isinstance(cron_status, dict) else None,
|
|
"error": cron_error,
|
|
},
|
|
"summary": {
|
|
"accountCount": len(account_rows),
|
|
"quarantinedCount": len(quarantined),
|
|
"runtimeSchedulableCount": len(runtime_schedulable),
|
|
"runtimeUnschedulableCount": len(runtime_unschedulable),
|
|
"historyCount": len(history),
|
|
"historyFrom": run_rows[0].get("at") if run_rows else None,
|
|
"historyTo": run_rows[-1].get("at") if run_rows else None,
|
|
"lastRun": state.get("lastRun"),
|
|
"runtimeSchedulable": state.get("runtimeSchedulable"),
|
|
},
|
|
"globalLedger": global_ledger,
|
|
"accounts": account_rows,
|
|
"runs": run_rows[-EVENT_LIMIT:],
|
|
"errors": {
|
|
"state": state_error,
|
|
"parse": parse_error,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
return result
|
|
|
|
payload = report()
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
raise SystemExit(0 if payload.get("ok") else 1)
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function cleanupProbesScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string {
|
|
return remotePythonScript("cleanup-probes", "", pool, target);
|
|
}
|
|
|
|
function sentinelImageStatusScript(pool: CodexPoolConfig, targetRuntime: CodexPoolRuntimeTarget): string {
|
|
const target = codexPoolSentinelRuntimeImage(pool.sentinel);
|
|
return remoteSentinelImageScript("status", target, pool.sentinel, null, targetRuntime);
|
|
}
|
|
|
|
function sentinelImageBuildScript(pool: CodexPoolConfig, targetRuntime: CodexPoolRuntimeTarget): string {
|
|
const target = codexPoolSentinelRuntimeImage(pool.sentinel);
|
|
const dockerfile = readFileSync(sentinelImageDockerfilePath, "utf8");
|
|
return remoteSentinelImageScript("build", target, pool.sentinel, dockerfile, targetRuntime);
|
|
}
|
|
|
|
function remoteSentinelImageScript(mode: "status" | "build", target: ReturnType<typeof codexPoolSentinelRuntimeImage>, sentinel: CodexPoolSentinelConfig, dockerfile: string | null, targetRuntime: CodexPoolRuntimeTarget): string {
|
|
const dockerfileB64 = dockerfile === null ? "" : Buffer.from(dockerfile, "utf8").toString("base64");
|
|
return `
|
|
set -eu
|
|
mode=${shQuote(mode)}
|
|
image=${shQuote(target.runtimeImage)}
|
|
repo=${shQuote("platform-infra/sub2api-account-sentinel")}
|
|
tag=${shQuote(target.tag)}
|
|
base_image=${shQuote(target.baseImage)}
|
|
openai_version=${shQuote(sentinel.sdk.openaiPythonVersion)}
|
|
base_image_cache_policy=${shQuote(targetRuntime.sentinelImageBuild.baseImageCachePolicy)}
|
|
work=/tmp/unidesk-sub2api-sentinel-image
|
|
mkdir -p "$work"
|
|
dockerfile_path="$work/sentinel.Dockerfile"
|
|
registry_has_tag=false
|
|
if curl -fsS --max-time 10 "http://127.0.0.1:5000/v2/$repo/tags/list" 2>/dev/null | grep -F '"'"$tag"'"' >/dev/null 2>&1; then
|
|
registry_has_tag=true
|
|
fi
|
|
local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)"
|
|
if [ "$mode" = "status" ]; then
|
|
if [ -n "$local_id" ]; then
|
|
python_version="$(docker run --rm "$image" python3 --version 2>/dev/null || true)"
|
|
openai_runtime_version="$(docker run --rm "$image" python3 -c 'import importlib.metadata; print(importlib.metadata.version("openai"))' 2>/dev/null || true)"
|
|
else
|
|
python_version=
|
|
openai_runtime_version=
|
|
fi
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({
|
|
"ok": bool("${"${registry_has_tag}"}" == "true" or "${"${local_id}"}"),
|
|
"mode": "status",
|
|
"image": "${target.runtimeImage}",
|
|
"baseImage": "${target.baseImage}",
|
|
"tag": "${target.tag}",
|
|
"localExists": bool("${"${local_id}"}"),
|
|
"localImageId": "${"${local_id}"}" or None,
|
|
"registryHasTag": "${"${registry_has_tag}"}" == "true",
|
|
"pythonVersion": "${"${python_version}"}" or None,
|
|
"openaiPythonVersion": "${"${openai_runtime_version}"}" or None,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
exit 0
|
|
fi
|
|
if [ "$registry_has_tag" = "true" ]; then
|
|
if [ -z "$local_id" ]; then
|
|
docker pull "$image" >/dev/null 2>&1 || true
|
|
local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)"
|
|
fi
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({
|
|
"ok": True,
|
|
"mode": "reused-existing",
|
|
"image": "${target.runtimeImage}",
|
|
"baseImage": "${target.baseImage}",
|
|
"tag": "${target.tag}",
|
|
"registryHasTag": True,
|
|
"localImageId": "${"${local_id}"}" or None,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
exit 0
|
|
fi
|
|
base64 -d > "$dockerfile_path" <<'UNIDESK_SENTINEL_DOCKERFILE_B64'
|
|
${dockerfileB64}
|
|
UNIDESK_SENTINEL_DOCKERFILE_B64
|
|
export NO_PROXY=${shQuote(targetRuntime.sentinelImageBuild.noProxy)}
|
|
export no_proxy=$NO_PROXY
|
|
set -- --pull
|
|
base_image_source="registry"
|
|
if [ "$base_image_cache_policy" = "local-if-present" ] && docker image inspect "$base_image" >/dev/null 2>&1; then
|
|
set --
|
|
base_image_source="local-cache"
|
|
fi
|
|
docker build "$@" \\
|
|
--build-arg BASE_IMAGE="$base_image" \\
|
|
--build-arg OPENAI_PYTHON_VERSION="$openai_version" \\
|
|
--build-arg HTTP_PROXY= --build-arg HTTPS_PROXY= --build-arg http_proxy= --build-arg https_proxy= \\
|
|
--build-arg NO_PROXY --build-arg no_proxy \\
|
|
-f "$dockerfile_path" \\
|
|
-t "$image" \\
|
|
"$work"
|
|
docker run --rm "$image" python3 -c 'import importlib.metadata, sys; expected=sys.argv[1]; actual=importlib.metadata.version("openai"); assert actual == expected, (actual, expected); print("openai", actual)' "$openai_version"
|
|
docker push "$image"
|
|
digest="$(docker image inspect "$image" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)"
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({
|
|
"ok": True,
|
|
"mode": "built-and-published",
|
|
"image": "${target.runtimeImage}",
|
|
"baseImage": "${target.baseImage}",
|
|
"tag": "${target.tag}",
|
|
"baseImageSource": "${"${base_image_source}"}",
|
|
"digest": "${"${digest}"}" or None,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function desiredAccountNames(pool: CodexPoolConfig): string[] {
|
|
return Object.keys(desiredAccountCapacityMap(pool)).sort();
|
|
}
|
|
|
|
function desiredAccountCapacityTotal(pool: CodexPoolConfig): number {
|
|
return Object.values(desiredAccountCapacityMap(pool)).reduce((total, value) => total + value, 0);
|
|
}
|
|
|
|
function desiredAccountCapacityMap(pool: CodexPoolConfig): Record<string, number> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const capacities: Record<string, number> = {};
|
|
for (const entry of pool.profiles) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
capacities[accountName] = entry.capacity ?? pool.defaultAccountCapacity;
|
|
}
|
|
return capacities;
|
|
}
|
|
|
|
function desiredAccountLoadFactorMap(pool: CodexPoolConfig): Record<string, number> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const loadFactors: Record<string, number> = {};
|
|
for (const entry of pool.profiles) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
loadFactors[accountName] = entry.loadFactor ?? pool.defaultAccountLoadFactor;
|
|
}
|
|
return loadFactors;
|
|
}
|
|
|
|
function desiredAccountWebSocketsV2ModeMap(pool: CodexPoolConfig): Record<string, OpenAIResponsesWebSocketsV2Mode | null> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const modes: Record<string, OpenAIResponsesWebSocketsV2Mode | null> = {};
|
|
for (const entry of pool.profiles) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
modes[accountName] = entry.openaiResponsesWebSocketsV2Mode;
|
|
}
|
|
return modes;
|
|
}
|
|
|
|
function desiredAccountTempUnschedulableMap(pool: CodexPoolConfig): Record<string, unknown> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const policies: Record<string, unknown> = {};
|
|
for (const entry of pool.profiles) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
policies[accountName] = renderSub2ApiTempUnschedulableCredentials(entry.tempUnschedulable);
|
|
}
|
|
return policies;
|
|
}
|
|
|
|
function remotePythonScript(mode: "sync" | "validate" | "trace" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string {
|
|
return `
|
|
set -u
|
|
python3 - <<'PY'
|
|
import base64
|
|
import json
|
|
import re
|
|
import secrets
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone, timedelta
|
|
from urllib.parse import quote
|
|
|
|
TARGET_ID = ${JSON.stringify(target.id)}
|
|
NAMESPACE = ${JSON.stringify(target.namespace)}
|
|
SERVICE_NAME = ${JSON.stringify(target.serviceName)}
|
|
SERVICE_DNS = ${JSON.stringify(target.serviceDns)}
|
|
FIELD_MANAGER = "${fieldManager}"
|
|
APP_SECRET_NAME = ${JSON.stringify(target.appSecretName)}
|
|
POOL_GROUP_NAME = "${pool.groupName}"
|
|
POOL_GROUP_DESCRIPTION = ${JSON.stringify(pool.groupDescription)}
|
|
POOL_API_KEY_NAME = "${pool.apiKeyName}"
|
|
POOL_API_KEY_SECRET_NAME = "${pool.apiKeySecretName}"
|
|
POOL_API_KEY_SECRET_KEY = "${pool.apiKeySecretKey}"
|
|
POOL_ADMIN_EMAIL_DEFAULT = ${JSON.stringify(pool.adminEmailDefault)}
|
|
MIN_OWNER_BALANCE_USD = ${JSON.stringify(pool.minOwnerBalanceUsd)}
|
|
MIN_OWNER_CONCURRENCY = ${JSON.stringify(pool.minOwnerConcurrency)}
|
|
MIN_OWNER_CONCURRENCY_SOURCE = ${JSON.stringify(pool.minOwnerConcurrencySource)}
|
|
POOL_DEFAULT_ACCOUNT_PRIORITY = ${JSON.stringify(pool.defaultAccountPriority)}
|
|
POOL_DEFAULT_ACCOUNT_CAPACITY = ${JSON.stringify(pool.defaultAccountCapacity)}
|
|
POOL_DEFAULT_ACCOUNT_LOAD_FACTOR = ${JSON.stringify(pool.defaultAccountLoadFactor)}
|
|
RESPONSES_SMOKE_MODEL = ${JSON.stringify(pool.localCodex.responsesSmokeModel)}
|
|
EXPECTED_ACCOUNT_CAPACITIES = ${JSON.stringify(desiredAccountCapacityMap(pool))}
|
|
EXPECTED_ACCOUNT_LOAD_FACTORS = ${JSON.stringify(desiredAccountLoadFactorMap(pool))}
|
|
EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))})
|
|
EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))})
|
|
MANUAL_ACCOUNT_PROTECTIONS = json.loads(${JSON.stringify(JSON.stringify(resolvedManualAccountProtections(pool, target)))})
|
|
SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))})
|
|
TARGET_EGRESS_PROXY = json.loads(${JSON.stringify(JSON.stringify(target.egressProxy))})
|
|
MODE = "${mode}"
|
|
PAYLOAD_B64 = "${encodedPayload}"
|
|
|
|
def run(cmd, input_bytes=None):
|
|
return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
def text(data, limit=4000):
|
|
if isinstance(data, bytes):
|
|
data = data.decode("utf-8", errors="replace")
|
|
return data[-limit:]
|
|
|
|
def kubectl(args, input_obj=None):
|
|
if isinstance(input_obj, str):
|
|
input_bytes = input_obj.encode("utf-8")
|
|
else:
|
|
input_bytes = input_obj
|
|
return run(["kubectl", *args], input_bytes)
|
|
|
|
def require_kubectl(args, input_obj=None, label="kubectl"):
|
|
proc = kubectl(args, input_obj)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"{label} failed: {text(proc.stderr, 1000)}")
|
|
return proc.stdout
|
|
|
|
def kube_json(args, label):
|
|
raw = require_kubectl([*args, "-o", "json"], label=label)
|
|
return json.loads(raw.decode("utf-8"))
|
|
|
|
def decode_secret_value(name, key):
|
|
data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {}
|
|
if key not in data:
|
|
return None
|
|
return base64.b64decode(data[key]).decode("utf-8")
|
|
|
|
def get_config_value(name, key):
|
|
data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {}
|
|
value = data.get(key)
|
|
return value if isinstance(value, str) and value else None
|
|
|
|
def select_app_pod():
|
|
pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or []
|
|
for pod in pods:
|
|
status = pod.get("status") or {}
|
|
if status.get("phase") != "Running":
|
|
continue
|
|
statuses = status.get("containerStatuses") or []
|
|
if statuses and all(item.get("ready") is True for item in statuses):
|
|
return pod["metadata"]["name"]
|
|
if pods:
|
|
return pods[0]["metadata"]["name"]
|
|
raise RuntimeError("sub2api app pod not found")
|
|
|
|
APP_POD = select_app_pod()
|
|
|
|
def parse_curl_output(proc):
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
marker = "\\n__HTTP_CODE__:"
|
|
pos = stdout.rfind(marker)
|
|
if pos < 0:
|
|
return {
|
|
"ok": False,
|
|
"httpStatus": 0,
|
|
"json": None,
|
|
"body": stdout,
|
|
"stderr": text(proc.stderr, 1000),
|
|
"transportExitCode": proc.returncode,
|
|
}
|
|
body = stdout[:pos]
|
|
status_text = stdout[pos + len(marker):].strip()
|
|
try:
|
|
http_status = int(status_text[-3:])
|
|
except ValueError:
|
|
http_status = 0
|
|
try:
|
|
parsed = json.loads(body) if body.strip() else None
|
|
except json.JSONDecodeError:
|
|
parsed = None
|
|
return {
|
|
"ok": proc.returncode == 0 and 200 <= http_status < 300,
|
|
"httpStatus": http_status,
|
|
"json": parsed,
|
|
"body": body,
|
|
"stderr": text(proc.stderr, 1000),
|
|
"transportExitCode": proc.returncode,
|
|
}
|
|
|
|
def utc_iso(offset_seconds=0):
|
|
return (datetime.now(timezone.utc) + timedelta(seconds=offset_seconds)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
def normalize_runtime_base_url(value):
|
|
if not isinstance(value, str):
|
|
return None
|
|
value = value.strip().rstrip("/")
|
|
return value or None
|
|
|
|
def empty_to_none(value):
|
|
return value if isinstance(value, str) and value else None
|
|
|
|
def sentinel_quality_gate_enabled():
|
|
return (SENTINEL_CONFIG.get("monitor") or {}).get("enabled") is True and (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is True
|
|
|
|
def account_notes_fingerprint(account):
|
|
notes = account.get("notes") if isinstance(account, dict) else None
|
|
if not isinstance(notes, str):
|
|
return None
|
|
match = re.search(r"fingerprint=([A-Za-z0-9_-]+)", notes)
|
|
return match.group(1) if match else None
|
|
|
|
def runtime_account_credentials(account):
|
|
credentials = account.get("credentials") if isinstance(account, dict) and isinstance(account.get("credentials"), dict) else {}
|
|
return credentials
|
|
|
|
def runtime_account_extra(account):
|
|
extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {}
|
|
return extra
|
|
|
|
def sentinel_probe_change_reasons(current, profile):
|
|
if not isinstance(current, dict) or current.get("id") is None:
|
|
return ["created"]
|
|
credentials = runtime_account_credentials(current)
|
|
extra = runtime_account_extra(current)
|
|
expected_base_url = normalize_runtime_base_url(profile.get("baseUrl"))
|
|
runtime_base_url = normalize_runtime_base_url(credentials.get("base_url"))
|
|
expected_user_agent = empty_to_none(profile.get("upstreamUserAgent"))
|
|
runtime_user_agent = empty_to_none(credentials.get("user_agent"))
|
|
expected_ws_mode = empty_to_none(profile.get("openaiResponsesWebSocketsV2Mode"))
|
|
runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode"))
|
|
expected_trust_upstream = profile.get("trustUpstream") is True
|
|
runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True
|
|
expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
|
runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False}
|
|
reasons = []
|
|
if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"):
|
|
reasons.append("profile")
|
|
if runtime_base_url != expected_base_url:
|
|
reasons.append("base-url")
|
|
if account_notes_fingerprint(current) != profile.get("apiKeyFingerprint"):
|
|
reasons.append("api-key-fingerprint")
|
|
if runtime_user_agent != expected_user_agent:
|
|
reasons.append("upstream-user-agent")
|
|
if runtime_ws_mode != expected_ws_mode:
|
|
reasons.append("responses-websockets-v2-mode")
|
|
if runtime_trust_upstream != expected_trust_upstream:
|
|
reasons.append("trust-upstream")
|
|
if runtime_protect != expected_protect:
|
|
reasons.append("sentinel-protect")
|
|
return reasons
|
|
|
|
def curl_api(method, path, bearer=None, payload=None):
|
|
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
script = r'''
|
|
set -eu
|
|
method="$1"
|
|
url="$2"
|
|
token="\${3:-}"
|
|
tmp="$(mktemp)"
|
|
trap 'rm -f "$tmp"' EXIT
|
|
cat > "$tmp"
|
|
if [ -n "$token" ]; then
|
|
if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url"
|
|
else
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url"
|
|
fi
|
|
else
|
|
if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"
|
|
else
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
|
fi
|
|
fi
|
|
'''
|
|
proc = run([
|
|
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
|
"--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or "",
|
|
], body)
|
|
return parse_curl_output(proc)
|
|
|
|
def envelope_data(parsed):
|
|
if isinstance(parsed, dict) and "data" in parsed:
|
|
return parsed.get("data")
|
|
return parsed
|
|
|
|
def ensure_success(resp, label):
|
|
parsed = resp.get("json")
|
|
code = parsed.get("code") if isinstance(parsed, dict) else None
|
|
if not resp.get("ok") or (code is not None and code != 0):
|
|
message = parsed.get("message") if isinstance(parsed, dict) else text(resp.get("body", ""), 500)
|
|
raise RuntimeError(f"{label} failed: http={resp.get('httpStatus')} message={message}")
|
|
return envelope_data(parsed)
|
|
|
|
def ensure_admin_compliance(token):
|
|
status_resp = curl_api("GET", "/api/v1/admin/compliance", bearer=token)
|
|
if status_resp.get("httpStatus") == 404:
|
|
return {
|
|
"ok": True,
|
|
"action": "not-supported-by-runtime",
|
|
"valuesPrinted": False,
|
|
}
|
|
status = ensure_success(status_resp, "get admin compliance")
|
|
if not isinstance(status, dict):
|
|
return {
|
|
"ok": True,
|
|
"action": "status-unstructured",
|
|
"valuesPrinted": False,
|
|
}
|
|
version = status.get("version")
|
|
required = status.get("required") is True
|
|
if not required:
|
|
return {
|
|
"ok": True,
|
|
"action": "already-acknowledged",
|
|
"version": version,
|
|
"requiredBefore": False,
|
|
"requiredAfter": False,
|
|
"phrasePrinted": False,
|
|
"valuesPrinted": False,
|
|
}
|
|
phrase = status.get("ack_phrase_zh") if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else status.get("ack_phrase_en")
|
|
language = "zh" if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else "en"
|
|
if not isinstance(phrase, str) or not phrase:
|
|
raise RuntimeError("admin compliance acknowledgement phrase missing")
|
|
accepted = ensure_success(
|
|
curl_api("POST", "/api/v1/admin/compliance/accept", bearer=token, payload={"phrase": phrase, "language": language}),
|
|
"accept admin compliance",
|
|
)
|
|
required_after = accepted.get("required") if isinstance(accepted, dict) else None
|
|
return {
|
|
"ok": required_after is False,
|
|
"action": "accepted",
|
|
"version": version,
|
|
"requiredBefore": True,
|
|
"requiredAfter": required_after,
|
|
"phrasePrinted": False,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def extract_items(data):
|
|
if isinstance(data, list):
|
|
return data
|
|
if isinstance(data, dict):
|
|
if isinstance(data.get("items"), list):
|
|
return data["items"]
|
|
for key in ("groups", "accounts", "api_keys", "keys"):
|
|
if isinstance(data.get(key), list):
|
|
return data[key]
|
|
return []
|
|
|
|
def find_access_token(data):
|
|
if isinstance(data, dict):
|
|
for key in ("access_token", "token"):
|
|
if isinstance(data.get(key), str) and data[key]:
|
|
return data[key]
|
|
for value in data.values():
|
|
token = find_access_token(value)
|
|
if token:
|
|
return token
|
|
return None
|
|
|
|
def login():
|
|
admin_email = get_config_value("sub2api-config", "ADMIN_EMAIL") or POOL_ADMIN_EMAIL_DEFAULT
|
|
admin_password = decode_secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD")
|
|
if not admin_password:
|
|
raise RuntimeError("ADMIN_PASSWORD missing from sub2api-secrets")
|
|
data = ensure_success(curl_api("POST", "/api/v1/auth/login", payload={"email": admin_email, "password": admin_password}), "admin login")
|
|
token = find_access_token(data)
|
|
if not token:
|
|
raise RuntimeError("admin login response did not contain access_token")
|
|
compliance = ensure_admin_compliance(token)
|
|
if compliance.get("ok") is not True:
|
|
raise RuntimeError("admin compliance acknowledgement failed")
|
|
return admin_email, token, compliance
|
|
|
|
def group_payload():
|
|
return {
|
|
"name": POOL_GROUP_NAME,
|
|
"description": POOL_GROUP_DESCRIPTION,
|
|
"platform": "openai",
|
|
"rate_multiplier": 1,
|
|
"is_exclusive": False,
|
|
"subscription_type": "standard",
|
|
"allow_messages_dispatch": True,
|
|
"require_oauth_only": False,
|
|
"require_privacy_set": False,
|
|
"rpm_limit": 0,
|
|
}
|
|
|
|
def list_groups(token):
|
|
data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")
|
|
return extract_items(data)
|
|
|
|
def ensure_group(token):
|
|
existing = next((item for item in list_groups(token) if item.get("name") == POOL_GROUP_NAME), None)
|
|
payload = group_payload()
|
|
if existing is None:
|
|
created = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=payload), "create group")
|
|
return created, "created"
|
|
group_id = existing.get("id")
|
|
if group_id is None:
|
|
raise RuntimeError("existing group has no id")
|
|
payload["status"] = "active"
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/groups/{group_id}", bearer=token, payload=payload), "update group")
|
|
return updated if isinstance(updated, dict) else existing, "updated"
|
|
|
|
def list_accounts_for_group(token, group_id):
|
|
path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500&platform=openai"
|
|
data = ensure_success(curl_api("GET", path, bearer=token), f"list accounts for group {group_id}")
|
|
return extract_items(data)
|
|
|
|
def account_group_ids(token, account):
|
|
if not isinstance(account, dict) or account.get("id") is None:
|
|
return []
|
|
account_id = account.get("id")
|
|
account_name = account.get("name")
|
|
ids = []
|
|
for group in list_groups(token):
|
|
group_id = group.get("id") if isinstance(group, dict) else None
|
|
if group_id is None:
|
|
continue
|
|
members = list_accounts_for_group(token, group_id)
|
|
if any(item.get("id") == account_id or item.get("name") == account_name for item in members if isinstance(item, dict)):
|
|
ids.append(group_id)
|
|
return sorted(set(ids))
|
|
|
|
def list_accounts(token):
|
|
path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-codex-")
|
|
data = ensure_success(curl_api("GET", path, bearer=token), "list accounts")
|
|
return extract_items(data)
|
|
|
|
def find_account_by_name(token, name):
|
|
path = "/api/v1/admin/accounts?page=1&page_size=50&platform=openai&search=" + quote(str(name))
|
|
data = ensure_success(curl_api("GET", path, bearer=token), "find account " + str(name))
|
|
for item in extract_items(data):
|
|
if isinstance(item, dict) and item.get("name") == name:
|
|
return item
|
|
return None
|
|
|
|
def list_proxy_candidates(token, search=""):
|
|
path = "/api/v1/admin/proxies?page=1&page_size=200"
|
|
if search:
|
|
path += "&search=" + quote(str(search))
|
|
data = ensure_success(curl_api("GET", path, bearer=token), "list proxies")
|
|
return extract_items(data)
|
|
|
|
def find_proxy_by_name(token, name):
|
|
for item in list_proxy_candidates(token, name):
|
|
if isinstance(item, dict) and item.get("name") == name:
|
|
return item
|
|
return None
|
|
|
|
def manual_binding_plan(binding, expected_kind):
|
|
if not isinstance(binding, dict):
|
|
return None
|
|
plan = binding.get("sourcePlan")
|
|
if not isinstance(plan, dict):
|
|
raise RuntimeError("manual account binding is missing resolved sourcePlan")
|
|
if plan.get("enabled") is not True:
|
|
raise RuntimeError(f"manual account binding source {plan.get('id')} is disabled")
|
|
if plan.get("kind") != expected_kind:
|
|
raise RuntimeError(f"manual account binding source {plan.get('id')} kind must be {expected_kind}")
|
|
return plan
|
|
|
|
def desired_manual_proxy_payload(protection):
|
|
binding = protection.get("proxyBinding") if isinstance(protection, dict) else None
|
|
if not isinstance(binding, dict) or binding.get("enabled") is not True:
|
|
return None
|
|
plan = manual_binding_plan(binding, "proxy")
|
|
if plan.get("provider") != "target-egress-proxy":
|
|
raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}")
|
|
target_proxy = plan.get("targetEgressProxy")
|
|
if not isinstance(target_proxy, dict):
|
|
raise RuntimeError(f"manual account proxyBinding source {plan.get('id')} requires an enabled target egressProxy")
|
|
service_name = target_proxy.get("serviceName")
|
|
listen_port = target_proxy.get("listenPort")
|
|
namespace = target_proxy.get("namespace") if isinstance(target_proxy.get("namespace"), str) and target_proxy.get("namespace") else NAMESPACE
|
|
proxy_name = binding.get("proxyName")
|
|
if not isinstance(service_name, str) or not service_name:
|
|
raise RuntimeError("target egressProxy serviceName is missing")
|
|
if not isinstance(listen_port, int) or listen_port <= 0:
|
|
raise RuntimeError("target egressProxy listenPort is missing")
|
|
if not isinstance(proxy_name, str) or not proxy_name:
|
|
raise RuntimeError("manual account proxyBinding proxyName is missing")
|
|
return {
|
|
"name": proxy_name,
|
|
"protocol": "http",
|
|
"host": f"{service_name}.{namespace}.svc.cluster.local",
|
|
"port": listen_port,
|
|
"fallback_mode": "none",
|
|
"expiry_warn_days": 0,
|
|
}
|
|
|
|
def proxy_needs_update(proxy, payload):
|
|
if not isinstance(proxy, dict):
|
|
return True
|
|
for key in ("name", "protocol", "host", "port", "fallback_mode", "expiry_warn_days"):
|
|
if proxy.get(key) != payload.get(key):
|
|
return True
|
|
return proxy.get("status") != "active"
|
|
|
|
def ensure_manual_proxy(token, payload):
|
|
current = find_proxy_by_name(token, payload["name"])
|
|
if current is None:
|
|
created = ensure_success(curl_api("POST", "/api/v1/admin/proxies", bearer=token, payload=payload), f"create proxy {payload['name']}")
|
|
return created, "created"
|
|
if proxy_needs_update(current, payload):
|
|
update_payload = dict(payload)
|
|
update_payload["status"] = "active"
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/proxies/{current['id']}", bearer=token, payload=update_payload), f"update proxy {payload['name']}")
|
|
return updated if isinstance(updated, dict) else current, "updated"
|
|
return current, "unchanged"
|
|
|
|
def manual_proxy_status(token, account, protection):
|
|
payload = desired_manual_proxy_payload(protection)
|
|
if payload is None:
|
|
return {
|
|
"enabled": False,
|
|
"ok": True,
|
|
"action": "not-configured",
|
|
"valuesPrinted": False,
|
|
}
|
|
binding = protection.get("proxyBinding") if isinstance(protection, dict) else None
|
|
plan = manual_binding_plan(binding, "proxy")
|
|
proxy = find_proxy_by_name(token, payload["name"])
|
|
runtime_proxy = account.get("proxy") if isinstance(account, dict) and isinstance(account.get("proxy"), dict) else None
|
|
binding_aligned = (
|
|
isinstance(account, dict)
|
|
and isinstance(proxy, dict)
|
|
and account.get("proxy_id") == proxy.get("id")
|
|
)
|
|
return {
|
|
"enabled": True,
|
|
"ok": binding_aligned,
|
|
"action": "validate",
|
|
"source": plan.get("id"),
|
|
"sourceProvider": plan.get("provider"),
|
|
"expectedProxyName": payload["name"],
|
|
"expectedProtocol": payload["protocol"],
|
|
"expectedHost": payload["host"],
|
|
"expectedPort": payload["port"],
|
|
"proxyRecordExists": isinstance(proxy, dict),
|
|
"proxyId": proxy.get("id") if isinstance(proxy, dict) else None,
|
|
"runtimeProxyId": account.get("proxy_id") if isinstance(account, dict) else None,
|
|
"runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None,
|
|
"bindingAligned": binding_aligned,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def ensure_manual_account_proxy_bindings(token):
|
|
items = []
|
|
for protection in MANUAL_ACCOUNT_PROTECTIONS:
|
|
if not isinstance(protection, dict):
|
|
continue
|
|
name = protection.get("accountName")
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
account = find_account_by_name(token, name)
|
|
payload = desired_manual_proxy_payload(protection)
|
|
if payload is None:
|
|
items.append({
|
|
"accountName": name,
|
|
"enabled": False,
|
|
"action": "not-configured",
|
|
"ok": True,
|
|
"valuesPrinted": False,
|
|
})
|
|
continue
|
|
if not isinstance(account, dict):
|
|
items.append({
|
|
"accountName": name,
|
|
"enabled": True,
|
|
"action": "account-missing",
|
|
"ok": False,
|
|
"expectedProxyName": payload["name"],
|
|
"valuesPrinted": False,
|
|
})
|
|
continue
|
|
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
|
if extra.get("unidesk_managed") is True:
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"enabled": True,
|
|
"action": "refused-managed-runtime-account",
|
|
"ok": False,
|
|
"expectedProxyName": payload["name"],
|
|
"valuesPrinted": False,
|
|
})
|
|
continue
|
|
proxy, proxy_action = ensure_manual_proxy(token, payload)
|
|
binding = protection.get("proxyBinding") if isinstance(protection, dict) else None
|
|
plan = manual_binding_plan(binding, "proxy")
|
|
proxy_id = proxy.get("id") if isinstance(proxy, dict) else None
|
|
if proxy_id is None:
|
|
raise RuntimeError(f"proxy {payload['name']} has no id")
|
|
action = "unchanged"
|
|
if account.get("proxy_id") != proxy_id:
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"proxy_id": proxy_id}), f"bind manual account proxy {name}")
|
|
account = updated if isinstance(updated, dict) else account
|
|
action = "bound"
|
|
runtime_proxy = account.get("proxy") if isinstance(account.get("proxy"), dict) else None
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"enabled": True,
|
|
"action": action,
|
|
"proxyAction": proxy_action,
|
|
"source": plan.get("id"),
|
|
"sourceProvider": plan.get("provider"),
|
|
"ok": account.get("proxy_id") == proxy_id,
|
|
"expectedProxyName": payload["name"],
|
|
"expectedProtocol": payload["protocol"],
|
|
"expectedHost": payload["host"],
|
|
"expectedPort": payload["port"],
|
|
"proxyId": proxy_id,
|
|
"runtimeProxyId": account.get("proxy_id"),
|
|
"runtimeProxyName": runtime_proxy.get("name") if isinstance(runtime_proxy, dict) else None,
|
|
"bindingAligned": account.get("proxy_id") == proxy_id,
|
|
"controlPolicy": "manual-protected: only proxy_id binding is YAML-controlled; credentials/status/schedulable are untouched",
|
|
"valuesPrinted": False,
|
|
})
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in items),
|
|
"itemCount": len(items),
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def manual_group_binding_plan(protection):
|
|
binding = protection.get("groupBinding") if isinstance(protection, dict) else None
|
|
if not isinstance(binding, dict) or binding.get("enabled") is not True:
|
|
return None
|
|
plan = manual_binding_plan(binding, "group")
|
|
if plan.get("provider") != "pool-group":
|
|
raise RuntimeError(f"manual account groupBinding source {plan.get('id')} uses unsupported provider {plan.get('provider')}")
|
|
return plan
|
|
|
|
def manual_group_binding_enabled(protection):
|
|
return manual_group_binding_plan(protection) is not None
|
|
|
|
def manual_group_status(token, account, protection, group_id):
|
|
plan = manual_group_binding_plan(protection)
|
|
if plan is None:
|
|
return {
|
|
"enabled": False,
|
|
"ok": True,
|
|
"action": "not-configured",
|
|
"valuesPrinted": False,
|
|
}
|
|
group_accounts = list_accounts_for_group(token, group_id)
|
|
account_id = account.get("id") if isinstance(account, dict) else None
|
|
account_name = account.get("name") if isinstance(account, dict) else None
|
|
binding_aligned = any(
|
|
item.get("id") == account_id or item.get("name") == account_name
|
|
for item in group_accounts
|
|
if isinstance(item, dict)
|
|
)
|
|
return {
|
|
"enabled": True,
|
|
"ok": binding_aligned,
|
|
"action": "validate",
|
|
"source": plan.get("id"),
|
|
"sourceProvider": plan.get("provider"),
|
|
"poolGroupName": POOL_GROUP_NAME,
|
|
"poolGroupId": group_id,
|
|
"bindingAligned": binding_aligned,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def ensure_manual_account_group_bindings(token, group_id):
|
|
items = []
|
|
for protection in MANUAL_ACCOUNT_PROTECTIONS:
|
|
if not isinstance(protection, dict):
|
|
continue
|
|
name = protection.get("accountName")
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
if not manual_group_binding_enabled(protection):
|
|
items.append({
|
|
"accountName": name,
|
|
"enabled": False,
|
|
"action": "not-configured",
|
|
"ok": True,
|
|
"valuesPrinted": False,
|
|
})
|
|
continue
|
|
plan = manual_group_binding_plan(protection)
|
|
account = find_account_by_name(token, name)
|
|
if not isinstance(account, dict):
|
|
items.append({
|
|
"accountName": name,
|
|
"enabled": True,
|
|
"action": "account-missing",
|
|
"ok": False,
|
|
"poolGroupName": POOL_GROUP_NAME,
|
|
"poolGroupId": group_id,
|
|
"valuesPrinted": False,
|
|
})
|
|
continue
|
|
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
|
if extra.get("unidesk_managed") is True:
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"enabled": True,
|
|
"action": "refused-managed-runtime-account",
|
|
"ok": False,
|
|
"poolGroupName": POOL_GROUP_NAME,
|
|
"poolGroupId": group_id,
|
|
"valuesPrinted": False,
|
|
})
|
|
continue
|
|
existing_group_ids = account_group_ids(token, account)
|
|
desired_group_ids = sorted(set(existing_group_ids + [group_id]))
|
|
action = "unchanged"
|
|
if group_id not in existing_group_ids:
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account['id']}", bearer=token, payload={"group_ids": desired_group_ids}), f"bind manual account group {name}")
|
|
account = updated if isinstance(updated, dict) else account
|
|
action = "bound"
|
|
binding_aligned = any(
|
|
item.get("id") == account.get("id") or item.get("name") == name
|
|
for item in list_accounts_for_group(token, group_id)
|
|
if isinstance(item, dict)
|
|
)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"enabled": True,
|
|
"ok": binding_aligned,
|
|
"action": action,
|
|
"source": plan.get("id") if isinstance(plan, dict) else None,
|
|
"sourceProvider": plan.get("provider") if isinstance(plan, dict) else None,
|
|
"poolGroupName": POOL_GROUP_NAME,
|
|
"poolGroupId": group_id,
|
|
"previousGroupIds": existing_group_ids,
|
|
"desiredGroupIds": desired_group_ids,
|
|
"bindingAligned": binding_aligned,
|
|
"controlPolicy": "manual-protected: only pool group membership is YAML-controlled; credentials/status/schedulable are untouched and sentinel does not probe it",
|
|
"valuesPrinted": False,
|
|
})
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in items),
|
|
"itemCount": len(items),
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def manual_account_protection_status(token, group_id=None):
|
|
items = []
|
|
desired_names = set(EXPECTED_ACCOUNT_CAPACITIES.keys())
|
|
for protection in MANUAL_ACCOUNT_PROTECTIONS:
|
|
if not isinstance(protection, dict):
|
|
continue
|
|
name = protection.get("accountName")
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
account = find_account_by_name(token, name)
|
|
extra = account.get("extra") if isinstance(account, dict) and isinstance(account.get("extra"), dict) else {}
|
|
proxy_status = manual_proxy_status(token, account, protection)
|
|
group_status = manual_group_status(token, account, protection, group_id) if group_id is not None else {"enabled": False, "ok": True, "action": "not-checked", "valuesPrinted": False}
|
|
items.append({
|
|
"accountName": name,
|
|
"reason": protection.get("reason") if isinstance(protection.get("reason"), str) else None,
|
|
"exists": isinstance(account, dict),
|
|
"accountId": account.get("id") if isinstance(account, dict) else None,
|
|
"status": account.get("status") if isinstance(account, dict) else None,
|
|
"schedulable": account.get("schedulable") if isinstance(account, dict) else None,
|
|
"inYamlProfiles": name in desired_names,
|
|
"runtimeMarkedUnideskManaged": extra.get("unidesk_managed") is True,
|
|
"proxyBinding": proxy_status,
|
|
"groupBinding": group_status,
|
|
"ok": proxy_status.get("ok") is True and group_status.get("ok") is True,
|
|
"controlPolicy": "manual-protected: no create/update/prune/probe/freeze; optional proxy_id and pool group membership binding only when configured",
|
|
"valuesPrinted": False,
|
|
})
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in items),
|
|
"protectedCount": len(items),
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def list_probe_accounts(token):
|
|
path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-probe-")
|
|
data = ensure_success(curl_api("GET", path, bearer=token), "list probe accounts")
|
|
return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")]
|
|
|
|
def list_probe_groups(token):
|
|
data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")
|
|
return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")]
|
|
|
|
def cleanup_probe_resources(token):
|
|
api_key_results = []
|
|
account_results = []
|
|
group_results = []
|
|
for item in list_user_keys(token):
|
|
name = item.get("name")
|
|
key_id = item.get("id")
|
|
if not isinstance(name, str) or not name.startswith("unidesk-probe-") or key_id is None:
|
|
continue
|
|
api_key_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{key_id}", "api-key"))
|
|
for item in list_probe_accounts(token):
|
|
account_id = item.get("id")
|
|
if account_id is None:
|
|
continue
|
|
account_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account"))
|
|
for item in list_probe_groups(token):
|
|
group_id = item.get("id")
|
|
if group_id is None:
|
|
continue
|
|
group_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group"))
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in api_key_results + account_results + group_results),
|
|
"apiKeysDeleted": len(api_key_results),
|
|
"accountsDeleted": len(account_results),
|
|
"groupsDeleted": len(group_results),
|
|
"items": {
|
|
"apiKeys": api_key_results,
|
|
"accounts": account_results,
|
|
"groups": group_results,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def temp_unschedulable_credentials(profile):
|
|
credentials = profile.get("tempUnschedulableCredentials")
|
|
if not isinstance(credentials, dict):
|
|
credentials = {}
|
|
return normalize_temp_unschedulable_credentials(credentials)
|
|
|
|
def account_payload(profile, group_id):
|
|
extra = {
|
|
"openai_responses_mode": "force_responses",
|
|
"unidesk_codex_profile": profile["profile"],
|
|
"unidesk_managed": True,
|
|
"unidesk_trust_upstream": profile.get("trustUpstream") is True,
|
|
"unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
|
|
}
|
|
ws_mode = profile.get("openaiResponsesWebSocketsV2Mode")
|
|
if ws_mode:
|
|
extra["openai_apikey_responses_websockets_v2_mode"] = ws_mode
|
|
extra["openai_apikey_responses_websockets_v2_enabled"] = ws_mode != "off"
|
|
credentials = {
|
|
"api_key": profile["apiKey"],
|
|
"base_url": profile["baseUrl"],
|
|
}
|
|
upstream_user_agent = profile.get("upstreamUserAgent")
|
|
if upstream_user_agent:
|
|
credentials["user_agent"] = upstream_user_agent
|
|
temp_unschedulable = temp_unschedulable_credentials(profile)
|
|
if temp_unschedulable["enabled"]:
|
|
credentials["temp_unschedulable_enabled"] = True
|
|
credentials["temp_unschedulable_rules"] = temp_unschedulable["rules"]
|
|
return {
|
|
"name": profile["accountName"],
|
|
"notes": f"UniDesk-managed Codex profile {profile['profile']} from {profile['configFile']} and {profile['authFile']}; secret source={profile['apiKeySource']}; fingerprint={profile['apiKeyFingerprint']}.",
|
|
"platform": "openai",
|
|
"type": "apikey",
|
|
"credentials": credentials,
|
|
"extra": extra,
|
|
"concurrency": int(profile.get("capacity", 5) or 5),
|
|
"priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
|
|
"rate_multiplier": 1,
|
|
"load_factor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR),
|
|
"group_ids": [group_id],
|
|
"confirm_mixed_channel_risk": True,
|
|
}
|
|
|
|
def planned_sentinel_account_results(profiles, existing_accounts):
|
|
existing = {item.get("name"): item for item in existing_accounts if isinstance(item, dict)}
|
|
results = []
|
|
for profile in profiles:
|
|
change_reasons = sentinel_probe_change_reasons(existing.get(profile["accountName"]), profile)
|
|
quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0
|
|
results.append({
|
|
"profile": profile["profile"],
|
|
"accountName": profile["accountName"],
|
|
"profileConfig": {
|
|
"accountName": profile["accountName"],
|
|
"trustUpstream": profile.get("trustUpstream") is True,
|
|
"sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
|
|
},
|
|
"sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"),
|
|
"sentinelProbeRequired": quality_gate_required,
|
|
"sentinelChangeReasons": change_reasons if quality_gate_required else [],
|
|
"sentinelProbePending": quality_gate_required,
|
|
"sentinelDefaultFrozen": False,
|
|
"valuesPrinted": False,
|
|
})
|
|
return results
|
|
|
|
def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_frozen_names=None, existing_accounts=None):
|
|
if not isinstance(protected_frozen_names, set):
|
|
protected_frozen_names = set()
|
|
if not isinstance(existing_accounts, list):
|
|
existing_accounts = list_accounts(token)
|
|
existing = {item.get("name"): item for item in existing_accounts}
|
|
desired_names = {profile["accountName"] for profile in profiles}
|
|
results = []
|
|
for profile in profiles:
|
|
payload = account_payload(profile, group_id)
|
|
current = existing.get(profile["accountName"])
|
|
change_reasons = sentinel_probe_change_reasons(current, profile)
|
|
quality_gate_required = sentinel_quality_gate_enabled() and len(change_reasons) > 0
|
|
keep_frozen = profile["accountName"] in protected_frozen_names
|
|
if current and current.get("id") is not None:
|
|
account_id = current["id"]
|
|
update_payload = dict(payload)
|
|
update_payload.pop("platform", None)
|
|
update_payload["status"] = "active"
|
|
data = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account_id}", bearer=token, payload=update_payload), f"update account {profile['accountName']}")
|
|
action = "updated"
|
|
else:
|
|
data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}")
|
|
action = "created"
|
|
results.append({
|
|
"profile": profile["profile"],
|
|
"accountName": profile["accountName"],
|
|
"profileConfig": {
|
|
"accountName": profile["accountName"],
|
|
"trustUpstream": profile.get("trustUpstream") is True,
|
|
"sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
|
|
},
|
|
"accountId": data.get("id") if isinstance(data, dict) else None,
|
|
"action": action,
|
|
"baseUrl": profile["baseUrl"],
|
|
"apiKeySource": profile["apiKeySource"],
|
|
"apiKeyFingerprint": profile["apiKeyFingerprint"],
|
|
"sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"),
|
|
"sentinelProbeRequired": quality_gate_required,
|
|
"sentinelChangeReasons": change_reasons if quality_gate_required else [],
|
|
"sentinelProbePending": quality_gate_required,
|
|
"sentinelDefaultFrozen": False,
|
|
"sentinelFreezeProtected": keep_frozen,
|
|
"schedulableControl": "sentinel-marker-restore",
|
|
"openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"),
|
|
"trustUpstream": profile.get("trustUpstream") is True,
|
|
"priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
|
|
"capacity": int(profile.get("capacity", 5) or 5),
|
|
"loadFactor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR),
|
|
"runtimeConcurrency": data.get("concurrency") if isinstance(data, dict) else None,
|
|
"runtimeLoadFactor": data.get("load_factor") if isinstance(data, dict) else None,
|
|
"runtimeSchedulable": data.get("schedulable") if isinstance(data, dict) else None,
|
|
"tempUnschedulableConfigured": bool(payload["credentials"].get("temp_unschedulable_enabled")),
|
|
"tempUnschedulableRuleCount": len(payload["credentials"].get("temp_unschedulable_rules") or []),
|
|
"upstreamUserAgentConfigured": bool(profile.get("upstreamUserAgent")),
|
|
"valuesPrinted": False,
|
|
})
|
|
prune_results = prune_removed_accounts(token, existing_accounts, desired_names) if prune_removed else []
|
|
return results, prune_results
|
|
|
|
def prune_removed_accounts(token, existing_accounts, desired_names):
|
|
results = []
|
|
for account in existing_accounts:
|
|
name = account.get("name")
|
|
account_id = account.get("id")
|
|
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
|
if not isinstance(name, str) or not name.startswith("unidesk-codex-"):
|
|
continue
|
|
if extra.get("unidesk_managed") is not True:
|
|
continue
|
|
if name in desired_names:
|
|
continue
|
|
if account_id is None:
|
|
raise RuntimeError(f"removed account {name} has no id")
|
|
ensure_success(curl_api("DELETE", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"delete removed account {name}")
|
|
results.append({
|
|
"accountName": name,
|
|
"accountId": account_id,
|
|
"profile": extra.get("unidesk_codex_profile"),
|
|
"action": "deleted",
|
|
"reason": "removed-from-yaml",
|
|
"valuesPrinted": False,
|
|
})
|
|
return results
|
|
|
|
def generate_api_key():
|
|
alphabet = string.ascii_letters + string.digits
|
|
return "sk-unidesk-codex-" + "".join(secrets.choice(alphabet) for _ in range(48))
|
|
|
|
def ensure_api_key_secret(group_id):
|
|
existing = None
|
|
try:
|
|
existing = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY)
|
|
except Exception:
|
|
existing = None
|
|
api_key = existing if existing else generate_api_key()
|
|
secret_action = "kept-existing" if existing else "created"
|
|
manifest = {
|
|
"apiVersion": "v1",
|
|
"kind": "Secret",
|
|
"metadata": {
|
|
"name": POOL_API_KEY_SECRET_NAME,
|
|
"namespace": NAMESPACE,
|
|
"labels": {
|
|
"app.kubernetes.io/name": "sub2api",
|
|
"app.kubernetes.io/part-of": "platform-infra",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/secret-purpose": "sub2api-codex-pool-api-key",
|
|
},
|
|
},
|
|
"type": "Opaque",
|
|
"stringData": {
|
|
POOL_API_KEY_SECRET_KEY: api_key,
|
|
"GROUP_ID": str(group_id),
|
|
"GROUP_NAME": POOL_GROUP_NAME,
|
|
"SERVICE_DNS": SERVICE_DNS,
|
|
},
|
|
}
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"apply API key secret failed: {text(proc.stderr, 1000)}")
|
|
return api_key, secret_action, text(proc.stdout, 1000)
|
|
|
|
def apply_sentinel_manifest(manifest):
|
|
if not isinstance(manifest, str) or not manifest.strip():
|
|
return {
|
|
"ok": False,
|
|
"action": "missing-manifest",
|
|
"valuesPrinted": False,
|
|
}
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest)
|
|
if proc.returncode != 0:
|
|
return {
|
|
"ok": False,
|
|
"action": "apply-failed",
|
|
"stdoutTail": text(proc.stdout, 2000),
|
|
"stderrTail": text(proc.stderr, 4000),
|
|
"valuesPrinted": False,
|
|
}
|
|
status = sentinel_runtime_status()
|
|
return {
|
|
"ok": status.get("ok") is True,
|
|
"action": "applied",
|
|
"stdoutTail": text(proc.stdout, 2000),
|
|
"runtime": status,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def safe_kube_json(args, label):
|
|
proc = kubectl([*args, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)}
|
|
try:
|
|
return json.loads(proc.stdout.decode("utf-8")), None
|
|
except Exception as exc:
|
|
return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)}
|
|
|
|
def sentinel_runtime_status():
|
|
cfg = SENTINEL_CONFIG
|
|
cronjob_name = cfg.get("cronJobName")
|
|
secret_name = cfg.get("credentialsSecretName")
|
|
configmap_name = cfg.get("configMapName")
|
|
state_name = cfg.get("stateConfigMapName")
|
|
cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
|
|
secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}")
|
|
configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}")
|
|
state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
|
|
state = None
|
|
if isinstance(state_cm, dict):
|
|
raw_state = (state_cm.get("data") or {}).get("state.json")
|
|
if isinstance(raw_state, str) and raw_state:
|
|
try:
|
|
state = json.loads(raw_state)
|
|
except Exception as exc:
|
|
state = {"parseError": str(exc)}
|
|
accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {}
|
|
quarantined = []
|
|
recent_accounts = []
|
|
for name, account_state in accounts.items():
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
|
quarantined.append({
|
|
"accountName": name,
|
|
"until": quarantine.get("until"),
|
|
"applied": quarantine.get("applied"),
|
|
"reason": quarantine.get("reason"),
|
|
"failureKind": quarantine.get("failureKind"),
|
|
"errorDetails": quarantine.get("errorDetails"),
|
|
"intervalMinutes": quarantine.get("intervalMinutes"),
|
|
"sentinelProtect": quarantine.get("sentinelProtect"),
|
|
})
|
|
last_probe = account_state.get("lastProbe")
|
|
if isinstance(last_probe, dict):
|
|
recent_accounts.append({
|
|
"accountName": name,
|
|
"lastProbeAt": account_state.get("lastProbeAt"),
|
|
"lastStatus": account_state.get("lastStatus"),
|
|
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
|
"ok": last_probe.get("ok"),
|
|
"purpose": last_probe.get("purpose"),
|
|
"httpStatus": last_probe.get("httpStatus"),
|
|
"durationMs": last_probe.get("durationMs"),
|
|
"markerMatched": last_probe.get("markerMatched"),
|
|
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
|
|
"outputHash": last_probe.get("outputHash"),
|
|
"outputPreview": last_probe.get("outputPreview"),
|
|
"responseBodyHash": last_probe.get("responseBodyHash"),
|
|
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
|
"error": last_probe.get("error"),
|
|
"errorDetails": last_probe.get("errorDetails"),
|
|
"usage": last_probe.get("usage"),
|
|
"failureKind": last_probe.get("failureKind"),
|
|
"requestShape": last_probe.get("requestShape"),
|
|
"action": last_probe.get("action"),
|
|
})
|
|
recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
|
|
last_run = state.get("lastRun") if isinstance(state, dict) else None
|
|
cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
|
|
secret_data = secret.get("data") if isinstance(secret, dict) else {}
|
|
configmap_data = configmap.get("data") if isinstance(configmap, dict) else {}
|
|
ok = cronjob is not None and secret is not None and configmap is not None
|
|
return {
|
|
"ok": ok,
|
|
"desired": {
|
|
"monitorEnabled": cfg.get("monitor", {}).get("enabled"),
|
|
"actionsEnabled": cfg.get("actions", {}).get("enabled"),
|
|
"schedule": cfg.get("schedule"),
|
|
"cronJobName": cronjob_name,
|
|
"configMapName": configmap_name,
|
|
"credentialsSecretName": secret_name,
|
|
"stateConfigMapName": state_name,
|
|
},
|
|
"cronJob": {
|
|
"exists": cronjob is not None,
|
|
"schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None,
|
|
"suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None,
|
|
"lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None,
|
|
"active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None,
|
|
"error": cronjob_error,
|
|
},
|
|
"secret": {
|
|
"exists": secret is not None,
|
|
"profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data,
|
|
"valuesPrinted": False,
|
|
"error": secret_error,
|
|
},
|
|
"configMap": {
|
|
"exists": configmap is not None,
|
|
"configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data,
|
|
"runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data,
|
|
"error": configmap_error,
|
|
},
|
|
"state": {
|
|
"exists": state_cm is not None,
|
|
"accountCount": len(accounts),
|
|
"quarantinedCount": len(quarantined),
|
|
"quarantined": quarantined[-10:],
|
|
"recentAccounts": recent_accounts[-12:],
|
|
"lastRun": last_run,
|
|
"error": state_error,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def parse_epoch_z(value):
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
|
except Exception:
|
|
return None
|
|
|
|
def sentinel_state_object():
|
|
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
|
|
if not state_name:
|
|
return None, None
|
|
obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
|
|
if not isinstance(obj, dict):
|
|
return None, None
|
|
raw_state = (obj.get("data") or {}).get("state.json")
|
|
if not isinstance(raw_state, str) or not raw_state:
|
|
return obj, None
|
|
try:
|
|
return obj, json.loads(raw_state)
|
|
except Exception:
|
|
return obj, None
|
|
|
|
def active_sentinel_quarantine_names():
|
|
_, state = sentinel_state_object()
|
|
if not isinstance(state, dict):
|
|
return set()
|
|
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
names = set()
|
|
for name, account_state in accounts_state.items():
|
|
if not isinstance(name, str) or not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if isinstance(quarantine, dict) and quarantine.get("active") is True and quarantine.get("applied") is True:
|
|
names.add(name)
|
|
return names
|
|
|
|
def default_sentinel_state():
|
|
return {"version": 1, "accounts": {}, "ledger": {}, "history": []}
|
|
|
|
def clamp_sentinel_freezes_for_config(state, now):
|
|
freeze_config = SENTINEL_CONFIG.get("freeze") if isinstance(SENTINEL_CONFIG.get("freeze"), dict) else {}
|
|
try:
|
|
max_interval = int(freeze_config.get("maxTtlMinutes") or 10)
|
|
except Exception:
|
|
max_interval = 10
|
|
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
now_epoch = time.time()
|
|
items = []
|
|
for name, account_state in accounts_state.items():
|
|
if not isinstance(name, str) or not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
|
|
continue
|
|
try:
|
|
interval = int(quarantine.get("intervalMinutes") or 0)
|
|
except Exception:
|
|
interval = 0
|
|
until_epoch = parse_epoch_z(quarantine.get("until"))
|
|
old_until = quarantine.get("until")
|
|
if interval <= max_interval and (until_epoch is None or until_epoch <= now_epoch + max_interval * 60):
|
|
continue
|
|
quarantine["previousIntervalMinutes"] = interval
|
|
quarantine["intervalMinutes"] = max_interval
|
|
quarantine["until"] = now
|
|
quarantine["clampedAt"] = now
|
|
quarantine["clampedBy"] = "sync-freeze-max-ttl"
|
|
account_state["nextProbeAfter"] = now
|
|
items.append({
|
|
"accountName": name,
|
|
"previousIntervalMinutes": interval,
|
|
"maxIntervalMinutes": max_interval,
|
|
"previousUntil": old_until,
|
|
"nextProbeAfter": now,
|
|
})
|
|
return items
|
|
|
|
def parse_iso_epoch(value):
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
|
except Exception:
|
|
return None
|
|
|
|
def profile_success_max_interval(profile):
|
|
cadence = SENTINEL_CONFIG.get("cadence") if isinstance(SENTINEL_CONFIG.get("cadence"), dict) else {}
|
|
legacy = cadence.get("successMaxIntervalMinutes")
|
|
if legacy is None:
|
|
legacy = cadence.get("trustedSuccessMaxIntervalMinutes") or cadence.get("untrustedSuccessMaxIntervalMinutes") or 1
|
|
if profile.get("trustUpstream") is True:
|
|
value = cadence.get("trustedSuccessMaxIntervalMinutes") or legacy
|
|
else:
|
|
value = cadence.get("untrustedSuccessMaxIntervalMinutes") or legacy
|
|
try:
|
|
return int(value)
|
|
except Exception:
|
|
return int(legacy)
|
|
|
|
def clamp_sentinel_success_cadence_for_config(state, profiles, now):
|
|
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
profile_map = {item.get("accountName"): item for item in profiles if isinstance(item, dict) and isinstance(item.get("accountName"), str)}
|
|
now_epoch = time.time()
|
|
items = []
|
|
for name, profile in profile_map.items():
|
|
account_state = accounts_state.get(name)
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
|
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
|
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
|
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile)
|
|
continue
|
|
try:
|
|
interval = int(account_state.get("successIntervalMinutes") or 0)
|
|
except Exception:
|
|
interval = 0
|
|
next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter"))
|
|
max_interval = profile_success_max_interval(profile)
|
|
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
|
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
|
account_state["successMaxIntervalMinutes"] = max_interval
|
|
if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60):
|
|
continue
|
|
old_next = account_state.get("nextProbeAfter")
|
|
account_state["previousSuccessIntervalMinutes"] = interval
|
|
account_state["successIntervalMinutes"] = min(interval, max_interval) if interval > 0 else interval
|
|
account_state["nextProbeAfter"] = now
|
|
account_state["cadenceClampedAt"] = now
|
|
account_state["cadenceClampedBy"] = "sync-success-max-interval"
|
|
items.append({
|
|
"accountName": name,
|
|
"trustUpstream": profile.get("trustUpstream") is True,
|
|
"previousSuccessIntervalMinutes": interval,
|
|
"maxIntervalMinutes": max_interval,
|
|
"previousNextProbeAfter": old_next,
|
|
"nextProbeAfter": now,
|
|
})
|
|
return items
|
|
|
|
def update_sentinel_state_configmap(obj, state):
|
|
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
|
|
if not state_name:
|
|
return {"ok": False, "reason": "state-configmap-missing"}
|
|
state_json = json.dumps(state, ensure_ascii=False, indent=2)
|
|
manifest = {
|
|
"apiVersion": "v1",
|
|
"kind": "ConfigMap",
|
|
"metadata": {
|
|
"name": state_name,
|
|
"namespace": NAMESPACE,
|
|
"labels": {
|
|
"app.kubernetes.io/name": SERVICE_NAME,
|
|
"app.kubernetes.io/component": "account-sentinel",
|
|
"app.kubernetes.io/managed-by": "unidesk-platform-infra",
|
|
},
|
|
},
|
|
"data": {"state.json": state_json},
|
|
}
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
|
action = "applied" if isinstance(obj, dict) else "created"
|
|
if proc.returncode != 0:
|
|
return {"ok": False, "reason": f"{action}-failed", "stderrTail": text(proc.stderr, 2000)}
|
|
return {"ok": True, "action": action}
|
|
|
|
def ensure_sentinel_state_for_sync(account_results, pending_only=False):
|
|
if not sentinel_quality_gate_enabled():
|
|
return {"ok": True, "skipped": True, "reason": "sentinel-quality-gate-disabled", "changedCount": 0, "items": [], "valuesPrinted": False}
|
|
state_obj, state = sentinel_state_object()
|
|
if not isinstance(state, dict):
|
|
state = default_sentinel_state()
|
|
state.setdefault("version", 1)
|
|
accounts_state = state.setdefault("accounts", {})
|
|
if not isinstance(accounts_state, dict):
|
|
accounts_state = {}
|
|
state["accounts"] = accounts_state
|
|
now = utc_iso()
|
|
items = []
|
|
clamped_items = [] if pending_only else clamp_sentinel_freezes_for_config(state, now)
|
|
cadence_clamped_items = [] if pending_only else clamp_sentinel_success_cadence_for_config(state, [item.get("profileConfig") for item in account_results if isinstance(item.get("profileConfig"), dict)], now)
|
|
changed_count = 0
|
|
fingerprint_only_count = 0
|
|
for item in account_results:
|
|
name = item.get("accountName")
|
|
if not isinstance(name, str) or not name:
|
|
continue
|
|
account_state = accounts_state.setdefault(name, {})
|
|
if not isinstance(account_state, dict):
|
|
account_state = {}
|
|
accounts_state[name] = account_state
|
|
fingerprint_value = item.get("sentinelProbeConfigFingerprint")
|
|
if isinstance(fingerprint_value, str) and fingerprint_value:
|
|
account_state["probeConfigFingerprint"] = fingerprint_value
|
|
if item.get("sentinelProbeRequired") is not True:
|
|
fingerprint_only_count += 1
|
|
continue
|
|
changed_count += 1
|
|
reasons = item.get("sentinelChangeReasons") if isinstance(item.get("sentinelChangeReasons"), list) else []
|
|
quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else None
|
|
if not (isinstance(quarantine, dict) and quarantine.get("active") is True):
|
|
account_state["quarantine"] = {
|
|
"active": False,
|
|
"reason": "yaml-account-change-pending-sentinel-probe",
|
|
"lastPendingAt": now,
|
|
"changeReasons": reasons,
|
|
}
|
|
account_state["nextProbeAfter"] = now
|
|
account_state["successStreak"] = 0
|
|
account_state["successIntervalMinutes"] = 0
|
|
profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {}
|
|
account_state["trustUpstream"] = profile_config.get("trustUpstream") is True
|
|
account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False}
|
|
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config)
|
|
account_state["lastStatus"] = "pending-sentinel-quality-gate"
|
|
account_state["qualityGate"] = {
|
|
"pending": True,
|
|
"reason": "yaml-account-change",
|
|
"changeReasons": reasons,
|
|
"markedAt": now,
|
|
"pendingOnly": pending_only,
|
|
"defaultFrozen": False,
|
|
}
|
|
items.append({"accountName": name, "changeReasons": reasons, "nextProbeAfter": now, "defaultFrozen": False, "defaultSchedulable": True, "pendingOnly": pending_only})
|
|
if changed_count <= 0 and len(clamped_items) <= 0 and len(cadence_clamped_items) <= 0:
|
|
return {"ok": True, "skipped": False, "reason": "no-new-or-changed-accounts", "changedCount": 0, "fingerprintOnlyCount": fingerprint_only_count, "clampedCount": 0, "cadenceClampedCount": 0, "items": [], "valuesPrinted": False}
|
|
update = update_sentinel_state_configmap(state_obj, state)
|
|
if pending_only and changed_count > 0:
|
|
reason = "new-or-changed-accounts-pending-probe-prepared-default-schedulable"
|
|
elif changed_count > 0 and (len(clamped_items) > 0 or len(cadence_clamped_items) > 0):
|
|
reason = "new-or-changed-accounts-default-schedulable-and-sentinel-cadence-clamped"
|
|
elif changed_count > 0:
|
|
reason = "new-or-changed-accounts-default-schedulable"
|
|
elif len(cadence_clamped_items) > 0:
|
|
reason = "success-cadence-clamped-to-current-config"
|
|
else:
|
|
reason = "freeze-backoff-clamped-to-current-config"
|
|
return {
|
|
"ok": update.get("ok") is True,
|
|
"skipped": False,
|
|
"reason": reason,
|
|
"changedCount": changed_count,
|
|
"fingerprintOnlyCount": fingerprint_only_count,
|
|
"clampedCount": len(clamped_items),
|
|
"cadenceClampedCount": len(cadence_clamped_items),
|
|
"pendingOnly": pending_only,
|
|
"items": items,
|
|
"clampedItems": clamped_items,
|
|
"cadenceClampedItems": cadence_clamped_items,
|
|
"update": update,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def sentinel_state_summary():
|
|
_, state = sentinel_state_object()
|
|
if not isinstance(state, dict):
|
|
return {"exists": False, "valuesPrinted": False}
|
|
accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
quarantined = []
|
|
recent_accounts = []
|
|
for name, account_state in accounts.items():
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
|
quarantined.append({
|
|
"accountName": name,
|
|
"until": quarantine.get("until"),
|
|
"applied": quarantine.get("applied"),
|
|
"reason": quarantine.get("reason"),
|
|
"failureKind": quarantine.get("failureKind"),
|
|
"errorDetails": quarantine.get("errorDetails"),
|
|
"intervalMinutes": quarantine.get("intervalMinutes"),
|
|
"sentinelProtect": quarantine.get("sentinelProtect"),
|
|
})
|
|
last_probe = account_state.get("lastProbe")
|
|
if isinstance(last_probe, dict):
|
|
recent_accounts.append({
|
|
"accountName": name,
|
|
"lastProbeAt": account_state.get("lastProbeAt"),
|
|
"lastStatus": account_state.get("lastStatus"),
|
|
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
|
"ok": last_probe.get("ok"),
|
|
"purpose": last_probe.get("purpose"),
|
|
"httpStatus": last_probe.get("httpStatus"),
|
|
"durationMs": last_probe.get("durationMs"),
|
|
"markerMatched": last_probe.get("markerMatched"),
|
|
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
|
|
"usage": last_probe.get("usage"),
|
|
"responseBodyHash": last_probe.get("responseBodyHash"),
|
|
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
|
"error": last_probe.get("error"),
|
|
"errorDetails": last_probe.get("errorDetails"),
|
|
"failureKind": last_probe.get("failureKind"),
|
|
"requestShape": last_probe.get("requestShape"),
|
|
"action": last_probe.get("action"),
|
|
})
|
|
recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
|
|
return {
|
|
"exists": True,
|
|
"accountCount": len(accounts),
|
|
"quarantinedCount": len(quarantined),
|
|
"quarantined": quarantined[-10:],
|
|
"recentAccounts": recent_accounts[-12:],
|
|
"lastRun": state.get("lastRun"),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def reassert_sentinel_freezes_after_sync(token):
|
|
if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True:
|
|
return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False}
|
|
_, state = sentinel_state_object()
|
|
if not isinstance(state, dict):
|
|
return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False}
|
|
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
for name, account_state in accounts_state.items():
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
|
|
continue
|
|
account = by_name.get(name)
|
|
if not account or account.get("id") is None:
|
|
items.append({"accountName": name, "ok": False, "reason": "account-not-found"})
|
|
continue
|
|
try:
|
|
ensure_success(
|
|
curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}),
|
|
f"reassert sentinel freeze for {name}",
|
|
)
|
|
items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")})
|
|
except Exception as exc:
|
|
items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)})
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in items),
|
|
"skipped": False,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def list_user_keys(token):
|
|
data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys")
|
|
return extract_items(data)
|
|
|
|
def ensure_sub2api_api_key(token, api_key, group_id):
|
|
keys = list_user_keys(token)
|
|
existing = next((item for item in keys if item.get("key") == api_key), None)
|
|
action = "kept-existing"
|
|
if existing is None:
|
|
existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None)
|
|
if existing is None or existing.get("key") != api_key:
|
|
payload = {
|
|
"name": POOL_API_KEY_NAME,
|
|
"group_id": group_id,
|
|
"custom_key": api_key,
|
|
"quota": 0,
|
|
"rate_limit_5h": 0,
|
|
"rate_limit_1d": 0,
|
|
"rate_limit_7d": 0,
|
|
}
|
|
created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key")
|
|
existing = created if isinstance(created, dict) else existing
|
|
action = "created"
|
|
elif existing.get("id") is not None and existing.get("group_id") != group_id:
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group")
|
|
existing = updated if isinstance(updated, dict) else existing
|
|
action = "updated-group"
|
|
return {
|
|
"action": action,
|
|
"id": existing.get("id") if isinstance(existing, dict) else None,
|
|
"name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME,
|
|
"groupId": existing.get("group_id") if isinstance(existing, dict) else group_id,
|
|
"userId": existing.get("user_id") if isinstance(existing, dict) else None,
|
|
}
|
|
|
|
def get_admin_user(token, user_id):
|
|
data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner")
|
|
if not isinstance(data, dict):
|
|
raise RuntimeError("API key owner response is not an object")
|
|
return data
|
|
|
|
def ensure_pool_owner_balance(token, user_id):
|
|
user = get_admin_user(token, user_id)
|
|
current = float(user.get("balance") or 0)
|
|
if current >= MIN_OWNER_BALANCE_USD:
|
|
return {
|
|
"action": "kept-existing",
|
|
"userId": user_id,
|
|
"balanceBefore": current,
|
|
"balanceAfter": current,
|
|
"minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
|
|
}
|
|
updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={
|
|
"balance": MIN_OWNER_BALANCE_USD,
|
|
"operation": "set",
|
|
"notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.",
|
|
}), "set API key owner balance")
|
|
after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD
|
|
return {
|
|
"action": "set",
|
|
"userId": user_id,
|
|
"balanceBefore": current,
|
|
"balanceAfter": after,
|
|
"minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
|
|
}
|
|
|
|
def ensure_pool_owner_concurrency(token, user_id):
|
|
user = get_admin_user(token, user_id)
|
|
try:
|
|
current = int(user.get("concurrency") or 0)
|
|
except Exception:
|
|
current = 0
|
|
if current >= MIN_OWNER_CONCURRENCY:
|
|
return {
|
|
"ok": True,
|
|
"action": "kept-existing",
|
|
"userId": user_id,
|
|
"concurrencyBefore": current,
|
|
"concurrencyAfter": current,
|
|
"minimumConcurrency": MIN_OWNER_CONCURRENCY,
|
|
"minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
|
}
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={
|
|
"concurrency": MIN_OWNER_CONCURRENCY,
|
|
}), "set API key owner concurrency")
|
|
try:
|
|
after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY
|
|
except Exception:
|
|
after = MIN_OWNER_CONCURRENCY
|
|
return {
|
|
"ok": after >= MIN_OWNER_CONCURRENCY,
|
|
"action": "set",
|
|
"userId": user_id,
|
|
"concurrencyBefore": current,
|
|
"concurrencyAfter": after,
|
|
"minimumConcurrency": MIN_OWNER_CONCURRENCY,
|
|
"minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
|
}
|
|
|
|
def validate_gateway(api_key):
|
|
resp = curl_api("GET", "/v1/models", bearer=api_key)
|
|
parsed = resp.get("json")
|
|
model_count = None
|
|
if isinstance(parsed, dict) and isinstance(parsed.get("data"), list):
|
|
model_count = len(parsed["data"])
|
|
return {
|
|
"ok": resp.get("ok"),
|
|
"httpStatus": resp.get("httpStatus"),
|
|
"transportExitCode": resp.get("transportExitCode"),
|
|
"modelCount": model_count,
|
|
"bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "",
|
|
"stderr": resp.get("stderr", ""),
|
|
"method": "GET /v1/models",
|
|
"serviceDns": SERVICE_DNS,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def response_output_preview(parsed):
|
|
if not isinstance(parsed, dict):
|
|
return ""
|
|
if isinstance(parsed.get("output_text"), str):
|
|
return parsed["output_text"][:240]
|
|
output = parsed.get("output")
|
|
if not isinstance(output, list):
|
|
return ""
|
|
parts = []
|
|
for item in output:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
for block in content:
|
|
if not isinstance(block, dict):
|
|
continue
|
|
text_value = block.get("text")
|
|
if isinstance(text_value, str) and text_value:
|
|
parts.append(text_value)
|
|
return "\\n".join(parts)[:240]
|
|
|
|
def request_log_evidence(request_id):
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=5m", "--tail=800"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
lines = [line for line in stdout.splitlines() if request_id in line]
|
|
failovers = []
|
|
final = None
|
|
for line in lines:
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
continue
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
continue
|
|
if "upstream_failover_switching" in line:
|
|
failovers.append({
|
|
"accountId": item.get("account_id"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
if "http request completed" in line:
|
|
final = {
|
|
"accountId": item.get("account_id"),
|
|
"statusCode": item.get("status_code"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
"path": item.get("path"),
|
|
}
|
|
return {
|
|
"requestId": request_id,
|
|
"matchedLogLineCount": len(lines),
|
|
"failovers": failovers,
|
|
"final": final,
|
|
"logsExitCode": proc.returncode,
|
|
"logsStderr": text(proc.stderr, 1000),
|
|
}
|
|
|
|
def recent_compact_gateway_evidence():
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
failures = []
|
|
successes = []
|
|
failovers = []
|
|
final_errors = []
|
|
context_canceled = []
|
|
for line in stdout.splitlines():
|
|
if "/responses/compact" not in line and "remote_compact" not in line:
|
|
continue
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
continue
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
continue
|
|
path = item.get("path")
|
|
entry = {
|
|
"requestId": item.get("request_id"),
|
|
"clientRequestId": item.get("client_request_id"),
|
|
"accountId": item.get("account_id"),
|
|
"statusCode": item.get("status_code"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
"path": path,
|
|
}
|
|
if "codex.remote_compact.failed" in line:
|
|
failures.append(entry)
|
|
elif "codex.remote_compact.succeeded" in line:
|
|
successes.append(entry)
|
|
elif "upstream_failover_switching" in line and path == "/responses/compact":
|
|
failovers.append({
|
|
**entry,
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
|
final_errors.append(entry)
|
|
if "context canceled" in line and path == "/responses/compact":
|
|
context_canceled.append(entry)
|
|
return {
|
|
"ok": True,
|
|
"degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
|
|
"window": "6h",
|
|
"tailLines": 2500,
|
|
"failureCount": len(failures),
|
|
"successCount": len(successes),
|
|
"failoverCount": len(failovers),
|
|
"finalErrorCount": len(final_errors),
|
|
"contextCanceledCount": len(context_canceled),
|
|
"recentFailures": failures[-5:],
|
|
"recentSuccesses": successes[-5:],
|
|
"recentFailovers": failovers[-8:],
|
|
"recentFinalErrors": final_errors[-5:],
|
|
"recentContextCanceled": context_canceled[-5:],
|
|
"logsExitCode": proc.returncode,
|
|
"logsStderr": text(proc.stderr, 1000),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def is_ignored_probe_noise(entry):
|
|
for value in (entry.get("requestId"), entry.get("clientRequestId")):
|
|
if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"):
|
|
return True
|
|
return False
|
|
|
|
def filter_ignored_probe_noise(items):
|
|
return [item for item in items if not is_ignored_probe_noise(item)]
|
|
|
|
def ignored_probe_noise(items, section):
|
|
result = []
|
|
for item in items:
|
|
if is_ignored_probe_noise(item):
|
|
probe = dict(item)
|
|
probe["section"] = section
|
|
result.append(probe)
|
|
return result
|
|
|
|
def group_by_request_id(items):
|
|
grouped = {}
|
|
for item in items:
|
|
request_id = item.get("requestId")
|
|
if not isinstance(request_id, str) or not request_id:
|
|
continue
|
|
grouped.setdefault(request_id, []).append(item)
|
|
return grouped
|
|
|
|
def failover_budget_exhausted_evidence(failovers, final_errors):
|
|
final_by_request = {}
|
|
for item in final_errors:
|
|
request_id = item.get("requestId")
|
|
if isinstance(request_id, str) and request_id:
|
|
final_by_request[request_id] = item
|
|
exhausted = []
|
|
for request_id, request_failovers in group_by_request_id(failovers).items():
|
|
final = final_by_request.get(request_id)
|
|
if not final:
|
|
continue
|
|
last = request_failovers[-1]
|
|
switch_count = last.get("switchCount")
|
|
max_switches = last.get("maxSwitches")
|
|
final_status = final.get("statusCode")
|
|
if (
|
|
isinstance(switch_count, int)
|
|
and isinstance(max_switches, int)
|
|
and max_switches > 0
|
|
and switch_count >= max_switches
|
|
and isinstance(final_status, int)
|
|
and final_status >= 500
|
|
):
|
|
exhausted.append({
|
|
"requestId": request_id,
|
|
"clientRequestId": final.get("clientRequestId") or last.get("clientRequestId"),
|
|
"path": final.get("path") or last.get("path"),
|
|
"finalAccountId": final.get("accountId"),
|
|
"finalStatusCode": final_status,
|
|
"switchCount": switch_count,
|
|
"maxSwitches": max_switches,
|
|
"lastFailoverAccountId": last.get("accountId"),
|
|
"lastUpstreamStatus": last.get("upstreamStatus"),
|
|
})
|
|
return exhausted
|
|
|
|
def recent_responses_gateway_evidence():
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
failovers = []
|
|
forward_failures = []
|
|
final_errors = []
|
|
context_canceled = []
|
|
slow_final_errors = []
|
|
for line in stdout.splitlines():
|
|
if '"/responses"' not in line and '"/v1/responses"' not in line:
|
|
continue
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
continue
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
continue
|
|
path = item.get("path")
|
|
if path not in ("/responses", "/v1/responses"):
|
|
continue
|
|
entry = {
|
|
"requestId": item.get("request_id"),
|
|
"clientRequestId": item.get("client_request_id"),
|
|
"accountId": item.get("account_id"),
|
|
"statusCode": item.get("status_code"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
"path": path,
|
|
}
|
|
if "upstream_failover_switching" in line:
|
|
failovers.append({
|
|
**entry,
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
elif "openai.forward_failed" in line:
|
|
forward_failures.append({
|
|
**entry,
|
|
"errorPreview": text(str(item.get("error") or ""), 500),
|
|
"fallbackErrorResponseWritten": item.get("fallback_error_response_written"),
|
|
"upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"),
|
|
})
|
|
elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
|
final_errors.append(entry)
|
|
latency_ms = item.get("latency_ms")
|
|
if isinstance(latency_ms, int) and latency_ms >= 30000:
|
|
slow_final_errors.append(entry)
|
|
if "context canceled" in line:
|
|
context_canceled.append(entry)
|
|
visible_failovers = filter_ignored_probe_noise(failovers)
|
|
visible_forward_failures = filter_ignored_probe_noise(forward_failures)
|
|
visible_final_errors = filter_ignored_probe_noise(final_errors)
|
|
visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors)
|
|
visible_context_canceled = filter_ignored_probe_noise(context_canceled)
|
|
failover_budget_exhausted = failover_budget_exhausted_evidence(visible_failovers, visible_final_errors)
|
|
probe_noise = (
|
|
ignored_probe_noise(failovers, "failovers")
|
|
+ ignored_probe_noise(forward_failures, "forwardFailures")
|
|
+ ignored_probe_noise(final_errors, "finalErrors")
|
|
+ ignored_probe_noise(slow_final_errors, "slowFinalErrors")
|
|
+ ignored_probe_noise(context_canceled, "contextCanceled")
|
|
)
|
|
return {
|
|
"ok": True,
|
|
"degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0,
|
|
"window": "6h",
|
|
"tailLines": 2500,
|
|
"failoverCount": len(visible_failovers),
|
|
"forwardFailureCount": len(visible_forward_failures),
|
|
"finalErrorCount": len(visible_final_errors),
|
|
"slowFinalErrorCount": len(visible_slow_final_errors),
|
|
"contextCanceledCount": len(visible_context_canceled),
|
|
"ignoredProbeNoiseCount": len(probe_noise),
|
|
"failoverBudgetExhausted": failover_budget_exhausted[-8:],
|
|
"rawCounts": {
|
|
"failoverCount": len(failovers),
|
|
"forwardFailureCount": len(forward_failures),
|
|
"finalErrorCount": len(final_errors),
|
|
"slowFinalErrorCount": len(slow_final_errors),
|
|
"contextCanceledCount": len(context_canceled),
|
|
},
|
|
"recentFailovers": visible_failovers[-8:],
|
|
"recentForwardFailures": visible_forward_failures[-8:],
|
|
"recentFinalErrors": visible_final_errors[-8:],
|
|
"recentSlowFinalErrors": visible_slow_final_errors[-5:],
|
|
"recentContextCanceled": visible_context_canceled[-5:],
|
|
"recentProbeNoise": probe_noise[-5:],
|
|
"logsExitCode": proc.returncode,
|
|
"logsStderr": text(proc.stderr, 1000),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def validate_gateway_responses(api_key):
|
|
request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
|
|
payload = {
|
|
"model": RESPONSES_SMOKE_MODEL,
|
|
"input": "Reply exactly: unidesk-sub2api-validate-ok",
|
|
"stream": False,
|
|
"store": False,
|
|
"max_output_tokens": 32,
|
|
}
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
script = r'''
|
|
set -eu
|
|
token="$1"
|
|
request_id="$2"
|
|
tmp="$(mktemp)"
|
|
trap 'rm -f "$tmp"' EXIT
|
|
cat > "$tmp"
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Content-Type: application/json' \
|
|
-H "X-Request-ID: $request_id" \
|
|
-H "OpenAI-Client-Request-ID: $request_id" \
|
|
--data-binary @"$tmp" \
|
|
http://127.0.0.1:8080/v1/responses
|
|
'''
|
|
started = time.time()
|
|
proc = run([
|
|
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
|
"--", "sh", "-c", script, "sh", api_key, request_id,
|
|
], body)
|
|
resp = parse_curl_output(proc)
|
|
evidence = request_log_evidence(request_id)
|
|
parsed = resp.get("json")
|
|
failover_count = len(evidence.get("failovers") or [])
|
|
return {
|
|
"ok": resp.get("ok"),
|
|
"degraded": failover_count > 0,
|
|
"outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"),
|
|
"httpStatus": resp.get("httpStatus"),
|
|
"transportExitCode": resp.get("transportExitCode"),
|
|
"method": "POST /v1/responses",
|
|
"model": RESPONSES_SMOKE_MODEL,
|
|
"requestId": request_id,
|
|
"durationMs": int((time.time() - started) * 1000),
|
|
"outputTextPreview": response_output_preview(parsed),
|
|
"bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800),
|
|
"stderr": resp.get("stderr", ""),
|
|
"evidence": evidence,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def bool_value(value):
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
if value.lower() == "true":
|
|
return True
|
|
if value.lower() == "false":
|
|
return False
|
|
return False
|
|
|
|
def normalize_temp_unschedulable_credentials(credentials):
|
|
if not isinstance(credentials, dict):
|
|
credentials = {}
|
|
enabled = bool_value(credentials.get("temp_unschedulable_enabled"))
|
|
raw_rules = credentials.get("temp_unschedulable_rules")
|
|
if isinstance(raw_rules, str):
|
|
try:
|
|
raw_rules = json.loads(raw_rules)
|
|
except json.JSONDecodeError:
|
|
raw_rules = []
|
|
rules = []
|
|
if isinstance(raw_rules, list):
|
|
for rule in raw_rules:
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
error_code = rule.get("error_code", rule.get("statusCode"))
|
|
duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes"))
|
|
keywords = rule.get("keywords")
|
|
if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list):
|
|
continue
|
|
clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()]
|
|
if not clean_keywords:
|
|
continue
|
|
description = rule.get("description") if isinstance(rule.get("description"), str) else ""
|
|
rules.append({
|
|
"error_code": error_code,
|
|
"keywords": clean_keywords,
|
|
"duration_minutes": duration_minutes,
|
|
"description": description,
|
|
})
|
|
return {
|
|
"enabled": enabled,
|
|
"rules": rules,
|
|
}
|
|
|
|
def summarize_temp_unschedulable_rules(rules):
|
|
return [{
|
|
"errorCode": rule.get("error_code"),
|
|
"durationMinutes": rule.get("duration_minutes"),
|
|
"keywordCount": len(rule.get("keywords") or []),
|
|
"keywords": rule.get("keywords") or [],
|
|
"hasDescription": bool(rule.get("description")),
|
|
} for rule in rules]
|
|
|
|
def success_body_reclassification_requirement():
|
|
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
|
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
|
if expected["enabled"] is not True:
|
|
continue
|
|
for rule in expected["rules"]:
|
|
error_code = rule.get("error_code")
|
|
keywords = rule.get("keywords") or []
|
|
if isinstance(error_code, int) and 200 <= error_code < 300 and keywords:
|
|
return {
|
|
"required": True,
|
|
"sourceAccountName": name,
|
|
"statusCode": error_code,
|
|
"keywords": keywords,
|
|
"representativeKeyword": keywords[0],
|
|
"durationMinutes": rule.get("duration_minutes"),
|
|
}
|
|
return {
|
|
"required": False,
|
|
"sourceAccountName": None,
|
|
"statusCode": None,
|
|
"keywords": [],
|
|
"representativeKeyword": None,
|
|
"durationMinutes": None,
|
|
}
|
|
|
|
def model_routing_400_failover_requirement():
|
|
preferred = ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"]
|
|
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
|
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
|
if expected["enabled"] is not True:
|
|
continue
|
|
for rule in expected["rules"]:
|
|
error_code = rule.get("error_code")
|
|
keywords = rule.get("keywords") or []
|
|
if error_code != 400 or not keywords:
|
|
continue
|
|
representative_keyword = next((item for item in preferred if item in keywords), keywords[0])
|
|
return {
|
|
"required": True,
|
|
"sourceAccountName": name,
|
|
"statusCode": error_code,
|
|
"keywords": keywords,
|
|
"representativeKeyword": representative_keyword,
|
|
"durationMinutes": rule.get("duration_minutes"),
|
|
}
|
|
return {
|
|
"required": False,
|
|
"sourceAccountName": None,
|
|
"statusCode": None,
|
|
"keywords": [],
|
|
"representativeKeyword": None,
|
|
"durationMinutes": None,
|
|
}
|
|
|
|
def delete_probe_resource(token, method, path, label):
|
|
if not path:
|
|
return {"label": label, "ok": True, "skipped": True}
|
|
resp = curl_api(method, path, bearer=token)
|
|
ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410)
|
|
return {
|
|
"label": label,
|
|
"ok": ok,
|
|
"method": method,
|
|
"path": path,
|
|
"httpStatus": resp.get("httpStatus"),
|
|
"transportExitCode": resp.get("transportExitCode"),
|
|
"bodyPreview": "" if ok else text(resp.get("body", ""), 500),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def validate_runtime_capabilities(token):
|
|
success_body = success_body_reclassification_requirement()
|
|
model_routing_400 = model_routing_400_failover_requirement()
|
|
return {
|
|
"ok": success_body.get("required") is not True and model_routing_400.get("required") is True,
|
|
"runtimeImage": app_pod_runtime_image(),
|
|
"successBodyReclassification": {
|
|
"ok": success_body.get("required") is not True,
|
|
"required": success_body.get("required"),
|
|
"outcome": "not-required-by-yaml" if success_body.get("required") is not True else "requires-source-review-and-real-traffic-evidence",
|
|
"requirement": success_body,
|
|
"valuesPrinted": False,
|
|
},
|
|
"modelRouting400Failover": {
|
|
"ok": model_routing_400.get("required") is True,
|
|
"required": model_routing_400.get("required"),
|
|
"outcome": "yaml-rules-present-runtime-observed-by-real-traffic",
|
|
"requirement": model_routing_400,
|
|
"evidence": "Use Sub2API source review plus validation.gatewayResponsesRecent and Artificer/real request ids for runtime proof; default validate does not create mock upstreams or temporary failover accounts.",
|
|
"valuesPrinted": False,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def app_pod_runtime_image():
|
|
try:
|
|
pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}")
|
|
spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else []
|
|
status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else []
|
|
spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {})
|
|
status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {})
|
|
return {
|
|
"pod": APP_POD,
|
|
"image": spec.get("image"),
|
|
"imageID": status.get("imageID"),
|
|
"ready": status.get("ready"),
|
|
"restartCount": status.get("restartCount"),
|
|
}
|
|
except Exception as exc:
|
|
return {"pod": APP_POD, "error": str(exc)}
|
|
|
|
def get_account_detail(token, account):
|
|
account_id = account.get("id") if isinstance(account, dict) else None
|
|
if account_id is None:
|
|
return account
|
|
data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}")
|
|
return data if isinstance(data, dict) else account
|
|
|
|
def account_temp_unschedulable_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
enabled_names = []
|
|
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
|
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedEnabled": expected["enabled"],
|
|
"runtimeEnabled": None,
|
|
"expectedRuleCount": len(expected["rules"]),
|
|
"runtimeRuleCount": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
detail = get_account_detail(token, account)
|
|
credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {}
|
|
runtime = normalize_temp_unschedulable_credentials(credentials)
|
|
temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil")
|
|
temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or ""
|
|
ok = runtime == expected
|
|
if expected["enabled"]:
|
|
enabled_names.append(name)
|
|
if not ok:
|
|
mismatched.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedEnabled": expected["enabled"],
|
|
"runtimeEnabled": runtime["enabled"],
|
|
"expectedRuleCount": len(expected["rules"]),
|
|
"runtimeRuleCount": len(runtime["rules"]),
|
|
"expectedRules": summarize_temp_unschedulable_rules(expected["rules"]),
|
|
"runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]),
|
|
"status": account.get("status"),
|
|
"schedulable": account.get("schedulable"),
|
|
"tempUnschedulableUntil": temp_until,
|
|
"tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "",
|
|
"tempUnschedulableSet": temp_until is not None or bool(temp_reason),
|
|
"ok": ok,
|
|
})
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0,
|
|
"desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE),
|
|
"enabled": enabled_names,
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def account_capacity_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
expected_capacity_total = 0
|
|
runtime_concurrency_total = 0
|
|
schedulable_runtime_concurrency_total = 0
|
|
available_runtime_concurrency_total = 0
|
|
temp_unschedulable_runtime_concurrency_total = 0
|
|
for name in sorted(EXPECTED_ACCOUNT_CAPACITIES):
|
|
expected = int(EXPECTED_ACCOUNT_CAPACITIES[name])
|
|
expected_capacity_total += expected
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedCapacity": expected,
|
|
"runtimeConcurrency": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
runtime = account.get("concurrency")
|
|
runtime_int = runtime if isinstance(runtime, int) else 0
|
|
runtime_concurrency_total += runtime_int
|
|
if account.get("schedulable") is True:
|
|
runtime_status = account.get("status")
|
|
if runtime_status is None or runtime_status == "active":
|
|
runtime_status_ok = True
|
|
else:
|
|
runtime_status_ok = False
|
|
if runtime_status_ok:
|
|
schedulable_runtime_concurrency_total += runtime_int
|
|
temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil")
|
|
temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason")
|
|
if temp_until is None and not bool(temp_reason):
|
|
available_runtime_concurrency_total += runtime_int
|
|
else:
|
|
temp_unschedulable_runtime_concurrency_total += runtime_int
|
|
ok = runtime == expected
|
|
if not ok:
|
|
mismatched.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedCapacity": expected,
|
|
"runtimeConcurrency": runtime,
|
|
"priority": account.get("priority"),
|
|
"status": account.get("status"),
|
|
"schedulable": account.get("schedulable"),
|
|
"ok": ok,
|
|
})
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0,
|
|
"defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY,
|
|
"desired": len(EXPECTED_ACCOUNT_CAPACITIES),
|
|
"totals": {
|
|
"expectedCapacityTotal": expected_capacity_total,
|
|
"runtimeConcurrencyTotal": runtime_concurrency_total,
|
|
"schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total,
|
|
"availableRuntimeConcurrencyTotal": available_runtime_concurrency_total,
|
|
"tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total,
|
|
"minOwnerConcurrency": MIN_OWNER_CONCURRENCY,
|
|
"minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
|
},
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def account_load_factor_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS):
|
|
expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name])
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedLoadFactor": expected,
|
|
"runtimeLoadFactor": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
runtime_raw = account.get("load_factor")
|
|
if runtime_raw is None:
|
|
detail = get_account_detail(token, account)
|
|
runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None
|
|
try:
|
|
runtime = int(runtime_raw)
|
|
except Exception:
|
|
runtime = runtime_raw
|
|
ok = runtime == expected
|
|
if not ok:
|
|
mismatched.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedLoadFactor": expected,
|
|
"runtimeLoadFactor": runtime,
|
|
"priority": account.get("priority"),
|
|
"status": account.get("status"),
|
|
"schedulable": account.get("schedulable"),
|
|
"ok": ok,
|
|
})
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0,
|
|
"defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR,
|
|
"desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS),
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def account_ws_v2_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
enabled_names = []
|
|
unschedulable_enabled = []
|
|
schedulable_enabled = []
|
|
for name in sorted(EXPECTED_ACCOUNT_WS_MODES):
|
|
expected_mode = EXPECTED_ACCOUNT_WS_MODES[name]
|
|
expected_enabled = expected_mode not in (None, "", "off")
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedMode": expected_mode,
|
|
"expectedEnabled": expected_enabled,
|
|
"runtimeMode": None,
|
|
"runtimeEnabled": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
|
runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode")
|
|
runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled")
|
|
schedulable = account.get("schedulable")
|
|
if expected_mode == "off":
|
|
ok = runtime_mode == "off" and runtime_enabled is False
|
|
elif expected_mode is None:
|
|
ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False)
|
|
else:
|
|
ok = runtime_mode == expected_mode and runtime_enabled is True
|
|
if not ok:
|
|
mismatched.append(name)
|
|
if expected_enabled:
|
|
enabled_names.append(name)
|
|
if schedulable is True:
|
|
schedulable_enabled.append(name)
|
|
else:
|
|
unschedulable_enabled.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedMode": expected_mode,
|
|
"expectedEnabled": expected_enabled,
|
|
"runtimeMode": runtime_mode,
|
|
"runtimeEnabled": runtime_enabled,
|
|
"status": account.get("status"),
|
|
"schedulable": schedulable,
|
|
"ok": ok and ((not expected_enabled) or schedulable is True),
|
|
})
|
|
availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok,
|
|
"desired": len(EXPECTED_ACCOUNT_WS_MODES),
|
|
"enabled": enabled_names,
|
|
"schedulableEnabled": schedulable_enabled,
|
|
"unschedulableEnabled": unschedulable_enabled,
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def api_key_preview(api_key):
|
|
if len(api_key) <= 14:
|
|
return "***"
|
|
return api_key[:10] + "..." + api_key[-4:]
|
|
|
|
def run_sync():
|
|
global MANUAL_ACCOUNT_PROTECTIONS
|
|
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8"))
|
|
manual_accounts_payload = payload.get("manualAccounts") if isinstance(payload.get("manualAccounts"), dict) else {}
|
|
resolved_manual_protections = manual_accounts_payload.get("protected")
|
|
if not isinstance(resolved_manual_protections, list):
|
|
raise RuntimeError("sync payload has no manualAccounts.protected binding plan")
|
|
MANUAL_ACCOUNT_PROTECTIONS = resolved_manual_protections
|
|
profiles = payload.get("profiles") or []
|
|
prune_removed = bool(payload.get("pruneRemoved"))
|
|
sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {}
|
|
if not profiles:
|
|
raise RuntimeError("sync payload has no profiles")
|
|
admin_email, token, admin_compliance = login()
|
|
group, group_action = ensure_group(token)
|
|
group_id = group.get("id") if isinstance(group, dict) else None
|
|
if group_id is None:
|
|
raise RuntimeError("pool group id missing after ensure")
|
|
existing_accounts = list_accounts(token)
|
|
planned_account_results = planned_sentinel_account_results(profiles, existing_accounts)
|
|
sentinel_quality_prepare = ensure_sentinel_state_for_sync(planned_account_results, True)
|
|
if sentinel_quality_prepare.get("ok") is not True:
|
|
raise RuntimeError("prepare sentinel pending probe failed: " + json.dumps(sentinel_quality_prepare, ensure_ascii=False))
|
|
protected_frozen_names = active_sentinel_quarantine_names()
|
|
account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed, protected_frozen_names, existing_accounts)
|
|
manual_account_proxy_bindings = ensure_manual_account_proxy_bindings(token)
|
|
manual_account_group_bindings = ensure_manual_account_group_bindings(token, group_id)
|
|
manual_account_protections = manual_account_protection_status(token, group_id)
|
|
capacity_status = account_capacity_status(token)
|
|
load_factor_status = account_load_factor_status(token)
|
|
ws_v2_status = account_ws_v2_status(token)
|
|
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
|
api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id)
|
|
api_key_result = ensure_sub2api_api_key(token, api_key, group_id)
|
|
owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"])
|
|
owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"])
|
|
gateway = validate_gateway(api_key)
|
|
responses_smoke = validate_gateway_responses(api_key)
|
|
compact_evidence = recent_compact_gateway_evidence()
|
|
responses_evidence = recent_responses_gateway_evidence()
|
|
runtime_capabilities = validate_runtime_capabilities(token)
|
|
sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest"))
|
|
sentinel_quality = ensure_sentinel_state_for_sync(account_results)
|
|
sentinel_reassert = reassert_sentinel_freezes_after_sync(token)
|
|
return {
|
|
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_proxy_bindings.get("ok") is True and manual_account_group_bindings.get("ok") is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and sentinel_quality_prepare.get("ok") is True and sentinel_quality.get("ok") is True and sentinel_reassert.get("ok") is True and runtime_capabilities.get("ok") is True,
|
|
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
|
|
"mode": "sync",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"},
|
|
"accounts": {
|
|
"desired": len(profiles),
|
|
"created": sum(1 for item in account_results if item["action"] == "created"),
|
|
"updated": sum(1 for item in account_results if item["action"] == "updated"),
|
|
"pruned": len(pruned_account_results),
|
|
"pruneMode": "explicit" if prune_removed else "disabled-by-default",
|
|
"items": account_results,
|
|
"prunedItems": pruned_account_results,
|
|
"processControl": {"schedulableRestore": "sentinel marker probe only; sync does not restore schedulable for existing accounts", "durableConfig": False},
|
|
"valuesPrinted": False,
|
|
},
|
|
"manualAccounts": {**manual_account_protections, "proxySync": manual_account_proxy_bindings, "groupSync": manual_account_group_bindings},
|
|
"capacity": capacity_status,
|
|
"loadFactor": load_factor_status,
|
|
"webSocketsV2": ws_v2_status,
|
|
"tempUnschedulable": temp_unschedulable_status,
|
|
"apiKey": {
|
|
"name": POOL_API_KEY_NAME,
|
|
"secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}",
|
|
"secretAction": secret_action,
|
|
"secretApply": secret_apply_stdout,
|
|
"sub2apiAction": api_key_result["action"],
|
|
"sub2apiId": api_key_result["id"],
|
|
"groupId": api_key_result["groupId"],
|
|
"userId": api_key_result["userId"],
|
|
"keyPreview": api_key_preview(api_key),
|
|
"valuesPrinted": False,
|
|
},
|
|
"ownerBalance": owner_balance,
|
|
"ownerConcurrency": owner_concurrency,
|
|
"sentinel": {**sentinel, "qualityGatePrepare": sentinel_quality_prepare, "qualityGate": sentinel_quality, "freezeReassert": sentinel_reassert},
|
|
"runtimeCapabilities": runtime_capabilities,
|
|
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
|
|
}
|
|
|
|
def run_validate():
|
|
api_key = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY)
|
|
if not api_key:
|
|
raise RuntimeError(f"{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY} missing")
|
|
admin_email, token, admin_compliance = login()
|
|
key_item = next((item for item in list_user_keys(token) if item.get("key") == api_key), None)
|
|
owner_balance = None
|
|
owner_concurrency = None
|
|
if key_item is not None and key_item.get("user_id") is not None:
|
|
owner_balance = ensure_pool_owner_balance(token, key_item["user_id"])
|
|
owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"])
|
|
capacity_status = account_capacity_status(token)
|
|
load_factor_status = account_load_factor_status(token)
|
|
ws_v2_status = account_ws_v2_status(token)
|
|
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
|
pool_group_id = key_item.get("group_id") if isinstance(key_item, dict) else None
|
|
manual_account_protections = manual_account_protection_status(token, pool_group_id)
|
|
gateway = validate_gateway(api_key)
|
|
responses_smoke = validate_gateway_responses(api_key)
|
|
compact_evidence = recent_compact_gateway_evidence()
|
|
responses_evidence = recent_responses_gateway_evidence()
|
|
runtime_capabilities = validate_runtime_capabilities(token)
|
|
sentinel = sentinel_runtime_status()
|
|
return {
|
|
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and manual_account_protections.get("ok") is True and sentinel.get("ok") is True and runtime_capabilities.get("ok") is True,
|
|
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
|
|
"mode": "validate",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"apiKey": {
|
|
"secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}",
|
|
"sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None,
|
|
"userId": key_item.get("user_id") if isinstance(key_item, dict) else None,
|
|
"groupId": key_item.get("group_id") if isinstance(key_item, dict) else None,
|
|
"keyPreview": api_key_preview(api_key),
|
|
"valuesPrinted": False,
|
|
},
|
|
"ownerBalance": owner_balance,
|
|
"ownerConcurrency": owner_concurrency,
|
|
"capacity": capacity_status,
|
|
"loadFactor": load_factor_status,
|
|
"webSocketsV2": ws_v2_status,
|
|
"tempUnschedulable": temp_unschedulable_status,
|
|
"manualAccounts": manual_account_protections,
|
|
"sentinel": sentinel,
|
|
"runtimeCapabilities": runtime_capabilities,
|
|
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
|
|
}
|
|
|
|
def parse_log_line(line):
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
return None
|
|
prefix = line[:json_start].rstrip()
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
return None
|
|
if not isinstance(item, dict):
|
|
return None
|
|
at = None
|
|
parts = prefix.split()
|
|
if parts:
|
|
at = parts[0]
|
|
message = ""
|
|
if len(parts) >= 4:
|
|
message = " ".join(parts[3:])
|
|
elif len(parts) >= 3:
|
|
message = parts[2]
|
|
elif len(parts) >= 1:
|
|
message = parts[-1]
|
|
item["_at"] = at
|
|
item["_message"] = message
|
|
item["_line"] = line
|
|
return item
|
|
|
|
def log_time_epoch(item):
|
|
at = item.get("_at") if isinstance(item, dict) else None
|
|
if not isinstance(at, str) or not at:
|
|
return None
|
|
try:
|
|
return datetime.strptime(at, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
|
|
except Exception:
|
|
try:
|
|
return datetime.fromisoformat(at.replace("Z", "+00:00")).timestamp()
|
|
except Exception:
|
|
return None
|
|
|
|
def event_base(item, account_names_by_id):
|
|
account_id = item.get("account_id")
|
|
if isinstance(account_id, str) and account_id.isdigit():
|
|
account_id = int(account_id)
|
|
account_name = account_names_by_id.get(account_id)
|
|
return {
|
|
"at": item.get("_at"),
|
|
"message": item.get("_message"),
|
|
"requestId": item.get("request_id"),
|
|
"clientRequestId": item.get("client_request_id"),
|
|
"path": item.get("path"),
|
|
"method": item.get("method"),
|
|
"model": item.get("model"),
|
|
"accountId": account_id,
|
|
"accountName": account_name,
|
|
"statusCode": item.get("status_code"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
}
|
|
|
|
def classify_trace_event(item, account_names_by_id):
|
|
message = str(item.get("_message") or "")
|
|
event = event_base(item, account_names_by_id)
|
|
if "content_moderation.gateway_check_start" in message:
|
|
event.update({
|
|
"type": "request-start",
|
|
"stream": item.get("stream"),
|
|
"bodyBytes": item.get("body_bytes"),
|
|
"groupId": item.get("group_id"),
|
|
"groupName": item.get("group_name"),
|
|
"apiKeyName": item.get("api_key_name"),
|
|
})
|
|
elif "content_moderation.gateway_check_done" in message:
|
|
event.update({
|
|
"type": "gateway-check",
|
|
"allowed": item.get("allowed"),
|
|
"blocked": item.get("blocked"),
|
|
"action": item.get("action"),
|
|
})
|
|
elif "openai.upstream_failover_switching" in message:
|
|
event.update({
|
|
"type": "failover",
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
elif "openai.account_select_failed" in message:
|
|
event.update({
|
|
"type": "select-failed",
|
|
"error": item.get("error"),
|
|
"excludedAccountCount": item.get("excluded_account_count"),
|
|
})
|
|
elif "account_upstream_error" in message:
|
|
event.update({
|
|
"type": "upstream-error",
|
|
"error": item.get("error"),
|
|
})
|
|
elif "account_temp_unschedulable" in message:
|
|
event.update({
|
|
"type": "temp-unschedulable",
|
|
"until": item.get("until") or item.get("temp_unschedulable_until"),
|
|
"ruleIndex": item.get("rule_index"),
|
|
"matchedKeyword": item.get("matched_keyword"),
|
|
"reason": item.get("reason") or item.get("error"),
|
|
})
|
|
elif "http request completed" in message:
|
|
event.update({
|
|
"type": "final",
|
|
"clientIp": item.get("client_ip"),
|
|
"protocol": item.get("protocol"),
|
|
"platform": item.get("platform"),
|
|
"completedAt": item.get("completed_at"),
|
|
})
|
|
elif "admin account schedulable updated" in message or "account schedulable updated" in message or "/schedulable" in str(item.get("path") or ""):
|
|
event.update({
|
|
"type": "admin-schedulable",
|
|
"schedulable": item.get("schedulable"),
|
|
})
|
|
else:
|
|
event.update({"type": "other"})
|
|
return event
|
|
|
|
def with_trace_phase(event, first_epoch, last_epoch):
|
|
epoch = None
|
|
at = event.get("at") if isinstance(event, dict) else None
|
|
if isinstance(at, str) and at:
|
|
epoch = log_time_epoch({"_at": at})
|
|
if epoch is None or first_epoch is None:
|
|
phase = "unknown"
|
|
elif epoch < first_epoch:
|
|
phase = "before-request"
|
|
elif last_epoch is not None and epoch > last_epoch:
|
|
phase = "after-request"
|
|
else:
|
|
phase = "during-request"
|
|
event["phase"] = phase
|
|
return event
|
|
|
|
def account_snapshot_from_runtime(token):
|
|
try:
|
|
accounts = list_accounts(token)
|
|
except Exception as exc:
|
|
return [], {"error": str(exc)}
|
|
rows = []
|
|
for item in accounts:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
rows.append({
|
|
"accountId": item.get("id"),
|
|
"accountName": item.get("name"),
|
|
"schedulable": item.get("schedulable"),
|
|
"status": item.get("status"),
|
|
"concurrency": item.get("concurrency"),
|
|
"priority": item.get("priority"),
|
|
"tempUnschedulableUntil": item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil"),
|
|
"tempUnschedulableSet": (item.get("temp_unschedulable_until") or item.get("tempUnschedulableUntil")) is not None or bool(item.get("temp_unschedulable_reason") or item.get("tempUnschedulableReason")),
|
|
})
|
|
rows.sort(key=lambda row: (str(row.get("accountName") or ""), int(row.get("accountId") or 0)))
|
|
return rows, None
|
|
|
|
def trace_reason(events, final_event):
|
|
failovers = [item for item in events if item.get("type") == "failover"]
|
|
select_failures = [item for item in events if item.get("type") == "select-failed"]
|
|
upstream_errors = [item for item in events if item.get("type") == "upstream-error"]
|
|
if failover_budget_exhausted(failovers, final_event):
|
|
return "failover-budget-exhausted"
|
|
if failovers and select_failures:
|
|
return "failover-attempted-no-candidate"
|
|
if failovers:
|
|
return "failover-attempted"
|
|
if select_failures:
|
|
return "account-select-failed"
|
|
if upstream_errors:
|
|
return "upstream-error"
|
|
if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") >= 400:
|
|
return "final-http-error"
|
|
if isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int):
|
|
return "completed"
|
|
return "unknown"
|
|
|
|
def failover_budget_exhausted(failovers, final_event):
|
|
if not failovers or not isinstance(final_event, dict):
|
|
return False
|
|
last = failovers[-1]
|
|
switch_count = last.get("switchCount")
|
|
max_switches = last.get("maxSwitches")
|
|
final_status = final_event.get("statusCode")
|
|
return (
|
|
isinstance(switch_count, int)
|
|
and isinstance(max_switches, int)
|
|
and max_switches > 0
|
|
and switch_count >= max_switches
|
|
and isinstance(final_status, int)
|
|
and final_status >= 500
|
|
)
|
|
|
|
def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot):
|
|
if not failover_budget_exhausted(failovers, final_event):
|
|
return []
|
|
tried = set()
|
|
for item in failovers:
|
|
account_id = item.get("accountId")
|
|
if isinstance(account_id, int):
|
|
tried.add(account_id)
|
|
final_account = final_event.get("accountId") if isinstance(final_event, dict) else None
|
|
if isinstance(final_account, int):
|
|
tried.add(final_account)
|
|
result = []
|
|
for item in account_snapshot:
|
|
account_id = item.get("accountId")
|
|
if not isinstance(account_id, int) or account_id in tried:
|
|
continue
|
|
if item.get("schedulable") is True and item.get("status") == "active" and item.get("tempUnschedulableSet") is not True:
|
|
result.append({
|
|
"accountId": account_id,
|
|
"accountName": item.get("accountName"),
|
|
"priority": item.get("priority"),
|
|
"concurrency": item.get("concurrency"),
|
|
})
|
|
return result
|
|
|
|
def run_trace():
|
|
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
|
|
request_id = payload.get("requestId")
|
|
since = payload.get("since") or "24h"
|
|
tail = int(payload.get("tail") or 20000)
|
|
context_seconds = int(payload.get("contextSeconds") or 300)
|
|
show_lines = bool(payload.get("showLines"))
|
|
if not isinstance(request_id, str) or not request_id:
|
|
raise RuntimeError("trace payload missing requestId")
|
|
admin_email, token, admin_compliance = login()
|
|
account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token)
|
|
account_names_by_id = {}
|
|
for row in account_snapshot:
|
|
account_id = row.get("accountId")
|
|
if isinstance(account_id, str) and account_id.isdigit():
|
|
account_id = int(account_id)
|
|
if isinstance(account_id, int) and isinstance(row.get("accountName"), str):
|
|
account_names_by_id[account_id] = row.get("accountName")
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
parsed_lines = []
|
|
matched = []
|
|
for line in stdout.splitlines():
|
|
parsed = parse_log_line(line)
|
|
if parsed is None:
|
|
continue
|
|
parsed_lines.append(parsed)
|
|
if request_id in line:
|
|
matched.append(parsed)
|
|
first_epoch = None
|
|
last_epoch = None
|
|
for item in matched:
|
|
epoch = log_time_epoch(item)
|
|
if epoch is None:
|
|
continue
|
|
first_epoch = epoch if first_epoch is None else min(first_epoch, epoch)
|
|
last_epoch = epoch if last_epoch is None else max(last_epoch, epoch)
|
|
window_lines = []
|
|
if first_epoch is not None:
|
|
start_epoch = first_epoch - context_seconds
|
|
end_epoch = (last_epoch if last_epoch is not None else first_epoch) + context_seconds
|
|
for item in parsed_lines:
|
|
epoch = log_time_epoch(item)
|
|
if epoch is not None and start_epoch <= epoch <= end_epoch:
|
|
window_lines.append(item)
|
|
else:
|
|
window_lines = matched
|
|
events = [classify_trace_event(item, account_names_by_id) for item in matched]
|
|
request_start = next((item for item in events if item.get("type") == "request-start"), None)
|
|
final_event = next((item for item in reversed(events) if item.get("type") == "final"), None)
|
|
failovers = [item for item in events if item.get("type") == "failover"]
|
|
select_failures = [item for item in events if item.get("type") == "select-failed"]
|
|
upstream_errors = [item for item in events if item.get("type") == "upstream-error"]
|
|
temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")]
|
|
admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))]
|
|
window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines]
|
|
final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400]
|
|
window_failovers = [item for item in window_events if item.get("type") == "failover"]
|
|
window_select_failures = [item for item in window_events if item.get("type") == "select-failed"]
|
|
untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot)
|
|
reason = trace_reason(events, final_event)
|
|
if not matched:
|
|
outcome = "not-found"
|
|
elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400:
|
|
outcome = "succeeded"
|
|
elif isinstance(final_event, dict):
|
|
outcome = "failed"
|
|
else:
|
|
outcome = "incomplete"
|
|
return {
|
|
"ok": proc.returncode == 0 and len(matched) > 0,
|
|
"mode": "trace",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"requestId": request_id,
|
|
"summary": {
|
|
"outcome": outcome,
|
|
"reason": reason,
|
|
"eventCount": len(events),
|
|
"matchedLineCount": len(matched),
|
|
"firstAt": events[0].get("at") if events else None,
|
|
"lastAt": events[-1].get("at") if events else None,
|
|
},
|
|
"window": {
|
|
"since": since,
|
|
"tail": tail,
|
|
"beforeSeconds": context_seconds,
|
|
"afterSeconds": context_seconds,
|
|
"lineCount": len(window_lines),
|
|
},
|
|
"request": request_start or {},
|
|
"final": final_event or {},
|
|
"events": events,
|
|
"failovers": failovers,
|
|
"selectFailures": select_failures,
|
|
"upstreamErrors": upstream_errors,
|
|
"tempUnschedulable": temp_unsched,
|
|
"adminSchedulable": admin_sched[-20:],
|
|
"windowStats": {
|
|
"matchedLines": len(matched),
|
|
"eventCount": len(window_events),
|
|
"finalErrorCount": len(final_errors),
|
|
"failoverCount": len(window_failovers),
|
|
"selectFailedCount": len(window_select_failures),
|
|
"tempUnschedulableCount": len(temp_unsched),
|
|
"adminSchedulableCount": len(admin_sched),
|
|
},
|
|
"diagnostics": {
|
|
"failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}),
|
|
"untriedSchedulableAccounts": untried_schedulable_accounts,
|
|
},
|
|
"accountSnapshot": account_snapshot,
|
|
"accountSnapshotError": account_snapshot_error,
|
|
"rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [],
|
|
"showLines": show_lines,
|
|
"logs": {
|
|
"exitCode": proc.returncode,
|
|
"stderrTail": text(proc.stderr, 1000),
|
|
"stdoutLineCount": len(stdout.splitlines()),
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def parse_embedded_json(stdout):
|
|
if not isinstance(stdout, str) or not stdout.strip():
|
|
return None
|
|
start = stdout.find("{")
|
|
end = stdout.rfind("}")
|
|
if start < 0 or end <= start:
|
|
return None
|
|
try:
|
|
return json.loads(stdout[start:end + 1])
|
|
except Exception:
|
|
return None
|
|
|
|
def inject_env(template, name, value):
|
|
spec = template.setdefault("spec", {})
|
|
containers = spec.setdefault("containers", [])
|
|
if not containers:
|
|
raise RuntimeError("sentinel job template has no containers")
|
|
container = containers[0]
|
|
env = container.setdefault("env", [])
|
|
env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)]
|
|
env.append({"name": name, "value": value})
|
|
|
|
def sentinel_probe_job_manifest(accounts):
|
|
cronjob_name = SENTINEL_CONFIG.get("cronJobName")
|
|
if not isinstance(cronjob_name, str) or not cronjob_name:
|
|
raise RuntimeError("sentinel cronJobName missing from config")
|
|
cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
|
|
job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {}
|
|
job_spec = copy_json(job_template.get("spec") or {})
|
|
if not isinstance(job_spec, dict) or not job_spec:
|
|
raise RuntimeError("sentinel CronJob jobTemplate.spec missing")
|
|
template = job_spec.get("template")
|
|
if not isinstance(template, dict):
|
|
raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing")
|
|
inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts))
|
|
job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600)
|
|
suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5))
|
|
job_name = ("sub2api-sentinel-probe-" + suffix)[:63]
|
|
return job_name, {
|
|
"apiVersion": "batch/v1",
|
|
"kind": "Job",
|
|
"metadata": {
|
|
"name": job_name,
|
|
"namespace": NAMESPACE,
|
|
"labels": {
|
|
"app.kubernetes.io/name": cronjob_name,
|
|
"app.kubernetes.io/part-of": "platform-infra",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe",
|
|
},
|
|
},
|
|
"spec": job_spec,
|
|
}
|
|
|
|
def copy_json(value):
|
|
return json.loads(json.dumps(value))
|
|
|
|
def job_condition(job, cond_type):
|
|
for item in ((job.get("status") or {}).get("conditions") or []):
|
|
if item.get("type") == cond_type and item.get("status") == "True":
|
|
return item
|
|
return None
|
|
|
|
def wait_sentinel_probe_job(job_name, timeout_seconds):
|
|
deadline = time.time() + timeout_seconds
|
|
latest = None
|
|
while time.time() < deadline:
|
|
latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}")
|
|
if isinstance(latest, dict):
|
|
complete = job_condition(latest, "Complete")
|
|
failed = job_condition(latest, "Failed")
|
|
if complete is not None:
|
|
return "succeeded", latest, complete
|
|
if failed is not None:
|
|
return "failed", latest, failed
|
|
time.sleep(2)
|
|
return "timeout", latest, None
|
|
|
|
def job_logs(job_name):
|
|
proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"])
|
|
return {
|
|
"exitCode": proc.returncode,
|
|
"stdout": proc.stdout.decode("utf-8", errors="replace"),
|
|
"stderr": text(proc.stderr, 4000),
|
|
}
|
|
|
|
def run_sentinel_probe():
|
|
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
|
|
accounts = payload.get("accounts") if isinstance(payload, dict) else None
|
|
if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts):
|
|
raise RuntimeError("sentinel-probe payload requires non-empty accounts")
|
|
job_name, manifest = sentinel_probe_job_manifest(accounts)
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}")
|
|
timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240)
|
|
status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds)
|
|
logs = job_logs(job_name)
|
|
parsed = parse_embedded_json(logs.get("stdout") or "")
|
|
results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else []
|
|
requested = set(accounts)
|
|
measured = {item.get("accountName") for item in results if isinstance(item, dict)}
|
|
missing = sorted(name for name in requested if name not in measured)
|
|
marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested)
|
|
job_ok = status == "succeeded" and isinstance(parsed, dict) and parsed.get("ok") is True
|
|
return {
|
|
"ok": job_ok and marker_ok,
|
|
"jobExecutionOk": job_ok,
|
|
"markerOk": marker_ok,
|
|
"mode": "sentinel-probe",
|
|
"namespace": NAMESPACE,
|
|
"requestedAccounts": accounts,
|
|
"missingAccounts": missing,
|
|
"job": {
|
|
"name": job_name,
|
|
"status": status,
|
|
"condition": condition,
|
|
"timeoutSeconds": timeout_seconds,
|
|
"applyStdoutTail": text(proc.stdout, 1200),
|
|
"logsExitCode": logs.get("exitCode"),
|
|
"logsStderrTail": logs.get("stderr"),
|
|
},
|
|
"probe": parsed,
|
|
"sentinelState": sentinel_state_summary(),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def run_cleanup_probes():
|
|
admin_email, token, admin_compliance = login()
|
|
cleanup = cleanup_probe_resources(token)
|
|
return {
|
|
"ok": cleanup.get("ok") is True,
|
|
"mode": "cleanup-probes",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"cleanup": cleanup,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
try:
|
|
if MODE == "sync":
|
|
result = run_sync()
|
|
elif MODE == "trace":
|
|
result = run_trace()
|
|
elif MODE == "cleanup-probes":
|
|
result = run_cleanup_probes()
|
|
elif MODE == "sentinel-probe":
|
|
result = run_sentinel_probe()
|
|
else:
|
|
result = run_validate()
|
|
except Exception as exc:
|
|
result = {
|
|
"ok": False,
|
|
"mode": MODE,
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": globals().get("APP_POD"),
|
|
"error": str(exc),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
sys.exit(0 if result.get("ok") else 1)
|
|
PY
|
|
`;
|
|
}
|
|
|
|
async function capture(config: UniDeskConfig, target: string, args: string[], input?: string): Promise<SshCaptureResult> {
|
|
return await runSshCommandCapture(config, target, args, input);
|
|
}
|
|
|
|
type RemoteCodexPoolMode = "sync" | "validate" | "sentinel-probe" | "sentinel-image-status" | "sentinel-image-build";
|
|
|
|
async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: RemoteCodexPoolMode, script: string, target = codexPoolRuntimeTarget()): Promise<SshCaptureResult> {
|
|
const jobName = `codex-pool-${mode}-${Date.now().toString(36)}`.slice(0, 63);
|
|
const startedAtMs = Date.now();
|
|
const start = await capture(config, target.route, ["sh"], remoteJobStartScript(jobName, script));
|
|
const started = parseJsonOutput(start.stdout);
|
|
if (start.exitCode !== 0 || boolField(started, "ok", false) !== true) return start;
|
|
|
|
let latest: RemoteCodexPoolJobStatus | null = null;
|
|
while (Date.now() - startedAtMs <= remoteJobTimeoutMs) {
|
|
await sleep(remoteJobPollMs);
|
|
const probe = await capture(config, target.route, ["sh"], remoteJobStatusScript(jobName));
|
|
const parsed = parseJsonOutput(probe.stdout);
|
|
latest = normalizeRemoteJobStatus(parsed);
|
|
process.stderr.write(`${JSON.stringify({
|
|
event: "platform-infra.sub2api.codex-pool.progress",
|
|
at: new Date().toISOString(),
|
|
mode,
|
|
jobName,
|
|
status: latest?.status ?? "unknown",
|
|
exitCode: latest?.exitCode ?? null,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
stdoutBytes: latest?.stdoutBytes ?? null,
|
|
stderrBytes: latest?.stderrBytes ?? null,
|
|
})}\n`);
|
|
if (probe.exitCode !== 0) {
|
|
return {
|
|
exitCode: probe.exitCode,
|
|
stdout: probe.stdout,
|
|
stderr: `${start.stderr}${probe.stderr}`,
|
|
};
|
|
}
|
|
if (latest?.status === "succeeded" || latest?.status === "failed") {
|
|
return {
|
|
exitCode: latest.exitCode ?? (latest.status === "succeeded" ? 0 : 1),
|
|
stdout: latest.stdout ?? "",
|
|
stderr: latest.stderr ?? "",
|
|
};
|
|
}
|
|
}
|
|
|
|
return {
|
|
exitCode: 124,
|
|
stdout: latest?.stdout ?? "",
|
|
stderr: [
|
|
latest?.stderr ?? "",
|
|
`remote codex-pool ${mode} job ${jobName} did not finish within ${remoteJobTimeoutMs}ms`,
|
|
`status command: ${codexPoolModeCommand(mode)}`,
|
|
].filter(Boolean).join("\n"),
|
|
};
|
|
}
|
|
|
|
function codexPoolModeCommand(mode: RemoteCodexPoolMode): string {
|
|
if (mode === "sync") return "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm";
|
|
if (mode === "sentinel-probe") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account <accountName> --confirm";
|
|
return "bun scripts/cli.ts platform-infra sub2api codex-pool validate";
|
|
}
|
|
|
|
interface RemoteCodexPoolJobStatus {
|
|
status: "running" | "succeeded" | "failed" | "unknown";
|
|
exitCode: number | null;
|
|
stdout: string | null;
|
|
stderr: string | null;
|
|
stdoutBytes: number | null;
|
|
stderrBytes: number | null;
|
|
}
|
|
|
|
function normalizeRemoteJobStatus(parsed: Record<string, unknown> | null): RemoteCodexPoolJobStatus | null {
|
|
if (parsed === null) return null;
|
|
const status = parsed.status === "running" || parsed.status === "succeeded" || parsed.status === "failed" || parsed.status === "unknown"
|
|
? parsed.status
|
|
: "unknown";
|
|
return {
|
|
status,
|
|
exitCode: typeof parsed.exitCode === "number" ? parsed.exitCode : null,
|
|
stdout: typeof parsed.stdout === "string" ? parsed.stdout : null,
|
|
stderr: typeof parsed.stderr === "string" ? parsed.stderr : null,
|
|
stdoutBytes: typeof parsed.stdoutBytes === "number" ? parsed.stdoutBytes : null,
|
|
stderrBytes: typeof parsed.stderrBytes === "number" ? parsed.stderrBytes : null,
|
|
};
|
|
}
|
|
|
|
function remoteJobStartScript(jobName: string, script: string): string {
|
|
const scriptB64 = Buffer.from(script, "utf8").toString("base64");
|
|
return `
|
|
set -eu
|
|
job=${shQuote(jobName)}
|
|
dir=${shQuote(remoteJobDir)}
|
|
mkdir -p "$dir"
|
|
chmod 700 "$dir"
|
|
script_path="$dir/$job.sh"
|
|
stdout_path="$dir/$job.stdout"
|
|
stderr_path="$dir/$job.stderr"
|
|
exit_path="$dir/$job.exit"
|
|
done_path="$dir/$job.done"
|
|
pid_path="$dir/$job.pid"
|
|
rm -f "$script_path" "$stdout_path" "$stderr_path" "$exit_path" "$done_path" "$pid_path"
|
|
base64 -d > "$script_path" <<'UNIDESK_REMOTE_SCRIPT_B64'
|
|
${scriptB64}
|
|
UNIDESK_REMOTE_SCRIPT_B64
|
|
chmod 700 "$script_path"
|
|
(
|
|
set +e
|
|
trap '' HUP
|
|
sh "$script_path" > "$stdout_path" 2> "$stderr_path"
|
|
code=$?
|
|
printf '%s' "$code" > "$exit_path"
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$done_path"
|
|
rm -f "$script_path"
|
|
) >/dev/null 2>&1 &
|
|
pid=$!
|
|
printf '%s' "$pid" > "$pid_path"
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({"ok": True, "jobName": ${JSON.stringify(jobName)}, "pid": int("${"${pid}"}"), "dir": ${JSON.stringify(remoteJobDir)}}))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function remoteJobStatusScript(jobName: string): string {
|
|
return `
|
|
set -eu
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import signal
|
|
|
|
job = ${JSON.stringify(jobName)}
|
|
directory = ${JSON.stringify(remoteJobDir)}
|
|
stdout_path = os.path.join(directory, job + ".stdout")
|
|
stderr_path = os.path.join(directory, job + ".stderr")
|
|
exit_path = os.path.join(directory, job + ".exit")
|
|
done_path = os.path.join(directory, job + ".done")
|
|
pid_path = os.path.join(directory, job + ".pid")
|
|
|
|
def read_text(path, limit=None):
|
|
try:
|
|
with open(path, "rb") as handle:
|
|
data = handle.read()
|
|
except FileNotFoundError:
|
|
return None
|
|
if limit is not None and len(data) > limit:
|
|
data = data[-limit:]
|
|
return data.decode("utf-8", errors="replace")
|
|
|
|
def file_size(path):
|
|
try:
|
|
return os.path.getsize(path)
|
|
except FileNotFoundError:
|
|
return 0
|
|
|
|
pid_text = read_text(pid_path)
|
|
done = os.path.exists(done_path)
|
|
exit_text = read_text(exit_path)
|
|
exit_code = None
|
|
if exit_text is not None:
|
|
try:
|
|
exit_code = int(exit_text.strip())
|
|
except ValueError:
|
|
exit_code = None
|
|
running = False
|
|
if pid_text is not None and not done:
|
|
try:
|
|
os.kill(int(pid_text.strip()), 0)
|
|
running = True
|
|
except Exception:
|
|
running = False
|
|
status = "succeeded" if done and exit_code == 0 else "failed" if done else "running" if running else "unknown"
|
|
print(json.dumps({
|
|
"ok": True,
|
|
"jobName": job,
|
|
"status": status,
|
|
"pid": int(pid_text.strip()) if pid_text and pid_text.strip().isdigit() else None,
|
|
"exitCode": exit_code,
|
|
"stdoutBytes": file_size(stdout_path),
|
|
"stderrBytes": file_size(stderr_path),
|
|
"stdout": read_text(stdout_path, 500000) if done else read_text(stdout_path, 12000),
|
|
"stderr": read_text(stderr_path, 120000) if done else read_text(stderr_path, 12000),
|
|
"doneAt": read_text(done_path),
|
|
}, ensure_ascii=False))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function shQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'\\''`)}'`;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function parseJsonOutput(stdout: string): Record<string, unknown> | null {
|
|
const trimmed = stdout.trim();
|
|
if (trimmed.length === 0) return null;
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start === -1 || end === -1 || end <= start) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function boolField(value: Record<string, unknown> | null, key: string, defaultValue: boolean): boolean {
|
|
if (value === null) return defaultValue;
|
|
const field = value[key];
|
|
return typeof field === "boolean" ? field : defaultValue;
|
|
}
|
|
|
|
function compactCapture(result: SshCaptureResult, options: { full?: boolean } = {}): Record<string, unknown> {
|
|
const full = options.full ?? false;
|
|
return {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
stdoutTail: full || result.exitCode !== 0 ? result.stdout.slice(-8000) : "",
|
|
stderrTail: full || result.exitCode !== 0 ? result.stderr.slice(-4000) : "",
|
|
};
|
|
}
|