Codex
5452 lines
226 KiB
TypeScript
5452 lines
226 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
import { homedir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { UniDeskConfig } from "./config";
|
|
import { rootPath } from "./config";
|
|
import type { RenderedCliResult } from "./output";
|
|
import {
|
|
codexPoolSentinelSummary,
|
|
codexPoolSentinelRuntimeImage,
|
|
defaultCodexPoolSentinelConfig,
|
|
readCodexPoolSentinelConfig,
|
|
renderCodexPoolSentinelManifest,
|
|
type CodexPoolSentinelConfig,
|
|
type CodexPoolSentinelProfileSecret,
|
|
} from "./platform-infra-sub2api-codex-sentinel";
|
|
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
|
|
|
const g14K3sRoute = "G14:k3s";
|
|
const namespace = "platform-infra";
|
|
const serviceName = "sub2api";
|
|
const serviceDns = `${serviceName}.${namespace}.svc.cluster.local:8080`;
|
|
const fieldManager = "unidesk-platform-infra";
|
|
const appSecretName = "sub2api-secrets";
|
|
const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml");
|
|
const sentinelImageDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile");
|
|
const defaultPoolGroupName = "unidesk-codex-pool";
|
|
const defaultPoolApiKeyName = "unidesk-codex-pool-api-key";
|
|
const defaultPoolApiKeySecretName = "sub2api-codex-pool-api-key";
|
|
const defaultPoolApiKeySecretKey = "API_KEY";
|
|
const defaultMinOwnerBalanceUsd = 1000;
|
|
const defaultAccountPriority = 10;
|
|
const remoteJobDir = "/tmp/unidesk-platform-infra-sub2api-codex-pool";
|
|
const remoteJobTimeoutMs = 15 * 60_000;
|
|
const remoteJobPollMs = 5_000;
|
|
|
|
interface DisclosureOptions {
|
|
full: boolean;
|
|
raw: boolean;
|
|
}
|
|
|
|
interface SyncOptions extends DisclosureOptions {
|
|
confirm: boolean;
|
|
pruneRemoved: boolean;
|
|
}
|
|
|
|
interface ConfirmOptions extends DisclosureOptions {
|
|
confirm: boolean;
|
|
}
|
|
|
|
interface SentinelProbeOptions extends ConfirmOptions {
|
|
accounts: string[];
|
|
}
|
|
|
|
interface SentinelReportOptions extends DisclosureOptions {
|
|
events: number;
|
|
}
|
|
|
|
interface SentinelImageOptions extends DisclosureOptions {
|
|
action: "status" | "build";
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
}
|
|
|
|
interface CodexProfile {
|
|
profile: string;
|
|
accountName: string;
|
|
configFile: string;
|
|
authFile: string;
|
|
provider: string;
|
|
baseUrl: string;
|
|
wireApi: string | null;
|
|
model: string | null;
|
|
envKey: string | null;
|
|
apiKey: string | null;
|
|
apiKeySource: "auth-json" | "env" | null;
|
|
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
|
|
upstreamUserAgent: string | null;
|
|
priority: number;
|
|
capacity: number;
|
|
loadFactor: number;
|
|
tempUnschedulable: CodexTempUnschedulablePolicy;
|
|
authOpenAIKeyShape: string;
|
|
ok: boolean;
|
|
error: string | null;
|
|
}
|
|
|
|
type OpenAIResponsesWebSocketsV2Mode = "off" | "ctx_pool" | "passthrough";
|
|
|
|
export interface CodexTempUnschedulableRule {
|
|
statusCode: number;
|
|
keywords: string[];
|
|
durationMinutes: number;
|
|
description: string | null;
|
|
}
|
|
|
|
export interface CodexTempUnschedulablePolicy {
|
|
enabled: boolean;
|
|
rules: CodexTempUnschedulableRule[];
|
|
}
|
|
|
|
interface CodexPoolConfig {
|
|
groupName: string;
|
|
apiKeyName: string;
|
|
apiKeySecretName: string;
|
|
apiKeySecretKey: string;
|
|
minOwnerBalanceUsd: number;
|
|
minOwnerConcurrency: number;
|
|
minOwnerConcurrencySource: "auto" | "yaml";
|
|
defaultAccountPriority: number;
|
|
defaultAccountCapacity: number;
|
|
defaultAccountLoadFactor: number;
|
|
defaultTempUnschedulable: CodexTempUnschedulablePolicy;
|
|
profiles: CodexPoolProfileConfig[];
|
|
publicExposure: CodexPoolPublicExposureConfig;
|
|
localCodex: CodexPoolLocalCodexConfig;
|
|
sentinel: CodexPoolSentinelConfig;
|
|
}
|
|
|
|
interface CodexPoolProfileConfig {
|
|
profile: string;
|
|
accountName: string | null;
|
|
configFile: string;
|
|
authFile: string;
|
|
fallbackConfigFile: string | null;
|
|
fallbackAuthFile: string | null;
|
|
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
|
|
upstreamUserAgent: string | null;
|
|
priority: number;
|
|
capacity: number | null;
|
|
loadFactor: number | null;
|
|
tempUnschedulable: CodexTempUnschedulablePolicy;
|
|
}
|
|
|
|
interface CodexPoolPublicExposureConfig {
|
|
enabled: boolean;
|
|
proxyName: string;
|
|
configMapName: string;
|
|
deploymentName: string;
|
|
frpcImage: string;
|
|
serverAddr: string;
|
|
serverPort: number;
|
|
remotePort: number;
|
|
localIP: string;
|
|
localPort: number;
|
|
publicBaseUrl: string;
|
|
masterBaseUrl: string;
|
|
masterFrps: {
|
|
configPath: string;
|
|
containerName: string;
|
|
};
|
|
masterCaddy: {
|
|
enabled: boolean;
|
|
domain: string;
|
|
configPath: string;
|
|
serviceName: string;
|
|
upstreamBaseUrl: string;
|
|
responseHeaderTimeoutSeconds: number;
|
|
};
|
|
}
|
|
|
|
interface CodexPoolLocalCodexConfig {
|
|
backupSuffix: string;
|
|
providerName: string;
|
|
wireApi: string;
|
|
supportsWebSockets: boolean;
|
|
responsesWebSocketsV2: boolean;
|
|
responsesSmokeModel: string;
|
|
}
|
|
|
|
interface CodexLocalConsumerTomlOptions {
|
|
providerName: string;
|
|
baseUrl: string;
|
|
wireApi: string;
|
|
supportsWebSockets: boolean;
|
|
responsesWebSocketsV2: boolean;
|
|
}
|
|
|
|
export function codexPoolHelp(): unknown {
|
|
const pool = readCodexPoolConfig();
|
|
return {
|
|
command: "platform-infra sub2api codex-pool plan|sync|validate|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
|
|
output: "json, except sentinel-report defaults to a ps-like text table",
|
|
usage: [
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm [--prune-removed]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool validate [--full|--raw]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report [--events 20|--full|--raw]",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm",
|
|
"bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
|
|
],
|
|
description: "Import YAML-selected ~/.codex API-key profiles into one Sub2API OpenAI pool, expose one unified API_KEY, and optionally configure this master server's ~/.codex consumer endpoint.",
|
|
target: {
|
|
route: g14K3sRoute,
|
|
namespace,
|
|
serviceDns,
|
|
configPath: codexPoolConfigPath,
|
|
poolGroupName: pool.groupName,
|
|
poolApiKeySecretName: pool.apiKeySecretName,
|
|
poolApiKeySecretKey: pool.apiKeySecretKey,
|
|
publicBaseUrl: pool.publicExposure.enabled ? pool.publicExposure.publicBaseUrl : null,
|
|
masterBaseUrl: pool.publicExposure.enabled ? pool.publicExposure.masterBaseUrl : null,
|
|
sentinelMonitorEnabled: pool.sentinel.monitor.enabled,
|
|
sentinelActionsEnabled: pool.sentinel.actions.enabled,
|
|
secretValuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const [action = "plan"] = args;
|
|
if (action === "plan") return codexPoolPlan(parseDisclosureOptions(args.slice(1)));
|
|
if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1)));
|
|
if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1)));
|
|
if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1)));
|
|
if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1)));
|
|
if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1)));
|
|
if (action === "cleanup-probes") return await codexPoolCleanupProbes(config, parseConfirmOptions(args.slice(1)));
|
|
if (action === "expose") return await codexPoolExpose(config, parseConfirmOptions(args.slice(1)));
|
|
if (action === "configure-local") return await codexPoolConfigureLocal(config, parseConfirmOptions(args.slice(1)));
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-platform-infra-sub2api-codex-pool-command",
|
|
args,
|
|
help: codexPoolHelp(),
|
|
};
|
|
}
|
|
|
|
function parseSyncOptions(args: string[]): SyncOptions {
|
|
validateOptions(args, new Set(["--confirm", "--prune-removed", "--full", "--raw"]));
|
|
const disclosure = parseDisclosureOptions(args.filter((arg) => arg !== "--confirm" && arg !== "--prune-removed"));
|
|
return { ...disclosure, confirm: args.includes("--confirm"), pruneRemoved: args.includes("--prune-removed") };
|
|
}
|
|
|
|
function parseConfirmOptions(args: string[]): ConfirmOptions {
|
|
validateOptions(args, new Set(["--confirm", "--full", "--raw"]));
|
|
const disclosure = parseDisclosureOptions(args.filter((arg) => arg !== "--confirm"));
|
|
return { ...disclosure, confirm: args.includes("--confirm") };
|
|
}
|
|
|
|
function parseSentinelImageOptions(args: string[]): SentinelImageOptions {
|
|
const [actionRaw = "status", ...rest] = args;
|
|
if (actionRaw !== "status" && actionRaw !== "build") throw new Error("sentinel-image usage: status|build [--dry-run|--confirm] [--full|--raw]");
|
|
let confirm = false;
|
|
let explicitDryRun = false;
|
|
const disclosureArgs: string[] = [];
|
|
for (const arg of rest) {
|
|
if (arg === "--confirm") {
|
|
confirm = true;
|
|
continue;
|
|
}
|
|
if (arg === "--dry-run") {
|
|
explicitDryRun = true;
|
|
continue;
|
|
}
|
|
if (arg === "--full" || arg === "--raw") {
|
|
disclosureArgs.push(arg);
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
if (confirm && explicitDryRun) throw new Error("sentinel-image accepts only one of --confirm or --dry-run");
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
return {
|
|
...disclosure,
|
|
action: actionRaw,
|
|
confirm,
|
|
dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
|
|
};
|
|
}
|
|
|
|
function parseSentinelProbeOptions(args: string[]): SentinelProbeOptions {
|
|
const accounts: string[] = [];
|
|
const disclosureArgs: string[] = [];
|
|
let confirm = false;
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--confirm") {
|
|
confirm = true;
|
|
continue;
|
|
}
|
|
if (arg === "--full" || arg === "--raw") {
|
|
disclosureArgs.push(arg);
|
|
continue;
|
|
}
|
|
if (arg === "--account") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--account requires an account name");
|
|
accounts.push(...splitAccountNames(value));
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--account=")) {
|
|
accounts.push(...splitAccountNames(arg.slice("--account=".length)));
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
const uniqueAccounts = [...new Set(accounts)];
|
|
if (uniqueAccounts.length === 0) throw new Error("sentinel-probe requires --account <accountName>");
|
|
for (const account of uniqueAccounts) validateKubernetesName(account, "--account", false);
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
return { ...disclosure, confirm, accounts: uniqueAccounts };
|
|
}
|
|
|
|
function parseSentinelReportOptions(args: string[]): SentinelReportOptions {
|
|
let events = 20;
|
|
let explicitEvents = false;
|
|
const disclosureArgs: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index]!;
|
|
if (arg === "--full" || arg === "--raw") {
|
|
disclosureArgs.push(arg);
|
|
continue;
|
|
}
|
|
if (arg === "--events") {
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--")) throw new Error("--events requires a positive integer");
|
|
events = readReportEventLimit(value, "--events");
|
|
explicitEvents = true;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--events=")) {
|
|
events = readReportEventLimit(arg.slice("--events=".length), "--events");
|
|
explicitEvents = true;
|
|
continue;
|
|
}
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
const disclosure = parseDisclosureOptions(disclosureArgs);
|
|
if (disclosure.full && !explicitEvents) events = 80;
|
|
return { ...disclosure, events };
|
|
}
|
|
|
|
function readReportEventLimit(raw: string, option: string): number {
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 1 || value > 200) throw new Error(`${option} must be an integer from 1 to 200`);
|
|
return value;
|
|
}
|
|
|
|
function parseDisclosureOptions(args: string[]): DisclosureOptions {
|
|
validateOptions(args, new Set(["--full", "--raw"]));
|
|
const raw = args.includes("--raw");
|
|
return { full: raw || args.includes("--full"), raw };
|
|
}
|
|
|
|
function splitAccountNames(value: string): string[] {
|
|
return value.split(",").map((item) => item.trim()).filter(Boolean);
|
|
}
|
|
|
|
function validateOptions(args: string[], booleanOptions: Set<string>): void {
|
|
for (const arg of args) {
|
|
if (booleanOptions.has(arg)) continue;
|
|
throw new Error(`unsupported option: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false }): Record<string, unknown> {
|
|
const pool = readCodexPoolConfig();
|
|
const profiles = collectCodexProfiles();
|
|
const ok = profiles.length > 0 && profiles.every((profile) => profile.ok);
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-plan",
|
|
source: {
|
|
directory: join(homedir(), ".codex"),
|
|
configPattern: "YAML-selected config files under ~/.codex",
|
|
authPattern: "YAML-selected auth files under ~/.codex",
|
|
valuesPrinted: false,
|
|
},
|
|
target: poolTarget(),
|
|
config: {
|
|
path: codexPoolConfigPath,
|
|
pool: options.full ? pool : codexPoolConfigSummary(pool),
|
|
},
|
|
profiles: options.full ? profiles.map(redactProfile) : profiles.map(compactProfile),
|
|
decision: {
|
|
accountType: "openai/apikey",
|
|
grouping: `All discovered Codex profiles are bound to one Sub2API group named ${pool.groupName}.`,
|
|
unifiedApiKey: `The client-facing API_KEY is controlled by k3s Secret ${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}.`,
|
|
sentinel: pool.sentinel.monitor.enabled
|
|
? `Account sentinel is enabled as k8s CronJob ${namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.`
|
|
: "Account sentinel monitoring is disabled by YAML.",
|
|
publicExposure: pool.publicExposure.enabled
|
|
? `Default Codex consumers use ${codexConsumerBaseUrl(pool)}; bounded master-local probes may use ${pool.publicExposure.masterBaseUrl}. FRP proxy ${pool.publicExposure.proxyName} maps public ${pool.publicExposure.publicBaseUrl} to ${pool.publicExposure.localIP}:${pool.publicExposure.localPort}.`
|
|
: "Public FRP exposure is disabled by YAML.",
|
|
idempotency: "sync reuses the group, account names, and k3s Secret when they already exist; credentials are updated from the current local Codex files; managed accounts missing from YAML are preserved unless --prune-removed is explicitly provided.",
|
|
configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.",
|
|
},
|
|
next: ok
|
|
? { sync: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" }
|
|
: { fix: "Ensure every discovered config.toml profile has a base_url and either auth.json OPENAI_API_KEY or the configured env_key present in this shell." },
|
|
};
|
|
}
|
|
|
|
async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const profiles = collectCodexProfiles();
|
|
const planOk = profiles.length > 0 && profiles.every((profile) => profile.ok);
|
|
if (!options.confirm || !planOk) {
|
|
return {
|
|
...codexPoolPlan(options),
|
|
ok: !options.confirm ? planOk : false,
|
|
mode: options.confirm ? "blocked-invalid-local-profile" : "dry-run",
|
|
next: options.confirm
|
|
? { fix: "Repair invalid local Codex profiles, then rerun sync --confirm." }
|
|
: { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" },
|
|
};
|
|
}
|
|
|
|
const sentinelImage = pool.sentinel.monitor.enabled
|
|
? await runCodexPoolSentinelImage(config, pool, { action: "build", confirm: true, dryRun: false, full: options.full, raw: false })
|
|
: { ok: true, mode: "skipped-monitor-disabled" };
|
|
if (sentinelImage.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-sync",
|
|
mode: "blocked-sentinel-image",
|
|
sentinelImage,
|
|
next: {
|
|
image: "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm",
|
|
},
|
|
};
|
|
}
|
|
|
|
const payload = {
|
|
pruneRemoved: options.pruneRemoved,
|
|
sentinel: {
|
|
manifest: renderCodexPoolSentinelManifest(pool.sentinel, sentinelProfileSecrets(profiles), {
|
|
namespace,
|
|
serviceName,
|
|
serviceDns,
|
|
appSecretName,
|
|
}),
|
|
summary: codexPoolSentinelSummary(pool.sentinel),
|
|
},
|
|
pool: {
|
|
groupName: pool.groupName,
|
|
apiKeyName: pool.apiKeyName,
|
|
apiKeySecretName: pool.apiKeySecretName,
|
|
apiKeySecretKey: pool.apiKeySecretKey,
|
|
minOwnerBalanceUsd: pool.minOwnerBalanceUsd,
|
|
minOwnerConcurrency: pool.minOwnerConcurrency,
|
|
defaultAccountPriority: pool.defaultAccountPriority,
|
|
defaultAccountCapacity: pool.defaultAccountCapacity,
|
|
defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
|
|
},
|
|
profiles: profiles.map((profile) => ({
|
|
profile: profile.profile,
|
|
accountName: profile.accountName,
|
|
configFile: profile.configFile,
|
|
authFile: profile.authFile,
|
|
provider: profile.provider,
|
|
baseUrl: profile.baseUrl,
|
|
wireApi: profile.wireApi,
|
|
model: profile.model,
|
|
apiKey: profile.apiKey,
|
|
apiKeySource: profile.apiKeySource,
|
|
apiKeyFingerprint: fingerprint(profile.apiKey ?? ""),
|
|
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
priority: profile.priority,
|
|
capacity: profile.capacity,
|
|
loadFactor: profile.loadFactor,
|
|
tempUnschedulable: profile.tempUnschedulable,
|
|
tempUnschedulableCredentials: renderSub2ApiTempUnschedulableCredentials(profile.tempUnschedulable),
|
|
})),
|
|
};
|
|
const result = await runRemoteCodexPoolScript(config, "sync", syncScript(payload, pool));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sync",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sync",
|
|
local: {
|
|
profileCount: profiles.length,
|
|
pruneRemoved: options.pruneRemoved,
|
|
invalidProfiles: profiles.filter((profile) => !profile.ok).map(compactProfile),
|
|
profiles: options.full ? profiles.map(redactProfile) : undefined,
|
|
valuesPrinted: false,
|
|
},
|
|
sentinelImage,
|
|
remote: parsed === null
|
|
? compactCapture(result, { full: options.full || result.exitCode !== 0 })
|
|
: options.full ? parsed : codexPoolSyncSummary(parsed),
|
|
next: {
|
|
validate: "bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function codexPoolSentinelImage(config: UniDeskConfig, options: SentinelImageOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
return await runCodexPoolSentinelImage(config, pool, options);
|
|
}
|
|
|
|
async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolConfig, options: SentinelImageOptions): Promise<Record<string, unknown>> {
|
|
const target = codexPoolSentinelRuntimeImage(pool.sentinel);
|
|
if (options.action === "build" && options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-image",
|
|
mode: "dry-run",
|
|
image: target,
|
|
dockerfile: sentinelImageDockerfilePath,
|
|
mutation: false,
|
|
next: {
|
|
confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm",
|
|
},
|
|
};
|
|
}
|
|
const mode: RemoteCodexPoolMode = options.action === "status" ? "sentinel-image-status" : "sentinel-image-build";
|
|
const script = options.action === "status" ? sentinelImageStatusScript(pool) : sentinelImageBuildScript(pool);
|
|
const result = await runRemoteCodexPoolScript(config, mode, script);
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-image",
|
|
mode: options.action,
|
|
image: target,
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-image",
|
|
mode: options.action,
|
|
image: target,
|
|
summary: parsed,
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolValidate(config: UniDeskConfig, options: DisclosureOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const result = await runRemoteCodexPoolScript(config, "validate", validateScript(pool));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-validate",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-validate",
|
|
summary: options.full ? parsed : codexPoolValidationSummary(parsed),
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolSentinelReport(config: UniDeskConfig, options: SentinelReportOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
const pool = readCodexPoolConfig();
|
|
const result = await capture(config, g14K3sRoute, ["script"], sentinelReportScript(pool, options.events));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
|
|
if (options.raw) {
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-report",
|
|
remote: compactCapture(result, { full: true }),
|
|
report: parsed,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const text = renderSentinelReport(parsed, {
|
|
events: options.events,
|
|
full: options.full,
|
|
remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }),
|
|
});
|
|
return renderedCliResult(ok, "platform-infra sub2api codex-pool sentinel-report", text);
|
|
}
|
|
|
|
async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelProbeOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const configuredAccounts = desiredAccountNames(pool);
|
|
const missing = options.accounts.filter((account) => !configuredAccounts.includes(account));
|
|
if (missing.length > 0) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
error: "account-not-in-yaml",
|
|
missing,
|
|
configuredAccounts,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
mode: "dry-run",
|
|
target: poolTarget(pool),
|
|
accounts: options.accounts,
|
|
effect: "Would create one Kubernetes Job from the managed sentinel CronJob and force an immediate marker probe for the requested account(s).",
|
|
next: {
|
|
confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account ${options.accounts.join(",")} --confirm`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const payload = {
|
|
accounts: options.accounts,
|
|
};
|
|
const result = await runRemoteCodexPoolScript(config, "sentinel-probe", sentinelProbeScript(payload, pool));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-sentinel-probe",
|
|
summary: options.full ? parsed : compactSentinelProbeResult(parsed),
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolCleanupProbes(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-cleanup-probes",
|
|
mode: "dry-run",
|
|
target: poolTarget(),
|
|
scope: "Only deletes temporary resources whose names start with unidesk-probe-.",
|
|
next: { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm" },
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const pool = readCodexPoolConfig();
|
|
const result = await capture(config, g14K3sRoute, ["script"], cleanupProbesScript(pool));
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
if (options.raw) {
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-cleanup-probes",
|
|
remote: compactCapture(result, { full: true }),
|
|
parsed,
|
|
};
|
|
}
|
|
return {
|
|
ok: result.exitCode === 0 && boolField(parsed, "ok", false),
|
|
action: "platform-infra-sub2api-codex-pool-cleanup-probes",
|
|
summary: parsed,
|
|
remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
|
};
|
|
}
|
|
|
|
async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
if (!pool.publicExposure.enabled) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
mode: "disabled-by-yaml",
|
|
target: poolTarget(pool),
|
|
};
|
|
}
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
mode: "dry-run",
|
|
target: poolTarget(pool),
|
|
publicExposure: publicExposureSummary(pool),
|
|
next: {
|
|
confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm",
|
|
},
|
|
};
|
|
}
|
|
const masterResult = await applyMasterFrpsAllowPort(pool);
|
|
const caddyResult = await applyMasterCaddySite(pool);
|
|
const remoteResult = await capture(config, g14K3sRoute, ["script"], exposeScript(pool));
|
|
const parsed = parseJsonOutput(remoteResult.stdout);
|
|
const publicProbe = await probePublicModels(pool, "without-api-key", undefined, "public");
|
|
const ok = masterResult.ok && caddyResult.ok && remoteResult.exitCode === 0 && boolField(parsed, "ok", false) && publicProbe.httpStatus === 401;
|
|
if (options.raw) {
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
masterFrps: masterResult,
|
|
masterCaddy: caddyResult,
|
|
remote: compactCapture(remoteResult, { full: true }),
|
|
parsed,
|
|
publicProbe,
|
|
};
|
|
}
|
|
return {
|
|
ok,
|
|
action: "platform-infra-sub2api-codex-pool-expose",
|
|
mode: "confirmed",
|
|
publicExposure: publicExposureSummary(pool),
|
|
masterFrps: masterResult,
|
|
masterCaddy: caddyResult,
|
|
remote: parsed ?? compactCapture(remoteResult, { full: options.full || remoteResult.exitCode !== 0 }),
|
|
publicProbe,
|
|
next: {
|
|
configureLocal: "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
|
|
validate: "bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
|
|
const pool = readCodexPoolConfig();
|
|
const codexDir = join(homedir(), ".codex");
|
|
const configPath = join(codexDir, "config.toml");
|
|
const authPath = join(codexDir, "auth.json");
|
|
const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`);
|
|
const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`);
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: true,
|
|
action: "platform-infra-sub2api-codex-pool-configure-local",
|
|
mode: "dry-run",
|
|
target: {
|
|
codexDir,
|
|
configPath,
|
|
authPath,
|
|
backupConfigPath,
|
|
backupAuthPath,
|
|
baseUrl: codexConsumerBaseUrl(pool),
|
|
providerName: pool.localCodex.providerName,
|
|
wireApi: pool.localCodex.wireApi,
|
|
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
|
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
|
responsesSmokeModel: pool.localCodex.responsesSmokeModel,
|
|
valuesPrinted: false,
|
|
},
|
|
next: {
|
|
confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
|
|
},
|
|
};
|
|
}
|
|
if (!pool.publicExposure.enabled) throw new Error(`${codexPoolConfigPath}.publicExposure.enabled must be true before configure-local`);
|
|
const keyResult = await fetchPoolApiKey(config, pool);
|
|
if (keyResult.apiKey === null) {
|
|
return {
|
|
ok: false,
|
|
action: "platform-infra-sub2api-codex-pool-configure-local",
|
|
error: keyResult.error ?? "pool API key missing",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const writeResult = writeLocalCodexConfig(pool, keyResult.apiKey);
|
|
const validateResult = await validatePublicGatewayWithKey(pool, keyResult.apiKey);
|
|
return {
|
|
ok: writeResult.ok && validateResult.ok,
|
|
action: "platform-infra-sub2api-codex-pool-configure-local",
|
|
mode: "confirmed",
|
|
local: writeResult,
|
|
validation: validateResult,
|
|
apiKey: {
|
|
secret: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`,
|
|
keyPreview: apiKeyPreview(keyResult.apiKey),
|
|
apiKeyFingerprint: fingerprint(keyResult.apiKey),
|
|
valuesPrinted: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function collectCodexProfiles(): CodexProfile[] {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const pool = readCodexPoolConfig();
|
|
if (!existsSync(codexDir)) return [];
|
|
const seenAccountNames = new Set<string>();
|
|
const configs = pool.profiles.length > 0
|
|
? pool.profiles
|
|
: discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
|
|
return configs.map((entry) => {
|
|
const resolved = resolveProfileFiles(codexDir, entry);
|
|
const profile = entry.profile;
|
|
const accountName = entry.accountName ?? uniqueAccountName(profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
const configFile = resolved.configFile;
|
|
const authFile = resolved.authFile;
|
|
const configPath = join(codexDir, configFile);
|
|
const authPath = join(codexDir, authFile);
|
|
const base: CodexProfile = {
|
|
profile,
|
|
accountName,
|
|
configFile,
|
|
authFile,
|
|
provider: "",
|
|
baseUrl: "",
|
|
wireApi: null,
|
|
model: null,
|
|
envKey: null,
|
|
apiKey: null,
|
|
apiKeySource: null,
|
|
openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent: entry.upstreamUserAgent,
|
|
priority: entry.priority,
|
|
capacity: entry.capacity ?? pool.defaultAccountCapacity,
|
|
loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor,
|
|
tempUnschedulable: entry.tempUnschedulable,
|
|
authOpenAIKeyShape: existsSync(authPath) ? "unknown" : "missing",
|
|
ok: false,
|
|
error: null,
|
|
};
|
|
|
|
try {
|
|
if (!existsSync(configPath)) throw new Error(`config file ${configFile} is missing`);
|
|
const parsed = Bun.TOML.parse(readFileSync(configPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) throw new Error("config is not a TOML object");
|
|
const providers = parsed.model_providers;
|
|
if (!isRecord(providers)) throw new Error("model_providers is missing");
|
|
const provider = stringValue(parsed.model_provider) ?? Object.keys(providers)[0] ?? "";
|
|
if (provider === "") throw new Error("model_provider is missing");
|
|
const providerConfig = providers[provider];
|
|
if (!isRecord(providerConfig)) throw new Error(`model provider ${provider} is missing`);
|
|
const baseUrl = normalizeBaseUrl(stringValue(providerConfig.base_url));
|
|
if (baseUrl === null) throw new Error(`model provider ${provider} base_url is missing or invalid`);
|
|
base.provider = provider;
|
|
base.baseUrl = baseUrl;
|
|
base.envKey = stringValue(providerConfig.env_key);
|
|
base.wireApi = stringValue(providerConfig.wire_api);
|
|
base.model = stringValue(parsed.model);
|
|
|
|
const auth = readAuthAPIKey(authPath);
|
|
base.authOpenAIKeyShape = auth.shape;
|
|
if (auth.apiKey !== null) {
|
|
base.apiKey = auth.apiKey;
|
|
base.apiKeySource = "auth-json";
|
|
} else if (base.envKey !== null && typeof process.env[base.envKey] === "string" && process.env[base.envKey]!.length > 0) {
|
|
base.apiKey = process.env[base.envKey]!;
|
|
base.apiKeySource = "env";
|
|
}
|
|
if (base.apiKey === null || base.apiKey.length === 0) {
|
|
throw new Error(base.envKey === null ? "auth OPENAI_API_KEY is missing or empty" : `auth OPENAI_API_KEY is missing and env ${base.envKey} is not present`);
|
|
}
|
|
base.ok = true;
|
|
return base;
|
|
} catch (error) {
|
|
base.error = error instanceof Error ? error.message : String(error);
|
|
return base;
|
|
}
|
|
});
|
|
}
|
|
|
|
function discoverCodexProfileConfigs(
|
|
codexDir: string,
|
|
defaultTempUnschedulable = defaultCodexTempUnschedulablePolicy(),
|
|
defaultPriority = defaultAccountPriority,
|
|
): CodexPoolProfileConfig[] {
|
|
return readdirSync(codexDir)
|
|
.filter((file) => file === "config.toml" || file.startsWith("config.toml."))
|
|
.sort((a, b) => {
|
|
if (a === "config.toml") return -1;
|
|
if (b === "config.toml") return 1;
|
|
return a.localeCompare(b);
|
|
})
|
|
.map((configFile) => {
|
|
const suffix = configFile === "config.toml" ? "" : configFile.slice("config.toml.".length);
|
|
const profile = suffix === "" ? "default" : suffix;
|
|
return {
|
|
profile,
|
|
accountName: null,
|
|
configFile,
|
|
authFile: suffix === "" ? "auth.json" : `auth.json.${suffix}`,
|
|
fallbackConfigFile: null,
|
|
fallbackAuthFile: null,
|
|
openaiResponsesWebSocketsV2Mode: null,
|
|
upstreamUserAgent: null,
|
|
priority: defaultPriority,
|
|
capacity: null,
|
|
loadFactor: null,
|
|
tempUnschedulable: defaultTempUnschedulable,
|
|
};
|
|
});
|
|
}
|
|
|
|
function resolveProfileFiles(codexDir: string, profile: CodexPoolProfileConfig): { configFile: string; authFile: string } {
|
|
const configFile = existsSync(join(codexDir, profile.configFile)) || profile.fallbackConfigFile === null
|
|
? profile.configFile
|
|
: profile.fallbackConfigFile;
|
|
const authFile = existsSync(join(codexDir, profile.authFile)) || profile.fallbackAuthFile === null
|
|
? profile.authFile
|
|
: profile.fallbackAuthFile;
|
|
return { configFile, authFile };
|
|
}
|
|
|
|
function readCodexPoolConfig(): CodexPoolConfig {
|
|
const defaults = defaultCodexPoolConfig();
|
|
if (!existsSync(codexPoolConfigPath)) return defaults;
|
|
const parsed = Bun.YAML.parse(readFileSync(codexPoolConfigPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) throw new Error(`${codexPoolConfigPath} must contain a YAML object`);
|
|
const pool = parsed.pool;
|
|
if (!isRecord(pool)) throw new Error(`${codexPoolConfigPath}.pool must be a YAML object`);
|
|
rejectSchedulableYamlField(pool, "pool");
|
|
const defaultTempUnschedulable = readTempUnschedulablePolicy(pool.defaultTempUnschedulable, "pool.defaultTempUnschedulable", defaults.defaultTempUnschedulable);
|
|
const defaultAccountPriorityValue = readAccountPriority(pool.defaultAccountPriority, "pool.defaultAccountPriority");
|
|
const defaultAccountCapacityValue = readAccountCapacity(pool.defaultAccountCapacity, "pool.defaultAccountCapacity");
|
|
const defaultAccountLoadFactorValue = readAccountLoadFactor(pool.defaultAccountLoadFactor, "pool.defaultAccountLoadFactor");
|
|
const profiles = readProfileConfig(
|
|
parsed.profiles,
|
|
defaults.profiles,
|
|
defaultTempUnschedulable,
|
|
defaultAccountPriorityValue,
|
|
);
|
|
const declaredAccountCapacity = desiredProfileCapacityTotal(profiles, defaultAccountCapacityValue);
|
|
const minOwnerConcurrencySource = pool.minOwnerConcurrency === undefined || pool.minOwnerConcurrency === null ? "auto" : "yaml";
|
|
const minOwnerConcurrency = minOwnerConcurrencySource === "auto"
|
|
? Math.max(1, declaredAccountCapacity)
|
|
: readOwnerConcurrency(pool.minOwnerConcurrency, "pool.minOwnerConcurrency");
|
|
const config: CodexPoolConfig = {
|
|
groupName: stringValue(pool.groupName) ?? defaults.groupName,
|
|
apiKeyName: stringValue(pool.apiKeyName) ?? defaults.apiKeyName,
|
|
apiKeySecretName: stringValue(pool.apiKeySecretName) ?? defaults.apiKeySecretName,
|
|
apiKeySecretKey: stringValue(pool.apiKeySecretKey) ?? defaults.apiKeySecretKey,
|
|
minOwnerBalanceUsd: numberValue(pool.minOwnerBalanceUsd) ?? defaults.minOwnerBalanceUsd,
|
|
minOwnerConcurrency,
|
|
minOwnerConcurrencySource,
|
|
defaultAccountPriority: defaultAccountPriorityValue,
|
|
defaultAccountCapacity: defaultAccountCapacityValue,
|
|
defaultAccountLoadFactor: defaultAccountLoadFactorValue,
|
|
defaultTempUnschedulable,
|
|
profiles,
|
|
publicExposure: readPublicExposureConfig(parsed.publicExposure, defaults.publicExposure),
|
|
localCodex: readLocalCodexConfig(parsed.localCodex, defaults.localCodex),
|
|
sentinel: readCodexPoolSentinelConfig(parsed.sentinel, defaults.sentinel, codexPoolConfigPath),
|
|
};
|
|
validateKubernetesName(config.groupName, "pool.groupName", false);
|
|
validateKubernetesName(config.apiKeySecretName, "pool.apiKeySecretName", true);
|
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(config.apiKeySecretKey)) {
|
|
throw new Error(`${codexPoolConfigPath}.pool.apiKeySecretKey must be a valid secret key name`);
|
|
}
|
|
if (config.minOwnerBalanceUsd <= 0) throw new Error(`${codexPoolConfigPath}.pool.minOwnerBalanceUsd must be > 0`);
|
|
if (declaredAccountCapacity > 0 && config.minOwnerConcurrency < declaredAccountCapacity) {
|
|
throw new Error(`${codexPoolConfigPath}.pool.minOwnerConcurrency must be >= the sum of declared account capacities (${declaredAccountCapacity})`);
|
|
}
|
|
return config;
|
|
}
|
|
|
|
function defaultCodexPoolConfig(): CodexPoolConfig {
|
|
return {
|
|
groupName: defaultPoolGroupName,
|
|
apiKeyName: defaultPoolApiKeyName,
|
|
apiKeySecretName: defaultPoolApiKeySecretName,
|
|
apiKeySecretKey: defaultPoolApiKeySecretKey,
|
|
minOwnerBalanceUsd: defaultMinOwnerBalanceUsd,
|
|
minOwnerConcurrency: 1,
|
|
minOwnerConcurrencySource: "auto",
|
|
defaultAccountPriority,
|
|
defaultAccountCapacity: 5,
|
|
defaultAccountLoadFactor: 1,
|
|
defaultTempUnschedulable: defaultCodexTempUnschedulablePolicy(),
|
|
profiles: [],
|
|
publicExposure: {
|
|
enabled: false,
|
|
proxyName: "platform-infra-sub2api",
|
|
configMapName: "sub2api-frpc-config",
|
|
deploymentName: "sub2api-frpc",
|
|
frpcImage: "fatedier/frpc:v0.68.1",
|
|
serverAddr: "74.48.78.17",
|
|
serverPort: 7000,
|
|
remotePort: 21880,
|
|
localIP: serviceDns.split(":")[0],
|
|
localPort: 8080,
|
|
publicBaseUrl: "http://74.48.78.17:21880",
|
|
masterBaseUrl: "http://127.0.0.1:21880",
|
|
masterFrps: {
|
|
configPath: "/opt/hwlab-frp/frps.dev.toml",
|
|
containerName: "hwlab-frps-dev",
|
|
},
|
|
masterCaddy: {
|
|
enabled: true,
|
|
domain: "sub2api.74-48-78-17.nip.io",
|
|
configPath: "/etc/caddy/Caddyfile",
|
|
serviceName: "caddy",
|
|
upstreamBaseUrl: "http://127.0.0.1:21880",
|
|
responseHeaderTimeoutSeconds: 180,
|
|
},
|
|
},
|
|
localCodex: {
|
|
backupSuffix: "pre-sub2api",
|
|
providerName: "OpenAI",
|
|
wireApi: "responses",
|
|
supportsWebSockets: true,
|
|
responsesWebSocketsV2: true,
|
|
responsesSmokeModel: "gpt-5.5",
|
|
},
|
|
sentinel: defaultCodexPoolSentinelConfig(),
|
|
};
|
|
}
|
|
|
|
export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePolicy {
|
|
return {
|
|
enabled: true,
|
|
rules: [
|
|
{
|
|
statusCode: 400,
|
|
keywords: ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "bad_response_status_code", "model_not_found", "no available channel for model", "unsupported", "not supported", "not support", "暂不支持", "可用模型"],
|
|
durationMinutes: 120,
|
|
description: "Stable upstream 400 model-routing or Responses encrypted-content compatibility failures should use another account.",
|
|
},
|
|
{
|
|
statusCode: 401,
|
|
keywords: ["unauthorized", "invalid api key", "invalid_api_key", "authentication", "recovered upstream error"],
|
|
durationMinutes: 120,
|
|
description: "Credential/auth failures should use the longest cooldown.",
|
|
},
|
|
{
|
|
statusCode: 403,
|
|
keywords: ["forbidden", "access denied", "quota", "billing", "capacity", "weekly limit", "less than 10% of your weekly limit left", "run /status for a breakdown", "recovered upstream error"],
|
|
durationMinutes: 120,
|
|
description: "Permission, quota, or account-state failures should use the longest cooldown.",
|
|
},
|
|
{
|
|
statusCode: 429,
|
|
keywords: ["capacity", "rate limit", "rate_limit", "quota", "weekly limit", "less than 10% of your weekly limit left", "run /status for a breakdown", "too many requests", "overloaded", "resource_exhausted", "recovered upstream error"],
|
|
durationMinutes: 10,
|
|
description: "Capacity and rate-limit responses are often temporary; start with a ten-minute cooldown and use another account.",
|
|
},
|
|
{
|
|
statusCode: 500,
|
|
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "recovered upstream error"],
|
|
durationMinutes: 10,
|
|
description: "Transient upstream server failures should start with a ten-minute cooldown and prefer another account.",
|
|
},
|
|
{
|
|
statusCode: 502,
|
|
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "websocket dial", "handshake response", "recovered upstream error"],
|
|
durationMinutes: 30,
|
|
description: "Gateway upstream failures, including recovered upstream error wrappers, should cool down longer.",
|
|
},
|
|
{
|
|
statusCode: 413,
|
|
keywords: ["openai_error", "payload too large", "request too large", "context length", "context window", "maximum context"],
|
|
durationMinutes: 30,
|
|
description: "Large-context upstream failures should cool down the selected account so a larger-context channel can handle the request.",
|
|
},
|
|
{
|
|
statusCode: 503,
|
|
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "recovered upstream error", "model_not_found", "no available channel for model"],
|
|
durationMinutes: 30,
|
|
description: "Service unavailable and upstream model-routing failures should cool down longer than one-off transient failures.",
|
|
},
|
|
{
|
|
statusCode: 504,
|
|
keywords: ["gateway timeout", "timeout", "upstream", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "recovered upstream error"],
|
|
durationMinutes: 30,
|
|
description: "Gateway timeout responses should cool down the selected account so another account can handle the next request.",
|
|
},
|
|
{
|
|
statusCode: 524,
|
|
keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "recovered upstream error"],
|
|
durationMinutes: 30,
|
|
description: "Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.",
|
|
},
|
|
{
|
|
statusCode: 529,
|
|
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "recovered upstream error"],
|
|
durationMinutes: 30,
|
|
description: "Provider overloaded responses should cool down longer than generic transient failures and use another account.",
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
function readProfileConfig(
|
|
value: unknown,
|
|
defaults: CodexPoolProfileConfig[],
|
|
defaultTempUnschedulable: CodexTempUnschedulablePolicy,
|
|
defaultPriority: number,
|
|
): CodexPoolProfileConfig[] {
|
|
if (!isRecord(value)) return defaults;
|
|
const entries = value.entries;
|
|
if (!Array.isArray(entries)) throw new Error(`${codexPoolConfigPath}.profiles.entries must be a YAML array`);
|
|
return entries.map((entry, index) => {
|
|
if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] must be an object`);
|
|
rejectSchedulableYamlField(entry, `profiles.entries[${index}]`);
|
|
const profile = stringValue(entry.profile);
|
|
const configFile = stringValue(entry.configFile);
|
|
const authFile = stringValue(entry.authFile);
|
|
if (profile === null || configFile === null || authFile === null) {
|
|
throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] requires profile, configFile, and authFile`);
|
|
}
|
|
const accountName = stringValue(entry.accountName);
|
|
if (accountName !== null) validateKubernetesName(accountName, `profiles.entries[${index}].accountName`, false);
|
|
validateCodexFileName(configFile, `profiles.entries[${index}].configFile`);
|
|
validateCodexFileName(authFile, `profiles.entries[${index}].authFile`);
|
|
const fallbackConfigFile = stringValue(entry.fallbackConfigFile);
|
|
const fallbackAuthFile = stringValue(entry.fallbackAuthFile);
|
|
if (fallbackConfigFile !== null) validateCodexFileName(fallbackConfigFile, `profiles.entries[${index}].fallbackConfigFile`);
|
|
if (fallbackAuthFile !== null) validateCodexFileName(fallbackAuthFile, `profiles.entries[${index}].fallbackAuthFile`);
|
|
const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`);
|
|
const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`);
|
|
const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority);
|
|
const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`);
|
|
const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`);
|
|
const tempUnschedulable = readTempUnschedulablePolicy(entry.tempUnschedulable, `profiles.entries[${index}].tempUnschedulable`, defaultTempUnschedulable);
|
|
return {
|
|
profile,
|
|
accountName,
|
|
configFile,
|
|
authFile,
|
|
fallbackConfigFile,
|
|
fallbackAuthFile,
|
|
openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent,
|
|
priority,
|
|
capacity,
|
|
loadFactor,
|
|
tempUnschedulable,
|
|
};
|
|
});
|
|
}
|
|
|
|
function desiredProfileCapacityTotal(profiles: CodexPoolProfileConfig[], defaultCapacity: number): number {
|
|
return profiles.reduce((total, profile) => total + (profile.capacity ?? defaultCapacity), 0);
|
|
}
|
|
|
|
function rejectSchedulableYamlField(value: Record<string, unknown>, key: string): void {
|
|
if (Object.prototype.hasOwnProperty.call(value, "schedulable")) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}.schedulable is process control, not durable YAML config; use codex-pool sync actions instead`);
|
|
}
|
|
}
|
|
|
|
function readOpenAIResponsesWebSocketsV2Mode(value: unknown, key: string): OpenAIResponsesWebSocketsV2Mode | null {
|
|
if (value === undefined || value === null) return null;
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`);
|
|
if (text === "off" || text === "ctx_pool" || text === "passthrough") return text;
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be one of off, ctx_pool, passthrough`);
|
|
}
|
|
|
|
function readUpstreamUserAgent(value: unknown, key: string): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
const text = stringValue(value);
|
|
if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`);
|
|
if (text.length > 512) throw new Error(`${codexPoolConfigPath}.${key} must be at most 512 characters`);
|
|
if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`);
|
|
return text;
|
|
}
|
|
|
|
function readAccountPriority(value: unknown, key: string, fallback = defaultAccountPriority): number {
|
|
if (value === undefined || value === null) return fallback;
|
|
const priority = numberValue(value);
|
|
if (priority === null || !Number.isInteger(priority) || priority < 0 || priority > 1000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 0 to 1000`);
|
|
}
|
|
return priority;
|
|
}
|
|
|
|
function readAccountCapacity(value: unknown, key: string): number {
|
|
const capacity = numberValue(value);
|
|
if (capacity === null || !Number.isInteger(capacity) || capacity < 1 || capacity > 1000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`);
|
|
}
|
|
return capacity;
|
|
}
|
|
|
|
function readOwnerConcurrency(value: unknown, key: string): number {
|
|
const concurrency = numberValue(value);
|
|
if (concurrency === null || !Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 100000`);
|
|
}
|
|
return concurrency;
|
|
}
|
|
|
|
function readAccountLoadFactor(value: unknown, key: string): number {
|
|
const loadFactor = numberValue(value);
|
|
if (loadFactor === null || !Number.isInteger(loadFactor) || loadFactor < 1 || loadFactor > 1000) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`);
|
|
}
|
|
return loadFactor;
|
|
}
|
|
|
|
function readCaddyTimeoutSeconds(value: unknown, key: string, fallback: number): number {
|
|
if (value === undefined || value === null) return fallback;
|
|
const seconds = numberValue(value);
|
|
if (seconds === null || !Number.isInteger(seconds) || seconds < 30 || seconds > 900) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 30 to 900`);
|
|
}
|
|
return seconds;
|
|
}
|
|
|
|
function readTempUnschedulablePolicy(value: unknown, key: string, fallback: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
|
|
if (value === undefined || value === null) return cloneTempUnschedulablePolicy(fallback);
|
|
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
|
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled);
|
|
const rules = value.rules === undefined || value.rules === null
|
|
? cloneTempUnschedulablePolicy(fallback).rules
|
|
: readTempUnschedulableRules(value.rules, `${key}.rules`);
|
|
if (enabled && rules.length === 0) throw new Error(`${codexPoolConfigPath}.${key}.rules must not be empty when enabled=true`);
|
|
return { enabled, rules };
|
|
}
|
|
|
|
function readTempUnschedulableRules(value: unknown, key: string): CodexTempUnschedulableRule[] {
|
|
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
|
return value.map((entry, index) => {
|
|
if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.${key}[${index}] must be an object`);
|
|
const statusCode = numberValue(entry.statusCode ?? entry.errorCode);
|
|
if (statusCode === null || !Number.isInteger(statusCode) || statusCode < 100 || statusCode > 599) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}[${index}].statusCode must be an HTTP status code from 100 to 599`);
|
|
}
|
|
const durationMinutes = numberValue(entry.durationMinutes);
|
|
if (durationMinutes === null || !Number.isInteger(durationMinutes) || durationMinutes < 1 || durationMinutes > 1440) {
|
|
throw new Error(`${codexPoolConfigPath}.${key}[${index}].durationMinutes must be an integer from 1 to 1440`);
|
|
}
|
|
const keywords = readTempUnschedulableKeywords(entry.keywords, `${key}[${index}].keywords`);
|
|
const description = stringValue(entry.description);
|
|
if (description !== null && description.length > 240) throw new Error(`${codexPoolConfigPath}.${key}[${index}].description must be at most 240 characters`);
|
|
return { statusCode, keywords, durationMinutes, description };
|
|
});
|
|
}
|
|
|
|
function readTempUnschedulableKeywords(value: unknown, key: string): string[] {
|
|
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
|
const seen = new Set<string>();
|
|
const keywords: string[] = [];
|
|
for (const item of value) {
|
|
const keyword = stringValue(item);
|
|
if (keyword === null) throw new Error(`${codexPoolConfigPath}.${key} entries must be non-empty strings`);
|
|
if (keyword.length > 120) throw new Error(`${codexPoolConfigPath}.${key} entries must be at most 120 characters`);
|
|
if (/[\r\n]/u.test(keyword)) throw new Error(`${codexPoolConfigPath}.${key} entries must not contain newlines`);
|
|
const normalized = keyword.toLowerCase();
|
|
if (seen.has(normalized)) continue;
|
|
seen.add(normalized);
|
|
keywords.push(keyword);
|
|
}
|
|
if (keywords.length === 0) throw new Error(`${codexPoolConfigPath}.${key} must not be empty`);
|
|
return keywords;
|
|
}
|
|
|
|
function cloneTempUnschedulablePolicy(policy: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
|
|
return {
|
|
enabled: policy.enabled,
|
|
rules: policy.rules.map((rule) => ({
|
|
statusCode: rule.statusCode,
|
|
keywords: [...rule.keywords],
|
|
durationMinutes: rule.durationMinutes,
|
|
description: rule.description,
|
|
})),
|
|
};
|
|
}
|
|
|
|
function readBooleanConfig(value: unknown, key: string, fallback: boolean): boolean {
|
|
if (value === undefined || value === null) return fallback;
|
|
const parsed = booleanValue(value);
|
|
if (parsed === null) throw new Error(`${codexPoolConfigPath}.${key} must be a boolean`);
|
|
return parsed;
|
|
}
|
|
|
|
function readPublicExposureConfig(value: unknown, defaults: CodexPoolPublicExposureConfig): CodexPoolPublicExposureConfig {
|
|
if (!isRecord(value)) return defaults;
|
|
const masterFrpsValue = isRecord(value.masterFrps) ? value.masterFrps : {};
|
|
const masterCaddyValue = isRecord(value.masterCaddy) ? value.masterCaddy : {};
|
|
const config: CodexPoolPublicExposureConfig = {
|
|
enabled: booleanValue(value.enabled) ?? defaults.enabled,
|
|
proxyName: stringValue(value.proxyName) ?? defaults.proxyName,
|
|
configMapName: stringValue(value.configMapName) ?? defaults.configMapName,
|
|
deploymentName: stringValue(value.deploymentName) ?? defaults.deploymentName,
|
|
frpcImage: stringValue(value.frpcImage) ?? defaults.frpcImage,
|
|
serverAddr: stringValue(value.serverAddr) ?? defaults.serverAddr,
|
|
serverPort: numberValue(value.serverPort) ?? defaults.serverPort,
|
|
remotePort: numberValue(value.remotePort) ?? defaults.remotePort,
|
|
localIP: stringValue(value.localIP) ?? defaults.localIP,
|
|
localPort: numberValue(value.localPort) ?? defaults.localPort,
|
|
publicBaseUrl: normalizeBaseUrl(stringValue(value.publicBaseUrl)) ?? defaults.publicBaseUrl,
|
|
masterBaseUrl: normalizeBaseUrl(stringValue(value.masterBaseUrl)) ?? defaults.masterBaseUrl,
|
|
masterFrps: {
|
|
configPath: stringValue(masterFrpsValue.configPath) ?? defaults.masterFrps.configPath,
|
|
containerName: stringValue(masterFrpsValue.containerName) ?? defaults.masterFrps.containerName,
|
|
},
|
|
masterCaddy: {
|
|
enabled: booleanValue(masterCaddyValue.enabled) ?? defaults.masterCaddy.enabled,
|
|
domain: stringValue(masterCaddyValue.domain) ?? defaults.masterCaddy.domain,
|
|
configPath: stringValue(masterCaddyValue.configPath) ?? defaults.masterCaddy.configPath,
|
|
serviceName: stringValue(masterCaddyValue.serviceName) ?? defaults.masterCaddy.serviceName,
|
|
upstreamBaseUrl: normalizeBaseUrl(stringValue(masterCaddyValue.upstreamBaseUrl)) ?? defaults.masterCaddy.upstreamBaseUrl,
|
|
responseHeaderTimeoutSeconds: readCaddyTimeoutSeconds(
|
|
masterCaddyValue.responseHeaderTimeoutSeconds,
|
|
"publicExposure.masterCaddy.responseHeaderTimeoutSeconds",
|
|
defaults.masterCaddy.responseHeaderTimeoutSeconds,
|
|
),
|
|
},
|
|
};
|
|
validateKubernetesName(config.configMapName, "publicExposure.configMapName", true);
|
|
validateKubernetesName(config.deploymentName, "publicExposure.deploymentName", true);
|
|
validateProxyName(config.proxyName, "publicExposure.proxyName");
|
|
validatePort(config.serverPort, "publicExposure.serverPort");
|
|
validatePort(config.remotePort, "publicExposure.remotePort");
|
|
validatePort(config.localPort, "publicExposure.localPort");
|
|
if (!/^[A-Za-z0-9._:/-]+$/u.test(config.frpcImage)) throw new Error(`${codexPoolConfigPath}.publicExposure.frpcImage has an unsupported format`);
|
|
if (!/^[A-Za-z0-9._:-]+$/u.test(config.serverAddr)) throw new Error(`${codexPoolConfigPath}.publicExposure.serverAddr has an unsupported format`);
|
|
if (!/^[A-Za-z0-9._:-]+$/u.test(config.localIP)) throw new Error(`${codexPoolConfigPath}.publicExposure.localIP has an unsupported format`);
|
|
if (!config.masterFrps.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterFrps.configPath must be absolute`);
|
|
validateProxyName(config.masterFrps.containerName, "publicExposure.masterFrps.containerName");
|
|
validatePublicHostname(config.masterCaddy.domain, "publicExposure.masterCaddy.domain");
|
|
if (!config.masterCaddy.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.configPath must be absolute`);
|
|
validateProxyName(config.masterCaddy.serviceName, "publicExposure.masterCaddy.serviceName");
|
|
const upstream = new URL(config.masterCaddy.upstreamBaseUrl);
|
|
if (upstream.protocol !== "http:") throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.upstreamBaseUrl must use http:// for the local upstream`);
|
|
return config;
|
|
}
|
|
|
|
function readLocalCodexConfig(value: unknown, defaults: CodexPoolLocalCodexConfig): CodexPoolLocalCodexConfig {
|
|
if (!isRecord(value)) return defaults;
|
|
const config = {
|
|
backupSuffix: stringValue(value.backupSuffix) ?? defaults.backupSuffix,
|
|
providerName: stringValue(value.providerName) ?? defaults.providerName,
|
|
wireApi: stringValue(value.wireApi) ?? defaults.wireApi,
|
|
supportsWebSockets: readBooleanConfig(value.supportsWebSockets, "localCodex.supportsWebSockets", defaults.supportsWebSockets),
|
|
responsesWebSocketsV2: readBooleanConfig(value.responsesWebSocketsV2, "localCodex.responsesWebSocketsV2", defaults.responsesWebSocketsV2),
|
|
responsesSmokeModel: stringValue(value.responsesSmokeModel) ?? defaults.responsesSmokeModel,
|
|
};
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(config.backupSuffix)) throw new Error(`${codexPoolConfigPath}.localCodex.backupSuffix has an unsupported format`);
|
|
validateProxyName(config.providerName, "localCodex.providerName");
|
|
validateProxyName(config.wireApi, "localCodex.wireApi");
|
|
validateModelName(config.responsesSmokeModel, "localCodex.responsesSmokeModel");
|
|
return config;
|
|
}
|
|
|
|
function validateCodexFileName(value: string, key: string): void {
|
|
if (!/^(config\.toml|config\.toml\.[A-Za-z0-9._-]+|auth\.json|auth\.json\.[A-Za-z0-9._-]+)$/u.test(value)) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} has an unsupported file name`);
|
|
}
|
|
}
|
|
|
|
function validateProxyName(value: string, key: string): void {
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`);
|
|
}
|
|
|
|
function validateModelName(value: string, key: string): void {
|
|
if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported model name`);
|
|
}
|
|
|
|
function validatePublicHostname(value: string, key: string): void {
|
|
if (value.length > 253 || !/^[A-Za-z0-9.-]+$/u.test(value) || value.startsWith(".") || value.endsWith(".")) {
|
|
throw new Error(`${codexPoolConfigPath}.${key} has an unsupported hostname format`);
|
|
}
|
|
}
|
|
|
|
function validatePort(value: number, key: string): void {
|
|
if (!Number.isInteger(value) || value < 1 || value > 65535) throw new Error(`${codexPoolConfigPath}.${key} must be an integer port`);
|
|
}
|
|
|
|
function readAuthAPIKey(authPath: string): { apiKey: string | null; shape: string } {
|
|
if (!existsSync(authPath)) return { apiKey: null, shape: "missing" };
|
|
const parsed = JSON.parse(readFileSync(authPath, "utf8")) as unknown;
|
|
if (!isRecord(parsed)) return { apiKey: null, shape: "non-object" };
|
|
const value = parsed.OPENAI_API_KEY;
|
|
const shape = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
|
|
if (typeof value === "string" && value.length > 0) return { apiKey: value, shape };
|
|
return { apiKey: null, shape };
|
|
}
|
|
|
|
function uniqueAccountName(profile: string, seen: Set<string>): string {
|
|
const normalized = profile
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9._-]+/gu, "-")
|
|
.replace(/^-+|-+$/gu, "") || "default";
|
|
let candidate = `unidesk-codex-${normalized}`;
|
|
let counter = 2;
|
|
while (seen.has(candidate)) {
|
|
candidate = `unidesk-codex-${normalized}-${counter}`;
|
|
counter += 1;
|
|
}
|
|
seen.add(candidate);
|
|
return candidate;
|
|
}
|
|
|
|
function redactProfile(profile: CodexProfile): Record<string, unknown> {
|
|
return {
|
|
profile: profile.profile,
|
|
accountName: profile.accountName,
|
|
configFile: profile.configFile,
|
|
authFile: profile.authFile,
|
|
provider: profile.provider || null,
|
|
baseUrl: profile.baseUrl || null,
|
|
wireApi: profile.wireApi,
|
|
model: profile.model,
|
|
envKey: profile.envKey,
|
|
apiKeySource: profile.apiKeySource,
|
|
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
priority: profile.priority,
|
|
capacity: profile.capacity,
|
|
loadFactor: profile.loadFactor,
|
|
tempUnschedulable: tempUnschedulableSummary(profile.tempUnschedulable),
|
|
apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0,
|
|
apiKeyBytes: profile.apiKey === null ? 0 : Buffer.byteLength(profile.apiKey, "utf8"),
|
|
apiKeyFingerprint: profile.apiKey === null ? null : fingerprint(profile.apiKey),
|
|
authOpenAIKeyShape: profile.authOpenAIKeyShape,
|
|
ok: profile.ok,
|
|
error: profile.error,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactProfile(profile: CodexProfile): Record<string, unknown> {
|
|
return {
|
|
profile: profile.profile,
|
|
accountName: profile.accountName,
|
|
provider: profile.provider || null,
|
|
model: profile.model,
|
|
priority: profile.priority,
|
|
capacity: profile.capacity,
|
|
loadFactor: profile.loadFactor,
|
|
tempUnschedulableEnabled: profile.tempUnschedulable.enabled && profile.tempUnschedulable.rules.length > 0,
|
|
tempUnschedulableRuleCount: profile.tempUnschedulable.enabled ? profile.tempUnschedulable.rules.length : 0,
|
|
apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0,
|
|
ok: profile.ok,
|
|
error: profile.error,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function renderSub2ApiTempUnschedulableCredentials(policy: CodexTempUnschedulablePolicy): Record<string, unknown> {
|
|
return {
|
|
temp_unschedulable_enabled: policy.enabled && policy.rules.length > 0,
|
|
temp_unschedulable_rules: policy.enabled
|
|
? policy.rules.map((rule) => ({
|
|
error_code: rule.statusCode,
|
|
keywords: [...rule.keywords],
|
|
duration_minutes: rule.durationMinutes,
|
|
description: rule.description ?? "",
|
|
}))
|
|
: [],
|
|
};
|
|
}
|
|
|
|
function tempUnschedulableSummary(policy: CodexTempUnschedulablePolicy): Record<string, unknown> {
|
|
return {
|
|
enabled: policy.enabled && policy.rules.length > 0,
|
|
ruleCount: policy.enabled ? policy.rules.length : 0,
|
|
statusCodes: policy.enabled ? policy.rules.map((rule) => rule.statusCode) : [],
|
|
};
|
|
}
|
|
|
|
function codexPoolConfigSummary(pool: CodexPoolConfig): Record<string, unknown> {
|
|
const accountCapacityTotal = desiredAccountCapacityTotal(pool);
|
|
return {
|
|
groupName: pool.groupName,
|
|
apiKeyName: pool.apiKeyName,
|
|
apiKeySecretName: pool.apiKeySecretName,
|
|
apiKeySecretKey: pool.apiKeySecretKey,
|
|
minOwnerBalanceUsd: pool.minOwnerBalanceUsd,
|
|
minOwnerConcurrency: pool.minOwnerConcurrency,
|
|
minOwnerConcurrencySource: pool.minOwnerConcurrencySource,
|
|
accountCapacityTotal,
|
|
defaultAccountPriority: pool.defaultAccountPriority,
|
|
defaultAccountCapacity: pool.defaultAccountCapacity,
|
|
defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
|
|
defaultTempUnschedulable: tempUnschedulableSummary(pool.defaultTempUnschedulable),
|
|
profileCount: pool.profiles.length,
|
|
publicExposure: publicExposureSummary(pool),
|
|
localCodex: pool.localCodex,
|
|
sentinel: codexPoolSentinelSummary(pool.sentinel),
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool plan --full",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactText(value: unknown, limit = 260): string {
|
|
if (typeof value !== "string") return "";
|
|
return value.length > limit ? `${value.slice(0, limit)}…` : value;
|
|
}
|
|
|
|
function recordArray(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.filter(isRecord) : [];
|
|
}
|
|
|
|
function pickSummaryFields(item: Record<string, unknown>, keys: string[]): Record<string, unknown> {
|
|
const result: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
if (item[key] !== undefined) result[key] = item[key];
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function compactStatusBlock(block: unknown, keys: string[]): Record<string, unknown> | null {
|
|
if (!isRecord(block)) return null;
|
|
const items = recordArray(block.items);
|
|
const attentionItems = items
|
|
.filter((item) => item.ok === false)
|
|
.map((item) => pickSummaryFields(item, keys));
|
|
return {
|
|
ok: block.ok,
|
|
desired: block.desired,
|
|
missing: block.missing,
|
|
mismatched: block.mismatched,
|
|
totals: block.totals,
|
|
itemCount: items.length,
|
|
attentionItems,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactTempUnschedulableStatus(block: unknown): Record<string, unknown> | null {
|
|
if (!isRecord(block)) return null;
|
|
const items = recordArray(block.items);
|
|
const compactItems = items.map((item) => {
|
|
const result = pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"expectedEnabled",
|
|
"runtimeEnabled",
|
|
"expectedRuleCount",
|
|
"runtimeRuleCount",
|
|
"status",
|
|
"schedulable",
|
|
"tempUnschedulableUntil",
|
|
"tempUnschedulableSet",
|
|
"ok",
|
|
]);
|
|
if (item.tempUnschedulableReasonPreview !== undefined) {
|
|
result.tempUnschedulableReason = tempUnschedulableReasonSummary(item.tempUnschedulableReasonPreview);
|
|
}
|
|
return result;
|
|
});
|
|
const frozenItems = compactItems.filter((item) => item.tempUnschedulableSet === true);
|
|
const focusedFrozenItems = uniqueByAccountName([
|
|
...frozenItems.filter((item) => isRecord(item.tempUnschedulableReason) && item.tempUnschedulableReason.statusCode === 400),
|
|
...frozenItems.filter((item) => item.schedulable === false),
|
|
...frozenItems,
|
|
]).slice(0, 6);
|
|
return {
|
|
ok: block.ok,
|
|
desired: block.desired,
|
|
enabledCount: Array.isArray(block.enabled) ? block.enabled.length : undefined,
|
|
missing: block.missing,
|
|
mismatched: block.mismatched,
|
|
itemCount: compactItems.length,
|
|
frozenCount: frozenItems.length,
|
|
frozen: focusedFrozenItems,
|
|
manuallyUnschedulable: compactItems.filter((item) => item.schedulable === false && item.tempUnschedulableSet !== true),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function tempUnschedulableReasonSummary(value: unknown): Record<string, unknown> {
|
|
const reason = compactText(value, 180);
|
|
const statusCodeMatch = reason.match(/"status_code":(\d{3})/u) ?? reason.match(/OpenAI\s+(\d{3})/u) ?? reason.match(/\((\d{3})\)/u);
|
|
const keywordMatch = reason.match(/"matched_keyword":"([^"]+)"/u);
|
|
const summary: Record<string, unknown> = {
|
|
statusCode: statusCodeMatch ? Number(statusCodeMatch[1]) : undefined,
|
|
matchedKeyword: keywordMatch ? keywordMatch[1] : undefined,
|
|
};
|
|
if (!statusCodeMatch || !keywordMatch) summary.preview = compactText(reason, 120);
|
|
return summary;
|
|
}
|
|
|
|
function uniqueByAccountName(items: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
const seen = new Set<unknown>();
|
|
const result: Record<string, unknown>[] = [];
|
|
for (const item of items) {
|
|
const key = item.accountName ?? item.accountId;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
result.push(item);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function compactRuntimeCapability(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
const probe = isRecord(block.probe) ? block.probe : {};
|
|
const logEvidence = isRecord(probe.logEvidence) ? probe.logEvidence : {};
|
|
const accountState = isRecord(probe.accountState) ? probe.accountState : {};
|
|
const resources = isRecord(block.resources) ? block.resources : {};
|
|
const requirement = isRecord(block.requirement) ? block.requirement : {};
|
|
return {
|
|
ok: block.ok,
|
|
required: block.required,
|
|
capability: block.capability,
|
|
outcome: block.outcome,
|
|
requirement: Object.keys(requirement).length === 0 ? undefined : {
|
|
statusCode: requirement.statusCode,
|
|
probeKeyword: requirement.probeKeyword,
|
|
durationMinutes: requirement.durationMinutes,
|
|
sourceAccountName: requirement.sourceAccountName,
|
|
},
|
|
probe: Object.keys(probe).length === 0 ? undefined : {
|
|
requestId: probe.requestId,
|
|
durationMs: probe.durationMs,
|
|
httpStatus: probe.httpStatus,
|
|
responseOk: probe.responseOk,
|
|
accountState: Object.keys(accountState).length === 0 ? undefined : {
|
|
accountId: accountState.accountId,
|
|
status: accountState.status,
|
|
schedulable: accountState.schedulable,
|
|
tempUnschedulableUntil: accountState.tempUnschedulableUntil,
|
|
tempUnschedulableSet: accountState.tempUnschedulableSet,
|
|
tempUnschedulableReasonPreview: compactText(accountState.tempUnschedulableReasonPreview, 180),
|
|
},
|
|
logEvidence: Object.keys(logEvidence).length === 0 ? undefined : {
|
|
matchedLogLineCount: logEvidence.matchedLogLineCount,
|
|
failovers: logEvidence.failovers,
|
|
final: logEvidence.final,
|
|
},
|
|
bodyPreview: probe.responseOk === false ? compactText(probe.bodyPreview, 240) : undefined,
|
|
},
|
|
resources: Object.keys(resources).length === 0 ? undefined : {
|
|
groupId: resources.groupId,
|
|
accountId: resources.accountId,
|
|
badAccountId: resources.badAccountId,
|
|
goodAccountId: resources.goodAccountId,
|
|
apiKeyId: resources.apiKeyId,
|
|
valuesPrinted: false,
|
|
},
|
|
message: block.message,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactGatewayModels(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
return pickSummaryFields(block, [
|
|
"ok",
|
|
"httpStatus",
|
|
"modelCount",
|
|
"method",
|
|
"serviceDns",
|
|
"valuesPrinted",
|
|
]);
|
|
}
|
|
|
|
function compactGatewayResponses(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
const evidence = isRecord(block.evidence) ? block.evidence : {};
|
|
return {
|
|
ok: block.ok,
|
|
degraded: block.degraded,
|
|
outcome: block.outcome,
|
|
httpStatus: block.httpStatus,
|
|
method: block.method,
|
|
model: block.model,
|
|
requestId: block.requestId,
|
|
durationMs: block.durationMs,
|
|
outputTextPreview: block.outputTextPreview,
|
|
evidence: Object.keys(evidence).length === 0 ? undefined : {
|
|
matchedLogLineCount: evidence.matchedLogLineCount,
|
|
failoverCount: Array.isArray(evidence.failovers) ? evidence.failovers.length : undefined,
|
|
failovers: Array.isArray(evidence.failovers) ? evidence.failovers.slice(-5) : undefined,
|
|
final: evidence.final,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactGatewayResponsesRecent(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
return {
|
|
ok: block.ok,
|
|
degraded: block.degraded,
|
|
window: block.window,
|
|
tailLines: block.tailLines,
|
|
failoverCount: block.failoverCount,
|
|
forwardFailureCount: block.forwardFailureCount,
|
|
finalErrorCount: block.finalErrorCount,
|
|
slowFinalErrorCount: block.slowFinalErrorCount,
|
|
contextCanceledCount: block.contextCanceledCount,
|
|
ignoredProbeNoiseCount: block.ignoredProbeNoiseCount,
|
|
recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined,
|
|
recentForwardFailures: Array.isArray(block.recentForwardFailures) ? block.recentForwardFailures.slice(-4).reverse().map((item) => ({
|
|
...item,
|
|
errorPreview: compactText(item.errorPreview, 220),
|
|
})) : undefined,
|
|
recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-3).reverse() : undefined,
|
|
recentSlowFinalErrors: Array.isArray(block.recentSlowFinalErrors) ? block.recentSlowFinalErrors.slice(-3).reverse() : undefined,
|
|
recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-3).reverse() : undefined,
|
|
logsExitCode: block.logsExitCode,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactGatewayCompactRecent(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
return {
|
|
ok: block.ok,
|
|
degraded: block.degraded,
|
|
window: block.window,
|
|
tailLines: block.tailLines,
|
|
failureCount: block.failureCount,
|
|
successCount: block.successCount,
|
|
failoverCount: block.failoverCount,
|
|
finalErrorCount: block.finalErrorCount,
|
|
contextCanceledCount: block.contextCanceledCount,
|
|
recentFailures: Array.isArray(block.recentFailures) ? block.recentFailures.slice(-2).reverse() : undefined,
|
|
recentSuccesses: Array.isArray(block.recentSuccesses) ? block.recentSuccesses.slice(-2).reverse() : undefined,
|
|
recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined,
|
|
recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-2).reverse() : undefined,
|
|
recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-2).reverse() : undefined,
|
|
logsExitCode: block.logsExitCode,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactSentinelStatus(block: unknown): unknown {
|
|
if (!isRecord(block)) return block;
|
|
const runtime = isRecord(block.runtime) ? block.runtime : block;
|
|
const desired = isRecord(runtime.desired) ? runtime.desired : {};
|
|
const cronJob = isRecord(runtime.cronJob) ? runtime.cronJob : {};
|
|
const secret = isRecord(runtime.secret) ? runtime.secret : {};
|
|
const configMap = isRecord(runtime.configMap) ? runtime.configMap : {};
|
|
const state = isRecord(runtime.state) ? runtime.state : {};
|
|
const freezeReassert = isRecord(block.freezeReassert) ? block.freezeReassert : {};
|
|
return {
|
|
ok: block.ok,
|
|
action: block.action,
|
|
desired: {
|
|
monitorEnabled: desired.monitorEnabled,
|
|
actionsEnabled: desired.actionsEnabled,
|
|
schedule: desired.schedule,
|
|
cronJobName: desired.cronJobName,
|
|
configMapName: desired.configMapName,
|
|
credentialsSecretName: desired.credentialsSecretName,
|
|
stateConfigMapName: desired.stateConfigMapName,
|
|
},
|
|
cronJob: {
|
|
exists: cronJob.exists,
|
|
schedule: cronJob.schedule,
|
|
suspend: cronJob.suspend,
|
|
lastScheduleTime: cronJob.lastScheduleTime,
|
|
active: cronJob.active,
|
|
error: cronJob.error,
|
|
},
|
|
secret: {
|
|
exists: secret.exists,
|
|
profileSecretPresent: secret.profileSecretPresent,
|
|
valuesPrinted: false,
|
|
error: secret.error,
|
|
},
|
|
configMap: {
|
|
exists: configMap.exists,
|
|
configPresent: configMap.configPresent,
|
|
runnerPresent: configMap.runnerPresent,
|
|
error: configMap.error,
|
|
},
|
|
state: {
|
|
exists: state.exists,
|
|
accountCount: state.accountCount,
|
|
quarantinedCount: state.quarantinedCount,
|
|
quarantined: state.quarantined,
|
|
recentAccounts: state.recentAccounts,
|
|
lastRun: state.lastRun,
|
|
error: state.error,
|
|
},
|
|
freezeReassert: Object.keys(freezeReassert).length > 0 ? {
|
|
ok: freezeReassert.ok,
|
|
skipped: freezeReassert.skipped,
|
|
reason: freezeReassert.reason,
|
|
itemCount: freezeReassert.itemCount,
|
|
attentionItems: freezeReassert.attentionItems,
|
|
} : undefined,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function compactSentinelProbeResult(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (parsed === null) return null;
|
|
const probe = isRecord(parsed.probe) ? parsed.probe : {};
|
|
const summary = isRecord(probe.summary) ? probe.summary : {};
|
|
const state = isRecord(parsed.sentinelState) ? parsed.sentinelState : {};
|
|
return {
|
|
ok: parsed.ok,
|
|
mode: parsed.mode,
|
|
namespace: parsed.namespace,
|
|
job: parsed.job,
|
|
requestedAccounts: parsed.requestedAccounts,
|
|
summary: {
|
|
at: summary.at,
|
|
monitorEnabled: summary.monitorEnabled,
|
|
actionsEnabled: summary.actionsEnabled,
|
|
selected: summary.selected,
|
|
okCount: summary.okCount,
|
|
mismatchCount: summary.mismatchCount,
|
|
markerMismatchCount: summary.markerMismatchCount,
|
|
transportFailureCount: summary.transportFailureCount,
|
|
actionsTaken: summary.actionsTaken,
|
|
selection: summary.selection,
|
|
},
|
|
results: recordArray(probe.results).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"purpose",
|
|
"ok",
|
|
"markerMatched",
|
|
"httpStatus",
|
|
"durationMs",
|
|
"usage",
|
|
"outputHash",
|
|
"outputPreview",
|
|
"responseBodyHash",
|
|
"responseBodyPreview",
|
|
"error",
|
|
"errorDetails",
|
|
"failureKind",
|
|
"sdk",
|
|
"requestShape",
|
|
])),
|
|
actions: recordArray(probe.actions).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"taken",
|
|
"type",
|
|
"error",
|
|
])),
|
|
sentinelState: {
|
|
quarantined: state.quarantined,
|
|
recentAccounts: state.recentAccounts,
|
|
lastRun: state.lastRun,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function renderedCliResult(ok: boolean, command: string, renderedText: string): RenderedCliResult {
|
|
return { ok, command, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
function renderSentinelReport(
|
|
parsed: Record<string, unknown> | null,
|
|
context: { events: number; full: boolean; remote: Record<string, unknown> },
|
|
): string {
|
|
if (parsed === null) {
|
|
return [
|
|
"SUB2API SENTINEL REPORT unavailable",
|
|
`remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
|
|
stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
const metadata = isRecord(parsed.metadata) ? parsed.metadata : {};
|
|
const cronJob = isRecord(parsed.cronJob) ? parsed.cronJob : {};
|
|
const summary = isRecord(parsed.summary) ? parsed.summary : {};
|
|
const accounts = recordArray(parsed.accounts);
|
|
const runs = recordArray(parsed.runs);
|
|
const globalLedger = isRecord(parsed.globalLedger) ? parsed.globalLedger : {};
|
|
const lines: string[] = [];
|
|
lines.push([
|
|
"SUB2API SENTINEL",
|
|
`ok=${parsed.ok === true ? "true" : "false"}`,
|
|
`accounts=${summary.accountCount ?? accounts.length}`,
|
|
`quarantined=${summary.quarantinedCount ?? "?"}`,
|
|
`history=${summary.historyCount ?? runs.length}`,
|
|
`window=${formatWindow(summary.historyFrom, summary.historyTo)}`,
|
|
].join(" "));
|
|
lines.push([
|
|
"CRON",
|
|
`schedule=${cronJob.schedule ?? "-"}`,
|
|
`last=${shortIso(cronJob.lastScheduleTime)}`,
|
|
`active=${cronJob.active ?? "-"}`,
|
|
`state=${metadata.namespace ?? namespace}/${metadata.stateConfigMapName ?? "-"}`,
|
|
`ledger=req:${globalLedger.requestCount ?? 0} tok:${formatNumber(globalLedger.totalTokens)} cost:$${formatCost(globalLedger.estimatedCostUsd)}`,
|
|
].join(" "));
|
|
lines.push("");
|
|
lines.push("ACCOUNTS");
|
|
lines.push(renderTable([
|
|
["ACCOUNT", "STATE", "Q", "F_MIN", "S_MIN", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
|
...accounts.map((account) => [
|
|
stringValue(account.account) ?? "-",
|
|
stringValue(account.status) ?? "-",
|
|
account.quarantineActive === true ? "Y" : "-",
|
|
textValue(account.freezeIntervalMin),
|
|
textValue(account.successIntervalMin),
|
|
textValue(account.probeCount),
|
|
shortIso(account.lastProbeAt),
|
|
textValue(account.lastHttp),
|
|
account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-",
|
|
shorten(stringValue(account.lastFailureKind) ?? "-", 20),
|
|
shorten(stringValue(account.lastAction) ?? "-", 16),
|
|
shortIso(account.nextProbeAfter),
|
|
textValue(account.observedLastToNextMin),
|
|
]),
|
|
]));
|
|
if (runs.length > 0 || context.full) {
|
|
lines.push("");
|
|
lines.push(`RUNS last=${Math.min(context.events, runs.length)}`);
|
|
lines.push(renderTable([
|
|
["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "REASSERT"],
|
|
...runs.slice(-context.events).map((run) => [
|
|
shortIso(run.at),
|
|
textValue(run.selected),
|
|
textValue(run.due),
|
|
textValue(run.ok),
|
|
textValue(run.mismatch),
|
|
textValue(run.transportFailures),
|
|
textValue(run.actionsTaken),
|
|
textValue(run.reasserts),
|
|
]),
|
|
]));
|
|
}
|
|
lines.push("");
|
|
lines.push("LEGEND Q=quarantined M=marker matched F_MIN=freeze interval S_MIN=success interval OBS_MIN=last probe to next probe minutes TF=transport failures");
|
|
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function renderTable(rows: string[][]): string {
|
|
if (rows.length === 0) return "";
|
|
const widths: number[] = [];
|
|
for (const row of rows) {
|
|
row.forEach((cell, index) => {
|
|
widths[index] = Math.max(widths[index] ?? 0, displayWidth(cell));
|
|
});
|
|
}
|
|
return rows.map((row) => row.map((cell, index) => padRight(cell, widths[index] ?? 0)).join(" ").trimEnd()).join("\n");
|
|
}
|
|
|
|
function padRight(value: string, width: number): string {
|
|
const pad = width - displayWidth(value);
|
|
return pad <= 0 ? value : `${value}${" ".repeat(pad)}`;
|
|
}
|
|
|
|
function displayWidth(value: string): number {
|
|
return [...value].reduce((width, char) => width + (char.charCodeAt(0) > 0x7f ? 2 : 1), 0);
|
|
}
|
|
|
|
function formatWindow(from: unknown, to: unknown): string {
|
|
const left = shortIso(from);
|
|
const right = shortIso(to);
|
|
return left === "-" && right === "-" ? "-" : `${left}..${right}`;
|
|
}
|
|
|
|
function shortIso(value: unknown): string {
|
|
const text = stringValue(value);
|
|
if (text === null) return "-";
|
|
return text.replace(/^\d{4}-/u, "").replace(/:00Z$/u, "Z").replace("T", " ");
|
|
}
|
|
|
|
function textValue(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Math.round(value * 10) / 10);
|
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
return String(value);
|
|
}
|
|
|
|
function shorten(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : `${value.slice(0, Math.max(0, maxChars - 1))}…`;
|
|
}
|
|
|
|
function formatNumber(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0";
|
|
if (Math.abs(num) >= 1_000_000) return `${(num / 1_000_000).toFixed(1)}M`;
|
|
if (Math.abs(num) >= 1_000) return `${(num / 1_000).toFixed(1)}K`;
|
|
return String(Math.round(num));
|
|
}
|
|
|
|
function formatCost(value: unknown): string {
|
|
const num = numberValue(value);
|
|
if (num === null) return "0.0000";
|
|
return num.toFixed(4);
|
|
}
|
|
|
|
function codexPoolValidationSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (parsed === null) return null;
|
|
const validation = isRecord(parsed.validation) ? parsed.validation : {};
|
|
const runtimeCapabilities = isRecord(parsed.runtimeCapabilities) ? parsed.runtimeCapabilities : {};
|
|
return {
|
|
ok: parsed.ok,
|
|
degraded: parsed.degraded,
|
|
mode: parsed.mode,
|
|
namespace: parsed.namespace,
|
|
serviceDns: parsed.serviceDns,
|
|
appPod: parsed.appPod,
|
|
admin: parsed.admin,
|
|
apiKey: parsed.apiKey,
|
|
ownerBalance: parsed.ownerBalance,
|
|
ownerConcurrency: parsed.ownerConcurrency,
|
|
capacity: compactStatusBlock(parsed.capacity, ["accountName", "accountId", "expectedCapacity", "runtimeConcurrency", "priority", "status", "schedulable", "ok"]),
|
|
loadFactor: compactStatusBlock(parsed.loadFactor, ["accountName", "accountId", "expectedLoadFactor", "runtimeLoadFactor", "priority", "status", "schedulable", "ok"]),
|
|
webSocketsV2: compactStatusBlock(parsed.webSocketsV2, ["accountName", "accountId", "expectedMode", "runtimeMode", "runtimeEnabled", "status", "schedulable", "ok"]),
|
|
tempUnschedulable: compactTempUnschedulableStatus(parsed.tempUnschedulable),
|
|
sentinel: compactSentinelStatus(parsed.sentinel),
|
|
runtimeCapabilities: {
|
|
ok: runtimeCapabilities.ok,
|
|
runtimeImage: runtimeCapabilities.runtimeImage,
|
|
successBodyReclassification: compactRuntimeCapability(runtimeCapabilities.successBodyReclassification),
|
|
modelRouting400Failover: compactRuntimeCapability(runtimeCapabilities.modelRouting400Failover),
|
|
valuesPrinted: false,
|
|
},
|
|
validation: {
|
|
gatewayModels: compactGatewayModels(validation.gatewayModels),
|
|
gatewayResponses: compactGatewayResponses(validation.gatewayResponses),
|
|
gatewayResponsesRecent: compactGatewayResponsesRecent(validation.gatewayResponsesRecent),
|
|
gatewayCompactRecent: compactGatewayCompactRecent(validation.gatewayCompactRecent),
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw",
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function codexPoolSyncSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
const base = codexPoolValidationSummary(parsed);
|
|
if (parsed === null || base === null) return base;
|
|
const accounts = isRecord(parsed.accounts) ? parsed.accounts : {};
|
|
const compactAccounts = recordArray(accounts.items).map((item) => pickSummaryFields(item, [
|
|
"accountName",
|
|
"accountId",
|
|
"action",
|
|
"status",
|
|
"runtimeSchedulable",
|
|
"priority",
|
|
"capacity",
|
|
"loadFactor",
|
|
"tempUnschedulableConfigured",
|
|
"tempUnschedulableRuleCount",
|
|
"ok",
|
|
]));
|
|
return {
|
|
...base,
|
|
pool: parsed.pool,
|
|
accounts: {
|
|
desired: accounts.desired,
|
|
created: accounts.created,
|
|
updated: accounts.updated,
|
|
pruned: accounts.pruned,
|
|
pruneMode: accounts.pruneMode,
|
|
itemCount: compactAccounts.length,
|
|
attentionItems: compactAccounts.filter((item) => item.ok === false || item.runtimeSchedulable === false),
|
|
prunedItems: accounts.prunedItems,
|
|
processControl: accounts.processControl,
|
|
valuesPrinted: false,
|
|
},
|
|
disclosure: {
|
|
full: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full",
|
|
raw: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw",
|
|
pruneRemoved: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --prune-removed",
|
|
},
|
|
};
|
|
}
|
|
|
|
function poolTarget(pool = readCodexPoolConfig()): Record<string, unknown> {
|
|
return {
|
|
route: g14K3sRoute,
|
|
namespace,
|
|
service: serviceName,
|
|
serviceDns,
|
|
configPath: codexPoolConfigPath,
|
|
groupName: pool.groupName,
|
|
apiKeyName: pool.apiKeyName,
|
|
apiKeySecret: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`,
|
|
minOwnerConcurrency: pool.minOwnerConcurrency,
|
|
minOwnerConcurrencySource: pool.minOwnerConcurrencySource,
|
|
accountCapacityTotal: desiredAccountCapacityTotal(pool),
|
|
defaultAccountPriority: pool.defaultAccountPriority,
|
|
defaultAccountCapacity: pool.defaultAccountCapacity,
|
|
defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
|
|
sentinel: {
|
|
monitorEnabled: pool.sentinel.monitor.enabled,
|
|
actionsEnabled: pool.sentinel.actions.enabled,
|
|
cronJobName: pool.sentinel.cronJobName,
|
|
stateConfigMapName: pool.sentinel.stateConfigMapName,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProfileSecret[] {
|
|
return profiles
|
|
.filter((profile) => profile.ok && profile.apiKey !== null && profile.apiKey.length > 0)
|
|
.map((profile) => ({
|
|
accountName: profile.accountName,
|
|
profile: profile.profile,
|
|
baseUrl: profile.baseUrl,
|
|
apiKey: profile.apiKey ?? "",
|
|
upstreamUserAgent: profile.upstreamUserAgent,
|
|
}));
|
|
}
|
|
|
|
function publicExposureSummary(pool: CodexPoolConfig): Record<string, unknown> {
|
|
return {
|
|
enabled: pool.publicExposure.enabled,
|
|
proxyName: pool.publicExposure.proxyName,
|
|
namespace,
|
|
configMapName: pool.publicExposure.configMapName,
|
|
deploymentName: pool.publicExposure.deploymentName,
|
|
frpcImage: pool.publicExposure.frpcImage,
|
|
frps: {
|
|
serverAddr: pool.publicExposure.serverAddr,
|
|
serverPort: pool.publicExposure.serverPort,
|
|
remotePort: pool.publicExposure.remotePort,
|
|
masterConfigPath: pool.publicExposure.masterFrps.configPath,
|
|
masterContainerName: pool.publicExposure.masterFrps.containerName,
|
|
},
|
|
caddy: {
|
|
enabled: pool.publicExposure.masterCaddy.enabled,
|
|
domain: pool.publicExposure.masterCaddy.domain,
|
|
configPath: pool.publicExposure.masterCaddy.configPath,
|
|
serviceName: pool.publicExposure.masterCaddy.serviceName,
|
|
upstreamBaseUrl: pool.publicExposure.masterCaddy.upstreamBaseUrl,
|
|
responseHeaderTimeoutSeconds: pool.publicExposure.masterCaddy.responseHeaderTimeoutSeconds,
|
|
},
|
|
upstream: {
|
|
localIP: pool.publicExposure.localIP,
|
|
localPort: pool.publicExposure.localPort,
|
|
serviceDns,
|
|
},
|
|
urls: {
|
|
publicBaseUrl: pool.publicExposure.publicBaseUrl,
|
|
masterBaseUrl: pool.publicExposure.masterBaseUrl,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function applyMasterFrpsAllowPort(pool: CodexPoolConfig): Promise<Record<string, unknown>> {
|
|
const path = pool.publicExposure.masterFrps.configPath;
|
|
const port = pool.publicExposure.remotePort;
|
|
const container = pool.publicExposure.masterFrps.containerName;
|
|
if (!existsSync(path)) {
|
|
return { ok: false, error: "master-frps-config-missing", path, valuesPrinted: false };
|
|
}
|
|
const before = readFileSync(path, "utf8");
|
|
const alreadyAllowed = frpsAllowPortExists(before, port);
|
|
let backupPath: string | null = null;
|
|
let action = "kept-existing";
|
|
if (!alreadyAllowed) {
|
|
const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z");
|
|
backupPath = `${path}.bak-sub2api-${stamp}`;
|
|
copyFileSync(path, backupPath);
|
|
const next = before.replace(/\s*$/u, "") + `\n\n[[allowPorts]]\nstart = ${port}\nend = ${port}\n`;
|
|
writeFileSync(path, next, "utf8");
|
|
chmodSync(path, statSync(backupPath).mode & 0o777);
|
|
action = "added-allow-port";
|
|
}
|
|
const restart = alreadyAllowed ? null : Bun.spawnSync(["docker", "restart", container]);
|
|
const inspect = Bun.spawnSync(["docker", "inspect", "-f", "{{.State.Running}}", container]);
|
|
const ok = alreadyAllowed || (restart?.exitCode === 0 && inspect.exitCode === 0 && String(inspect.stdout).trim() === "true");
|
|
return {
|
|
ok,
|
|
action,
|
|
path,
|
|
backupPath,
|
|
remotePort: port,
|
|
containerName: container,
|
|
restart: restart === null ? null : {
|
|
exitCode: restart.exitCode,
|
|
stdoutTail: Buffer.from(restart.stdout).toString("utf8").slice(-1000),
|
|
stderrTail: Buffer.from(restart.stderr).toString("utf8").slice(-1000),
|
|
},
|
|
inspect: {
|
|
exitCode: inspect.exitCode,
|
|
running: String(inspect.stdout).trim() === "true",
|
|
stderrTail: Buffer.from(inspect.stderr).toString("utf8").slice(-1000),
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
|
|
async function applyMasterCaddySite(pool: CodexPoolConfig): Promise<Record<string, unknown>> {
|
|
const caddy = pool.publicExposure.masterCaddy;
|
|
if (!caddy.enabled) return { ok: true, action: "disabled-by-yaml", valuesPrinted: false };
|
|
const path = caddy.configPath;
|
|
if (!existsSync(path)) return { ok: false, error: "master-caddy-config-missing", path, valuesPrinted: false };
|
|
const before = readFileSync(path, "utf8");
|
|
const desiredBlock = renderCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds);
|
|
const existing = caddySiteBlock(before, caddy.domain);
|
|
const alreadyConfigured = existing === desiredBlock;
|
|
let backupPath: string | null = null;
|
|
let action = "kept-existing";
|
|
if (!alreadyConfigured) {
|
|
const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z");
|
|
backupPath = `${path}.bak-sub2api-https-${stamp}`;
|
|
copyFileSync(path, backupPath);
|
|
const next = existing === null
|
|
? `${before.replace(/\s*$/u, "")}\n\n${desiredBlock}\n`
|
|
: before.replace(existing, desiredBlock);
|
|
writeFileSync(path, next, "utf8");
|
|
chmodSync(path, statSync(backupPath).mode & 0o777);
|
|
action = existing === null ? "added-site" : "updated-site";
|
|
}
|
|
const validate = Bun.spawnSync(["caddy", "validate", "--config", path, "--adapter", "caddyfile"]);
|
|
let reload: ReturnType<typeof Bun.spawnSync> | null = null;
|
|
if (validate.exitCode === 0 && !alreadyConfigured) {
|
|
reload = Bun.spawnSync(["systemctl", "reload", caddy.serviceName]);
|
|
}
|
|
const active = Bun.spawnSync(["systemctl", "is-active", caddy.serviceName]);
|
|
const ok = validate.exitCode === 0 && (reload === null || reload.exitCode === 0) && active.exitCode === 0 && String(active.stdout).trim() === "active";
|
|
return {
|
|
ok,
|
|
action,
|
|
path,
|
|
backupPath,
|
|
domain: caddy.domain,
|
|
upstreamBaseUrl: caddy.upstreamBaseUrl,
|
|
serviceName: caddy.serviceName,
|
|
responseHeaderTimeoutSeconds: caddy.responseHeaderTimeoutSeconds,
|
|
validate: {
|
|
exitCode: validate.exitCode,
|
|
stdoutTail: Buffer.from(validate.stdout).toString("utf8").slice(-1000),
|
|
stderrTail: Buffer.from(validate.stderr).toString("utf8").slice(-2000),
|
|
},
|
|
reload: reload === null ? null : {
|
|
exitCode: reload.exitCode,
|
|
stdoutTail: Buffer.from(reload.stdout).toString("utf8").slice(-1000),
|
|
stderrTail: Buffer.from(reload.stderr).toString("utf8").slice(-1000),
|
|
},
|
|
active: {
|
|
exitCode: active.exitCode,
|
|
stdoutTail: Buffer.from(active.stdout).toString("utf8").slice(-1000),
|
|
stderrTail: Buffer.from(active.stderr).toString("utf8").slice(-1000),
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function renderCaddySiteBlock(domain: string, upstreamBaseUrl: string, responseHeaderTimeoutSeconds = 180): string {
|
|
const upstream = new URL(upstreamBaseUrl);
|
|
const upstreamHost = `${upstream.hostname}${upstream.port ? `:${upstream.port}` : ""}`;
|
|
return `${domain} {
|
|
encode zstd gzip
|
|
reverse_proxy ${upstreamHost} {
|
|
header_up Host {host}
|
|
header_up X-Real-IP {remote_host}
|
|
transport http {
|
|
dial_timeout 5s
|
|
response_header_timeout ${responseHeaderTimeoutSeconds}s
|
|
}
|
|
}
|
|
}`;
|
|
}
|
|
|
|
function caddySiteBlock(text: string, domain: string): string | null {
|
|
const startMatch = new RegExp(`(^|\\n)${escapeRegExp(domain)}\\s*\\{`, "u").exec(text);
|
|
if (startMatch === null) return null;
|
|
const start = startMatch.index + (startMatch[1] === "\n" ? 1 : 0);
|
|
const open = text.indexOf("{", start);
|
|
if (open < 0) return null;
|
|
let depth = 0;
|
|
for (let index = open; index < text.length; index += 1) {
|
|
const ch = text[index];
|
|
if (ch === "{") depth += 1;
|
|
if (ch === "}") {
|
|
depth -= 1;
|
|
if (depth === 0) return text.slice(start, index + 1);
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function frpsAllowPortExists(toml: string, port: number): boolean {
|
|
const sections = toml.split(/(?=\[\[allowPorts\]\])/u);
|
|
return sections.some((section) => {
|
|
if (!section.includes("[[allowPorts]]")) return false;
|
|
const start = section.match(/^\s*start\s*=\s*(\d+)\s*$/mu);
|
|
const end = section.match(/^\s*end\s*=\s*(\d+)\s*$/mu);
|
|
return Number(start?.[1]) === port && Number(end?.[1]) === port;
|
|
});
|
|
}
|
|
|
|
function exposeScript(pool: CodexPoolConfig): string {
|
|
const manifest = frpcManifest(pool);
|
|
const encoded = Buffer.from(manifest, "utf8").toString("base64");
|
|
return `
|
|
set -u
|
|
tmp="$(mktemp -d)"
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
manifest="$tmp/sub2api-frpc.yaml"
|
|
printf '%s' '${encoded}' | base64 -d > "$manifest"
|
|
ns_out="$tmp/ns.out"
|
|
ns_err="$tmp/ns.err"
|
|
apply_out="$tmp/apply.out"
|
|
apply_err="$tmp/apply.err"
|
|
rollout_out="$tmp/rollout.out"
|
|
rollout_err="$tmp/rollout.err"
|
|
kubectl get namespace ${namespace} >"$ns_out" 2>"$ns_err"
|
|
ns_rc=$?
|
|
apply_rc=1
|
|
rollout_rc=1
|
|
if [ "$ns_rc" -eq 0 ]; then
|
|
kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$apply_out" 2>"$apply_err"
|
|
apply_rc=$?
|
|
if [ "$apply_rc" -eq 0 ]; then
|
|
kubectl -n ${namespace} rollout status deployment/${pool.publicExposure.deploymentName} --timeout=30s >"$rollout_out" 2>"$rollout_err"
|
|
rollout_rc=$?
|
|
else
|
|
printf '%s\\n' 'skipped because apply failed' >"$rollout_err"
|
|
fi
|
|
else
|
|
: >"$apply_out"
|
|
printf '%s\\n' 'skipped because namespace is missing' >"$apply_err"
|
|
: >"$rollout_out"
|
|
printf '%s\\n' 'skipped because namespace is missing' >"$rollout_err"
|
|
fi
|
|
pods_json="$tmp/pods.json"
|
|
pods_err="$tmp/pods.err"
|
|
kubectl -n ${namespace} get pods -l app.kubernetes.io/name=${pool.publicExposure.deploymentName} -o json >"$pods_json" 2>"$pods_err"
|
|
pods_rc=$?
|
|
logs_out="$tmp/logs.out"
|
|
logs_err="$tmp/logs.err"
|
|
kubectl -n ${namespace} logs deployment/${pool.publicExposure.deploymentName} --tail=80 >"$logs_out" 2>"$logs_err"
|
|
logs_rc=$?
|
|
python3 - "$tmp" "$ns_rc" "$apply_rc" "$rollout_rc" "$pods_rc" "$logs_rc" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
tmp = sys.argv[1]
|
|
ns_rc, apply_rc, rollout_rc, pods_rc, logs_rc = [int(value) for value in sys.argv[2:]]
|
|
|
|
def text(name, limit=4000):
|
|
path = os.path.join(tmp, name)
|
|
try:
|
|
return open(path, encoding="utf-8", errors="replace").read()[-limit:]
|
|
except FileNotFoundError:
|
|
return ""
|
|
|
|
def load_json(name):
|
|
path = os.path.join(tmp, name)
|
|
try:
|
|
return json.load(open(path, encoding="utf-8"))
|
|
except Exception:
|
|
return None
|
|
|
|
pods_obj = load_json("pods.json")
|
|
pods = []
|
|
if isinstance(pods_obj, dict):
|
|
for item in pods_obj.get("items") or []:
|
|
status = item.get("status") or {}
|
|
statuses = status.get("containerStatuses") or []
|
|
pods.append({
|
|
"name": item.get("metadata", {}).get("name"),
|
|
"phase": status.get("phase"),
|
|
"ready": all(cs.get("ready") is True for cs in statuses) if statuses else False,
|
|
"restarts": sum(int(cs.get("restartCount") or 0) for cs in statuses),
|
|
})
|
|
|
|
logs = text("logs.out", 4000)
|
|
payload = {
|
|
"ok": ns_rc == 0 and apply_rc == 0 and rollout_rc == 0 and pods_rc == 0 and len(pods) > 0 and all(p["ready"] for p in pods),
|
|
"namespace": "${namespace}",
|
|
"proxy": {
|
|
"name": "${pool.publicExposure.proxyName}",
|
|
"remotePort": ${JSON.stringify(pool.publicExposure.remotePort)},
|
|
"publicBaseUrl": "${pool.publicExposure.publicBaseUrl}",
|
|
"masterBaseUrl": "${pool.publicExposure.masterBaseUrl}",
|
|
"localIP": "${pool.publicExposure.localIP}",
|
|
"localPort": ${JSON.stringify(pool.publicExposure.localPort)},
|
|
},
|
|
"resources": {
|
|
"configMap": "${pool.publicExposure.configMapName}",
|
|
"deployment": "${pool.publicExposure.deploymentName}",
|
|
"image": "${pool.publicExposure.frpcImage}",
|
|
},
|
|
"steps": {
|
|
"namespace": {"exitCode": ns_rc, "stdout": text("ns.out"), "stderr": text("ns.err")},
|
|
"apply": {"exitCode": apply_rc, "stdout": text("apply.out"), "stderr": text("apply.err")},
|
|
"rollout": {"exitCode": rollout_rc, "stdout": text("rollout.out"), "stderr": text("rollout.err")},
|
|
"pods": {"exitCode": pods_rc, "items": pods, "stderr": text("pods.err")},
|
|
"logs": {
|
|
"exitCode": logs_rc,
|
|
"startProxySuccess": "start proxy success" in logs,
|
|
"stdoutTail": logs,
|
|
"stderrTail": text("logs.err"),
|
|
},
|
|
},
|
|
}
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
sys.exit(0 if payload["ok"] else 1)
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function frpcManifest(pool: CodexPoolConfig): string {
|
|
return `apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: ${pool.publicExposure.configMapName}
|
|
namespace: ${namespace}
|
|
labels:
|
|
app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
|
|
app.kubernetes.io/part-of: platform-infra
|
|
app.kubernetes.io/managed-by: unidesk
|
|
data:
|
|
frpc.toml: |
|
|
serverAddr = "${pool.publicExposure.serverAddr}"
|
|
serverPort = ${pool.publicExposure.serverPort}
|
|
loginFailExit = true
|
|
|
|
[[proxies]]
|
|
name = "${pool.publicExposure.proxyName}"
|
|
type = "tcp"
|
|
localIP = "${pool.publicExposure.localIP}"
|
|
localPort = ${pool.publicExposure.localPort}
|
|
remotePort = ${pool.publicExposure.remotePort}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: ${pool.publicExposure.deploymentName}
|
|
namespace: ${namespace}
|
|
labels:
|
|
app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
|
|
app.kubernetes.io/part-of: platform-infra
|
|
app.kubernetes.io/managed-by: unidesk
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
|
|
app.kubernetes.io/part-of: platform-infra
|
|
app.kubernetes.io/managed-by: unidesk
|
|
spec:
|
|
containers:
|
|
- name: frpc
|
|
image: ${pool.publicExposure.frpcImage}
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- -c
|
|
- /etc/frp/frpc.toml
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/frp
|
|
readOnly: true
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: ${pool.publicExposure.configMapName}
|
|
`;
|
|
}
|
|
|
|
async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig): Promise<{ apiKey: string | null; error: string | null }> {
|
|
const result = await capture(config, g14K3sRoute, ["script"], `
|
|
set -u
|
|
kubectl -n ${namespace} get secret ${pool.apiKeySecretName} -o json
|
|
`);
|
|
if (result.exitCode !== 0) {
|
|
return { apiKey: null, error: `read pool API key secret failed: ${result.stderr.slice(-1000)}` };
|
|
}
|
|
const parsed = parseJsonOutput(result.stdout);
|
|
const data = isRecord(parsed?.data) ? parsed.data : null;
|
|
const encoded = typeof data?.[pool.apiKeySecretKey] === "string" ? data[pool.apiKeySecretKey] : null;
|
|
if (encoded === null) return { apiKey: null, error: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey} missing` };
|
|
try {
|
|
const apiKey = Buffer.from(encoded, "base64").toString("utf8");
|
|
return apiKey.length > 0 ? { apiKey, error: null } : { apiKey: null, error: "decoded API key is empty" };
|
|
} catch (error) {
|
|
return { apiKey: null, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string): Record<string, unknown> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
mkdirSync(codexDir, { recursive: true, mode: 0o700 });
|
|
const configPath = join(codexDir, "config.toml");
|
|
const authPath = join(codexDir, "auth.json");
|
|
const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`);
|
|
const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`);
|
|
if (!existsSync(configPath)) throw new Error(`${configPath} missing`);
|
|
if (!existsSync(authPath)) throw new Error(`${authPath} missing`);
|
|
const configBackupAction = copyIfMissing(configPath, backupConfigPath);
|
|
const authBackupAction = copyIfMissing(authPath, backupAuthPath);
|
|
const currentToml = readFileSync(configPath, "utf8");
|
|
const nextToml = updateCodexConfigToml(currentToml, pool);
|
|
writeFileSync(configPath, nextToml, { encoding: "utf8", mode: 0o600 });
|
|
chmodSync(configPath, 0o600);
|
|
writeFileSync(authPath, `${JSON.stringify({ OPENAI_API_KEY: apiKey }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
chmodSync(authPath, 0o600);
|
|
return {
|
|
ok: true,
|
|
codexDir,
|
|
config: {
|
|
path: configPath,
|
|
bytes: statSync(configPath).size,
|
|
backupPath: backupConfigPath,
|
|
backupAction: configBackupAction,
|
|
backupBytes: statSync(backupConfigPath).size,
|
|
},
|
|
auth: {
|
|
path: authPath,
|
|
bytes: statSync(authPath).size,
|
|
backupPath: backupAuthPath,
|
|
backupAction: authBackupAction,
|
|
backupBytes: statSync(backupAuthPath).size,
|
|
openaiApiKeyShape: "string",
|
|
openaiApiKeyBytes: Buffer.byteLength(apiKey, "utf8"),
|
|
openaiApiKeyFingerprint: fingerprint(apiKey),
|
|
valuesPrinted: false,
|
|
},
|
|
provider: {
|
|
name: pool.localCodex.providerName,
|
|
baseUrl: codexConsumerBaseUrl(pool),
|
|
wireApi: pool.localCodex.wireApi,
|
|
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
|
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function copyIfMissing(source: string, target: string): "created" | "kept-existing" {
|
|
if (existsSync(target)) return "kept-existing";
|
|
copyFileSync(source, target);
|
|
chmodSync(target, 0o600);
|
|
return "created";
|
|
}
|
|
|
|
function updateCodexConfigToml(current: string, pool: CodexPoolConfig): string {
|
|
return renderCodexLocalConsumerToml(current, {
|
|
providerName: pool.localCodex.providerName,
|
|
baseUrl: codexConsumerBaseUrl(pool),
|
|
wireApi: pool.localCodex.wireApi,
|
|
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
|
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
|
});
|
|
}
|
|
|
|
function codexConsumerBaseUrl(pool: CodexPoolConfig): string {
|
|
return `${pool.publicExposure.publicBaseUrl.replace(/\/+$/u, "")}/`;
|
|
}
|
|
|
|
export function renderCodexLocalConsumerToml(current: string, options: CodexLocalConsumerTomlOptions): string {
|
|
let next = upsertTopLevelTomlString(current, "model_provider", options.providerName);
|
|
next = upsertProviderSection(next, options);
|
|
next = upsertTomlSectionBoolean(next, "features", "responses_websockets_v2", options.responsesWebSocketsV2);
|
|
return next.endsWith("\n") ? next : `${next}\n`;
|
|
}
|
|
|
|
function upsertTopLevelTomlString(text: string, key: string, value: string): string {
|
|
const lines = text.split(/\r?\n/u);
|
|
const firstSection = lines.findIndex((line) => /^\s*\[/.test(line));
|
|
const end = firstSection === -1 ? lines.length : firstSection;
|
|
for (let index = 0; index < end; index += 1) {
|
|
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
|
|
lines[index] = `${key} = ${tomlString(value)}`;
|
|
return lines.join("\n");
|
|
}
|
|
}
|
|
lines.splice(0, 0, `${key} = ${tomlString(value)}`);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function upsertProviderSection(text: string, options: CodexLocalConsumerTomlOptions): string {
|
|
const { providerName, baseUrl, wireApi, supportsWebSockets } = options;
|
|
const header = `[model_providers.${providerName}]`;
|
|
const lines = text.split(/\r?\n/u);
|
|
const start = lines.findIndex((line) => line.trim() === header);
|
|
const canonical = [
|
|
`name = ${tomlString(providerName)}`,
|
|
`base_url = ${tomlString(baseUrl)}`,
|
|
`wire_api = ${tomlString(wireApi)}`,
|
|
"requires_openai_auth = true",
|
|
`supports_websockets = ${tomlBoolean(supportsWebSockets)}`,
|
|
];
|
|
if (start === -1) {
|
|
const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== "";
|
|
return [...lines, ...(needsBlank ? [""] : []), header, ...canonical].join("\n");
|
|
}
|
|
let end = lines.length;
|
|
for (let index = start + 1; index < lines.length; index += 1) {
|
|
if (/^\s*\[/.test(lines[index])) {
|
|
end = index;
|
|
break;
|
|
}
|
|
}
|
|
const preserved = lines.slice(start + 1, end).filter((line) => {
|
|
return !/^\s*(name|base_url|wire_api|requires_openai_auth|supports_websockets|env_key)\s*=/u.test(line);
|
|
});
|
|
const replacement = [header, ...canonical, ...preserved.filter((line) => line.trim() !== "")];
|
|
return [...lines.slice(0, start), ...replacement, ...lines.slice(end)].join("\n");
|
|
}
|
|
|
|
function upsertTomlSectionBoolean(text: string, sectionName: string, key: string, value: boolean): string {
|
|
const header = `[${sectionName}]`;
|
|
const lines = text.split(/\r?\n/u);
|
|
const start = lines.findIndex((line) => line.trim() === header);
|
|
const assignment = `${key} = ${tomlBoolean(value)}`;
|
|
if (start === -1) {
|
|
const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== "";
|
|
return [...lines, ...(needsBlank ? [""] : []), header, assignment].join("\n");
|
|
}
|
|
let end = lines.length;
|
|
for (let index = start + 1; index < lines.length; index += 1) {
|
|
if (/^\s*\[/.test(lines[index])) {
|
|
end = index;
|
|
break;
|
|
}
|
|
}
|
|
for (let index = start + 1; index < end; index += 1) {
|
|
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
|
|
lines[index] = assignment;
|
|
return lines.join("\n");
|
|
}
|
|
}
|
|
return [...lines.slice(0, start + 1), assignment, ...lines.slice(start + 1)].join("\n");
|
|
}
|
|
|
|
async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: string): Promise<Record<string, unknown>> {
|
|
const probe = await probePublicModels(pool, "with-api-key", apiKey, "public");
|
|
return {
|
|
...probe,
|
|
ok: probe.ok === true,
|
|
apiKeyFingerprint: fingerprint(apiKey),
|
|
keyPreview: apiKeyPreview(apiKey),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, base: "master" | "public" = "master"): Promise<Record<string, unknown>> {
|
|
const baseUrl = base === "public" ? pool.publicExposure.publicBaseUrl : pool.publicExposure.masterBaseUrl;
|
|
const url = `${baseUrl.replace(/\/+$/u, "")}/v1/models`;
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), 8000);
|
|
try {
|
|
const response = await fetch(url, {
|
|
method: "GET",
|
|
headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
|
|
signal: controller.signal,
|
|
});
|
|
const body = await response.text();
|
|
let parsed: unknown = null;
|
|
try {
|
|
parsed = body.trim().length > 0 ? JSON.parse(body) : null;
|
|
} catch {
|
|
parsed = null;
|
|
}
|
|
const modelCount = isRecord(parsed) && Array.isArray(parsed.data) ? parsed.data.length : null;
|
|
return {
|
|
ok: response.ok,
|
|
mode,
|
|
method: "GET /v1/models",
|
|
baseUrl,
|
|
httpStatus: response.status,
|
|
modelCount,
|
|
bodyPreview: response.ok ? "" : body.slice(0, 500),
|
|
valuesPrinted: false,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
mode,
|
|
method: "GET /v1/models",
|
|
baseUrl,
|
|
httpStatus: 0,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
valuesPrinted: false,
|
|
};
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
function apiKeyPreview(apiKey: string): string {
|
|
if (apiKey.length <= 14) return "***";
|
|
return `${apiKey.slice(0, 10)}...${apiKey.slice(-4)}`;
|
|
}
|
|
|
|
function tomlString(value: string): string {
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function tomlBoolean(value: boolean): string {
|
|
return value ? "true" : "false";
|
|
}
|
|
|
|
function escapeRegExp(value: string): string {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
}
|
|
|
|
function normalizeBaseUrl(value: string | null): string | null {
|
|
if (value === null) return null;
|
|
const trimmed = value.trim().replace(/\/+$/u, "");
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = new URL(trimmed);
|
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
return parsed.toString().replace(/\/+$/u, "");
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function stringValue(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
function numberValue(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function booleanValue(value: unknown): boolean | null {
|
|
if (typeof value === "boolean") return value;
|
|
if (typeof value === "string") {
|
|
if (value.trim() === "true") return true;
|
|
if (value.trim() === "false") return false;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function validateKubernetesName(value: string, key: string, strictDns: boolean): void {
|
|
const pattern = strictDns ? /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u : /^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u;
|
|
if (!pattern.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`);
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function fingerprint(value: string): string {
|
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
}
|
|
|
|
function syncScript(payload: unknown, pool: CodexPoolConfig): string {
|
|
const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
|
return remotePythonScript("sync", encoded, pool);
|
|
}
|
|
|
|
function validateScript(pool: CodexPoolConfig): string {
|
|
return remotePythonScript("validate", "", pool);
|
|
}
|
|
|
|
function sentinelProbeScript(payload: unknown, pool: CodexPoolConfig): string {
|
|
const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
|
|
return remotePythonScript("sentinel-probe", encoded, pool);
|
|
}
|
|
|
|
function sentinelReportScript(pool: CodexPoolConfig, events: number): string {
|
|
const stateName = pool.sentinel.stateConfigMapName;
|
|
const cronJobName = pool.sentinel.cronJobName;
|
|
return `
|
|
set -eu
|
|
python3 - <<'PY'
|
|
import json
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
|
|
NAMESPACE = ${JSON.stringify(namespace)}
|
|
STATE_NAME = ${JSON.stringify(stateName)}
|
|
CRONJOB_NAME = ${JSON.stringify(cronJobName)}
|
|
EVENT_LIMIT = ${JSON.stringify(events)}
|
|
|
|
def run(cmd):
|
|
return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
def text(data, limit=2000):
|
|
if isinstance(data, bytes):
|
|
data = data.decode("utf-8", errors="replace")
|
|
return data[-limit:]
|
|
|
|
def kube_json(args):
|
|
proc = run(["kubectl", *args, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return None, text(proc.stderr)
|
|
try:
|
|
return json.loads(proc.stdout.decode("utf-8")), None
|
|
except Exception as exc:
|
|
return None, str(exc)
|
|
|
|
def parse_iso(value):
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except Exception:
|
|
return None
|
|
|
|
def minutes_between(a, b):
|
|
left = parse_iso(a)
|
|
right = parse_iso(b)
|
|
if left is None or right is None:
|
|
return None
|
|
return round((right - left).total_seconds() / 60, 1)
|
|
|
|
def day_ledgers(state):
|
|
ledger = {}
|
|
raw = state.get("ledger")
|
|
if isinstance(raw, dict):
|
|
for item in raw.values():
|
|
if not isinstance(item, dict):
|
|
continue
|
|
add_ledger(ledger, item)
|
|
return ledger
|
|
|
|
def add_ledger(target, source):
|
|
target["inputTokens"] = target.get("inputTokens", 0) + int(source.get("inputTokens") or 0)
|
|
target["outputTokens"] = target.get("outputTokens", 0) + int(source.get("outputTokens") or 0)
|
|
target["totalTokens"] = target.get("totalTokens", 0) + int(source.get("totalTokens") or 0)
|
|
target["estimatedCostUsd"] = target.get("estimatedCostUsd", 0.0) + float(source.get("estimatedCostUsd") or 0)
|
|
target["requestCount"] = target.get("requestCount", 0) + int(source.get("requestCount") or 0)
|
|
|
|
def account_ledger(account_state):
|
|
total = {}
|
|
raw = account_state.get("daily")
|
|
if isinstance(raw, dict):
|
|
for item in raw.values():
|
|
if isinstance(item, dict):
|
|
add_ledger(total, item)
|
|
return total
|
|
|
|
def action_type(probe):
|
|
action = probe.get("action") if isinstance(probe, dict) else None
|
|
if isinstance(action, dict):
|
|
value = action.get("type")
|
|
if value:
|
|
return value
|
|
return "taken" if action.get("taken") is True else ""
|
|
return ""
|
|
|
|
def error_code(probe):
|
|
details = probe.get("errorDetails") if isinstance(probe, dict) else None
|
|
if not isinstance(details, dict):
|
|
return ""
|
|
openai_error = details.get("openaiError")
|
|
if isinstance(openai_error, dict):
|
|
return openai_error.get("code") or openai_error.get("type") or ""
|
|
return details.get("kind") or ""
|
|
|
|
def report():
|
|
cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME])
|
|
state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME])
|
|
state = {}
|
|
parse_error = None
|
|
if isinstance(state_cm, dict):
|
|
raw_state = (state_cm.get("data") or {}).get("state.json")
|
|
if isinstance(raw_state, str) and raw_state.strip():
|
|
try:
|
|
state = json.loads(raw_state)
|
|
except Exception as exc:
|
|
parse_error = str(exc)
|
|
accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
history = state.get("history") if isinstance(state.get("history"), list) else []
|
|
account_rows = []
|
|
for name, account_state in sorted(accounts.items()):
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
probe = account_state.get("lastProbe") if isinstance(account_state.get("lastProbe"), dict) else {}
|
|
quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else {}
|
|
ledger = account_ledger(account_state)
|
|
account_rows.append({
|
|
"account": name,
|
|
"status": account_state.get("lastStatus"),
|
|
"quarantineActive": quarantine.get("active") is True,
|
|
"quarantineApplied": quarantine.get("applied") if isinstance(quarantine, dict) else None,
|
|
"freezeIntervalMin": quarantine.get("intervalMinutes") if isinstance(quarantine, dict) else None,
|
|
"freezeUntil": quarantine.get("until") if isinstance(quarantine, dict) else None,
|
|
"successStreak": account_state.get("successStreak") or 0,
|
|
"successIntervalMin": account_state.get("successIntervalMinutes") or 0,
|
|
"probeCount": ledger.get("requestCount", 0),
|
|
"inputTokens": ledger.get("inputTokens", 0),
|
|
"outputTokens": ledger.get("outputTokens", 0),
|
|
"totalTokens": ledger.get("totalTokens", 0),
|
|
"estimatedCostUsd": round(float(ledger.get("estimatedCostUsd", 0)), 6),
|
|
"lastProbeAt": account_state.get("lastProbeAt"),
|
|
"lastPurpose": probe.get("purpose"),
|
|
"lastHttp": probe.get("httpStatus"),
|
|
"lastMarker": probe.get("markerMatched"),
|
|
"lastFailureKind": probe.get("failureKind"),
|
|
"lastErrorCode": error_code(probe),
|
|
"lastAction": action_type(probe),
|
|
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
|
"observedLastToNextMin": minutes_between(account_state.get("lastProbeAt"), account_state.get("nextProbeAfter")),
|
|
"requestShape": probe.get("requestShape"),
|
|
})
|
|
run_rows = []
|
|
for item in history:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
selection = item.get("selection") if isinstance(item.get("selection"), dict) else {}
|
|
run_rows.append({
|
|
"at": item.get("at"),
|
|
"selected": item.get("selected"),
|
|
"due": selection.get("due"),
|
|
"ok": item.get("okCount"),
|
|
"mismatch": item.get("mismatchCount") if item.get("mismatchCount") is not None else item.get("markerMismatchCount"),
|
|
"transportFailures": item.get("transportFailureCount"),
|
|
"actionsTaken": item.get("actionsTaken"),
|
|
"reasserts": len(item.get("reconcile") or []),
|
|
})
|
|
quarantined = [item for item in account_rows if item.get("quarantineActive") is True]
|
|
cron_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
|
|
cron_status = cronjob.get("status") if isinstance(cronjob, dict) else {}
|
|
global_ledger = day_ledgers(state)
|
|
result = {
|
|
"ok": state_error is None and parse_error is None,
|
|
"metadata": {
|
|
"namespace": NAMESPACE,
|
|
"stateConfigMapName": STATE_NAME,
|
|
"cronJobName": CRONJOB_NAME,
|
|
"generatedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"valuesPrinted": False,
|
|
},
|
|
"cronJob": {
|
|
"exists": isinstance(cronjob, dict),
|
|
"schedule": cron_spec.get("schedule") if isinstance(cron_spec, dict) else None,
|
|
"suspend": cron_spec.get("suspend") if isinstance(cron_spec, dict) else None,
|
|
"lastScheduleTime": cron_status.get("lastScheduleTime") if isinstance(cron_status, dict) else None,
|
|
"active": len(cron_status.get("active") or []) if isinstance(cron_status, dict) else None,
|
|
"error": cron_error,
|
|
},
|
|
"summary": {
|
|
"accountCount": len(account_rows),
|
|
"quarantinedCount": len(quarantined),
|
|
"historyCount": len(history),
|
|
"historyFrom": run_rows[0].get("at") if run_rows else None,
|
|
"historyTo": run_rows[-1].get("at") if run_rows else None,
|
|
"lastRun": state.get("lastRun"),
|
|
},
|
|
"globalLedger": global_ledger,
|
|
"accounts": account_rows,
|
|
"runs": run_rows[-EVENT_LIMIT:],
|
|
"errors": {
|
|
"state": state_error,
|
|
"parse": parse_error,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
return result
|
|
|
|
payload = report()
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
raise SystemExit(0 if payload.get("ok") else 1)
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function cleanupProbesScript(pool: CodexPoolConfig): string {
|
|
return remotePythonScript("cleanup-probes", "", pool);
|
|
}
|
|
|
|
function sentinelImageStatusScript(pool: CodexPoolConfig): string {
|
|
const target = codexPoolSentinelRuntimeImage(pool.sentinel);
|
|
return remoteSentinelImageScript("status", target, pool.sentinel, null);
|
|
}
|
|
|
|
function sentinelImageBuildScript(pool: CodexPoolConfig): string {
|
|
const target = codexPoolSentinelRuntimeImage(pool.sentinel);
|
|
const dockerfile = readFileSync(sentinelImageDockerfilePath, "utf8");
|
|
return remoteSentinelImageScript("build", target, pool.sentinel, dockerfile);
|
|
}
|
|
|
|
function remoteSentinelImageScript(mode: "status" | "build", target: ReturnType<typeof codexPoolSentinelRuntimeImage>, sentinel: CodexPoolSentinelConfig, dockerfile: string | null): string {
|
|
const dockerfileB64 = dockerfile === null ? "" : Buffer.from(dockerfile, "utf8").toString("base64");
|
|
return `
|
|
set -eu
|
|
mode=${shQuote(mode)}
|
|
image=${shQuote(target.runtimeImage)}
|
|
repo=${shQuote("platform-infra/sub2api-account-sentinel")}
|
|
tag=${shQuote(target.tag)}
|
|
base_image=${shQuote(target.baseImage)}
|
|
openai_version=${shQuote(sentinel.sdk.openaiPythonVersion)}
|
|
work=/tmp/unidesk-sub2api-sentinel-image
|
|
mkdir -p "$work"
|
|
dockerfile_path="$work/sentinel.Dockerfile"
|
|
registry_has_tag=false
|
|
if curl -fsS --max-time 10 "http://127.0.0.1:5000/v2/$repo/tags/list" 2>/dev/null | grep -F '"'"$tag"'"' >/dev/null 2>&1; then
|
|
registry_has_tag=true
|
|
fi
|
|
local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)"
|
|
if [ "$mode" = "status" ]; then
|
|
if [ -n "$local_id" ]; then
|
|
python_version="$(docker run --rm "$image" python3 --version 2>/dev/null || true)"
|
|
openai_runtime_version="$(docker run --rm "$image" python3 -c 'import importlib.metadata; print(importlib.metadata.version("openai"))' 2>/dev/null || true)"
|
|
else
|
|
python_version=
|
|
openai_runtime_version=
|
|
fi
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({
|
|
"ok": bool("${"${registry_has_tag}"}" == "true" or "${"${local_id}"}"),
|
|
"mode": "status",
|
|
"image": "${target.runtimeImage}",
|
|
"baseImage": "${target.baseImage}",
|
|
"tag": "${target.tag}",
|
|
"localExists": bool("${"${local_id}"}"),
|
|
"localImageId": "${"${local_id}"}" or None,
|
|
"registryHasTag": "${"${registry_has_tag}"}" == "true",
|
|
"pythonVersion": "${"${python_version}"}" or None,
|
|
"openaiPythonVersion": "${"${openai_runtime_version}"}" or None,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
exit 0
|
|
fi
|
|
if [ "$registry_has_tag" = "true" ]; then
|
|
if [ -z "$local_id" ]; then
|
|
docker pull "$image" >/dev/null 2>&1 || true
|
|
local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)"
|
|
fi
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({
|
|
"ok": True,
|
|
"mode": "reused-existing",
|
|
"image": "${target.runtimeImage}",
|
|
"baseImage": "${target.baseImage}",
|
|
"tag": "${target.tag}",
|
|
"registryHasTag": True,
|
|
"localImageId": "${"${local_id}"}" or None,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
exit 0
|
|
fi
|
|
base64 -d > "$dockerfile_path" <<'UNIDESK_SENTINEL_DOCKERFILE_B64'
|
|
${dockerfileB64}
|
|
UNIDESK_SENTINEL_DOCKERFILE_B64
|
|
export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000
|
|
export no_proxy=$NO_PROXY
|
|
docker build --pull \\
|
|
--build-arg BASE_IMAGE="$base_image" \\
|
|
--build-arg OPENAI_PYTHON_VERSION="$openai_version" \\
|
|
--build-arg HTTP_PROXY= --build-arg HTTPS_PROXY= --build-arg http_proxy= --build-arg https_proxy= \\
|
|
--build-arg NO_PROXY --build-arg no_proxy \\
|
|
-f "$dockerfile_path" \\
|
|
-t "$image" \\
|
|
"$work"
|
|
docker run --rm "$image" python3 -c 'import importlib.metadata, sys; expected=sys.argv[1]; actual=importlib.metadata.version("openai"); assert actual == expected, (actual, expected); print("openai", actual)' "$openai_version"
|
|
docker push "$image"
|
|
digest="$(docker image inspect "$image" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)"
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({
|
|
"ok": True,
|
|
"mode": "built-and-published",
|
|
"image": "${target.runtimeImage}",
|
|
"baseImage": "${target.baseImage}",
|
|
"tag": "${target.tag}",
|
|
"digest": "${"${digest}"}" or None,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function desiredAccountNames(pool: CodexPoolConfig): string[] {
|
|
return Object.keys(desiredAccountCapacityMap(pool)).sort();
|
|
}
|
|
|
|
function desiredAccountCapacityTotal(pool: CodexPoolConfig): number {
|
|
return Object.values(desiredAccountCapacityMap(pool)).reduce((total, value) => total + value, 0);
|
|
}
|
|
|
|
function desiredAccountCapacityMap(pool: CodexPoolConfig): Record<string, number> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const configs = pool.profiles.length > 0
|
|
? pool.profiles
|
|
: discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
|
|
const capacities: Record<string, number> = {};
|
|
for (const entry of configs) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
capacities[accountName] = entry.capacity ?? pool.defaultAccountCapacity;
|
|
}
|
|
return capacities;
|
|
}
|
|
|
|
function desiredAccountLoadFactorMap(pool: CodexPoolConfig): Record<string, number> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const configs = pool.profiles.length > 0
|
|
? pool.profiles
|
|
: discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
|
|
const loadFactors: Record<string, number> = {};
|
|
for (const entry of configs) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
loadFactors[accountName] = entry.loadFactor ?? pool.defaultAccountLoadFactor;
|
|
}
|
|
return loadFactors;
|
|
}
|
|
|
|
function desiredAccountWebSocketsV2ModeMap(pool: CodexPoolConfig): Record<string, OpenAIResponsesWebSocketsV2Mode | null> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const configs = pool.profiles.length > 0
|
|
? pool.profiles
|
|
: discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
|
|
const modes: Record<string, OpenAIResponsesWebSocketsV2Mode | null> = {};
|
|
for (const entry of configs) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
modes[accountName] = entry.openaiResponsesWebSocketsV2Mode;
|
|
}
|
|
return modes;
|
|
}
|
|
|
|
function desiredAccountTempUnschedulableMap(pool: CodexPoolConfig): Record<string, unknown> {
|
|
const codexDir = join(homedir(), ".codex");
|
|
const seenAccountNames = new Set<string>();
|
|
const configs = pool.profiles.length > 0
|
|
? pool.profiles
|
|
: discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
|
|
const policies: Record<string, unknown> = {};
|
|
for (const entry of configs) {
|
|
const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
|
|
seenAccountNames.add(accountName);
|
|
policies[accountName] = renderSub2ApiTempUnschedulableCredentials(entry.tempUnschedulable);
|
|
}
|
|
return policies;
|
|
}
|
|
|
|
function remotePythonScript(mode: "sync" | "validate" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig): string {
|
|
return `
|
|
set -u
|
|
python3 - <<'PY'
|
|
import base64
|
|
import json
|
|
import secrets
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime
|
|
from urllib.parse import quote
|
|
|
|
NAMESPACE = "${namespace}"
|
|
SERVICE_NAME = "${serviceName}"
|
|
SERVICE_DNS = "${serviceDns}"
|
|
FIELD_MANAGER = "${fieldManager}"
|
|
APP_SECRET_NAME = "${appSecretName}"
|
|
POOL_GROUP_NAME = "${pool.groupName}"
|
|
POOL_API_KEY_NAME = "${pool.apiKeyName}"
|
|
POOL_API_KEY_SECRET_NAME = "${pool.apiKeySecretName}"
|
|
POOL_API_KEY_SECRET_KEY = "${pool.apiKeySecretKey}"
|
|
MIN_OWNER_BALANCE_USD = ${JSON.stringify(pool.minOwnerBalanceUsd)}
|
|
MIN_OWNER_CONCURRENCY = ${JSON.stringify(pool.minOwnerConcurrency)}
|
|
MIN_OWNER_CONCURRENCY_SOURCE = ${JSON.stringify(pool.minOwnerConcurrencySource)}
|
|
POOL_DEFAULT_ACCOUNT_PRIORITY = ${JSON.stringify(pool.defaultAccountPriority)}
|
|
POOL_DEFAULT_ACCOUNT_CAPACITY = ${JSON.stringify(pool.defaultAccountCapacity)}
|
|
POOL_DEFAULT_ACCOUNT_LOAD_FACTOR = ${JSON.stringify(pool.defaultAccountLoadFactor)}
|
|
RESPONSES_SMOKE_MODEL = ${JSON.stringify(pool.localCodex.responsesSmokeModel)}
|
|
EXPECTED_ACCOUNT_CAPACITIES = ${JSON.stringify(desiredAccountCapacityMap(pool))}
|
|
EXPECTED_ACCOUNT_LOAD_FACTORS = ${JSON.stringify(desiredAccountLoadFactorMap(pool))}
|
|
EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))})
|
|
EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))})
|
|
SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))})
|
|
MODE = "${mode}"
|
|
PAYLOAD_B64 = "${encodedPayload}"
|
|
|
|
def run(cmd, input_bytes=None):
|
|
return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
def text(data, limit=4000):
|
|
if isinstance(data, bytes):
|
|
data = data.decode("utf-8", errors="replace")
|
|
return data[-limit:]
|
|
|
|
def kubectl(args, input_obj=None):
|
|
if isinstance(input_obj, str):
|
|
input_bytes = input_obj.encode("utf-8")
|
|
else:
|
|
input_bytes = input_obj
|
|
return run(["kubectl", *args], input_bytes)
|
|
|
|
def require_kubectl(args, input_obj=None, label="kubectl"):
|
|
proc = kubectl(args, input_obj)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"{label} failed: {text(proc.stderr, 1000)}")
|
|
return proc.stdout
|
|
|
|
def kube_json(args, label):
|
|
raw = require_kubectl([*args, "-o", "json"], label=label)
|
|
return json.loads(raw.decode("utf-8"))
|
|
|
|
def decode_secret_value(name, key):
|
|
data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {}
|
|
if key not in data:
|
|
return None
|
|
return base64.b64decode(data[key]).decode("utf-8")
|
|
|
|
def get_config_value(name, key):
|
|
data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {}
|
|
value = data.get(key)
|
|
return value if isinstance(value, str) and value else None
|
|
|
|
def select_app_pod():
|
|
pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or []
|
|
for pod in pods:
|
|
status = pod.get("status") or {}
|
|
if status.get("phase") != "Running":
|
|
continue
|
|
statuses = status.get("containerStatuses") or []
|
|
if statuses and all(item.get("ready") is True for item in statuses):
|
|
return pod["metadata"]["name"]
|
|
if pods:
|
|
return pods[0]["metadata"]["name"]
|
|
raise RuntimeError("sub2api app pod not found")
|
|
|
|
APP_POD = select_app_pod()
|
|
|
|
def parse_curl_output(proc):
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
marker = "\\n__HTTP_CODE__:"
|
|
pos = stdout.rfind(marker)
|
|
if pos < 0:
|
|
return {
|
|
"ok": False,
|
|
"httpStatus": 0,
|
|
"json": None,
|
|
"body": stdout,
|
|
"stderr": text(proc.stderr, 1000),
|
|
"transportExitCode": proc.returncode,
|
|
}
|
|
body = stdout[:pos]
|
|
status_text = stdout[pos + len(marker):].strip()
|
|
try:
|
|
http_status = int(status_text[-3:])
|
|
except ValueError:
|
|
http_status = 0
|
|
try:
|
|
parsed = json.loads(body) if body.strip() else None
|
|
except json.JSONDecodeError:
|
|
parsed = None
|
|
return {
|
|
"ok": proc.returncode == 0 and 200 <= http_status < 300,
|
|
"httpStatus": http_status,
|
|
"json": parsed,
|
|
"body": body,
|
|
"stderr": text(proc.stderr, 1000),
|
|
"transportExitCode": proc.returncode,
|
|
}
|
|
|
|
def curl_api(method, path, bearer=None, payload=None):
|
|
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
script = r'''
|
|
set -eu
|
|
method="$1"
|
|
url="$2"
|
|
token="\${3:-}"
|
|
tmp="$(mktemp)"
|
|
trap 'rm -f "$tmp"' EXIT
|
|
cat > "$tmp"
|
|
if [ -n "$token" ]; then
|
|
if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url"
|
|
else
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url"
|
|
fi
|
|
else
|
|
if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"
|
|
else
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
|
fi
|
|
fi
|
|
'''
|
|
proc = run([
|
|
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
|
"--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or "",
|
|
], body)
|
|
return parse_curl_output(proc)
|
|
|
|
def envelope_data(parsed):
|
|
if isinstance(parsed, dict) and "data" in parsed:
|
|
return parsed.get("data")
|
|
return parsed
|
|
|
|
def ensure_success(resp, label):
|
|
parsed = resp.get("json")
|
|
code = parsed.get("code") if isinstance(parsed, dict) else None
|
|
if not resp.get("ok") or (code is not None and code != 0):
|
|
message = parsed.get("message") if isinstance(parsed, dict) else text(resp.get("body", ""), 500)
|
|
raise RuntimeError(f"{label} failed: http={resp.get('httpStatus')} message={message}")
|
|
return envelope_data(parsed)
|
|
|
|
def ensure_admin_compliance(token):
|
|
status_resp = curl_api("GET", "/api/v1/admin/compliance", bearer=token)
|
|
if status_resp.get("httpStatus") == 404:
|
|
return {
|
|
"ok": True,
|
|
"action": "not-supported-by-runtime",
|
|
"valuesPrinted": False,
|
|
}
|
|
status = ensure_success(status_resp, "get admin compliance")
|
|
if not isinstance(status, dict):
|
|
return {
|
|
"ok": True,
|
|
"action": "status-unstructured",
|
|
"valuesPrinted": False,
|
|
}
|
|
version = status.get("version")
|
|
required = status.get("required") is True
|
|
if not required:
|
|
return {
|
|
"ok": True,
|
|
"action": "already-acknowledged",
|
|
"version": version,
|
|
"requiredBefore": False,
|
|
"requiredAfter": False,
|
|
"phrasePrinted": False,
|
|
"valuesPrinted": False,
|
|
}
|
|
phrase = status.get("ack_phrase_zh") if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else status.get("ack_phrase_en")
|
|
language = "zh" if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else "en"
|
|
if not isinstance(phrase, str) or not phrase:
|
|
raise RuntimeError("admin compliance acknowledgement phrase missing")
|
|
accepted = ensure_success(
|
|
curl_api("POST", "/api/v1/admin/compliance/accept", bearer=token, payload={"phrase": phrase, "language": language}),
|
|
"accept admin compliance",
|
|
)
|
|
required_after = accepted.get("required") if isinstance(accepted, dict) else None
|
|
return {
|
|
"ok": required_after is False,
|
|
"action": "accepted",
|
|
"version": version,
|
|
"requiredBefore": True,
|
|
"requiredAfter": required_after,
|
|
"phrasePrinted": False,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def extract_items(data):
|
|
if isinstance(data, list):
|
|
return data
|
|
if isinstance(data, dict):
|
|
if isinstance(data.get("items"), list):
|
|
return data["items"]
|
|
for key in ("groups", "accounts", "api_keys", "keys"):
|
|
if isinstance(data.get(key), list):
|
|
return data[key]
|
|
return []
|
|
|
|
def find_access_token(data):
|
|
if isinstance(data, dict):
|
|
for key in ("access_token", "token"):
|
|
if isinstance(data.get(key), str) and data[key]:
|
|
return data[key]
|
|
for value in data.values():
|
|
token = find_access_token(value)
|
|
if token:
|
|
return token
|
|
return None
|
|
|
|
def login():
|
|
admin_email = get_config_value("sub2api-config", "ADMIN_EMAIL") or "admin@sub2api.platform-infra.local"
|
|
admin_password = decode_secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD")
|
|
if not admin_password:
|
|
raise RuntimeError("ADMIN_PASSWORD missing from sub2api-secrets")
|
|
data = ensure_success(curl_api("POST", "/api/v1/auth/login", payload={"email": admin_email, "password": admin_password}), "admin login")
|
|
token = find_access_token(data)
|
|
if not token:
|
|
raise RuntimeError("admin login response did not contain access_token")
|
|
compliance = ensure_admin_compliance(token)
|
|
if compliance.get("ok") is not True:
|
|
raise RuntimeError("admin compliance acknowledgement failed")
|
|
return admin_email, token, compliance
|
|
|
|
def group_payload():
|
|
return {
|
|
"name": POOL_GROUP_NAME,
|
|
"description": "UniDesk-managed Codex API-key pool for G14 k3s internal Sub2API clients.",
|
|
"platform": "openai",
|
|
"rate_multiplier": 1,
|
|
"is_exclusive": False,
|
|
"subscription_type": "standard",
|
|
"allow_messages_dispatch": True,
|
|
"require_oauth_only": False,
|
|
"require_privacy_set": False,
|
|
"rpm_limit": 0,
|
|
}
|
|
|
|
def ensure_group(token):
|
|
existing_data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")
|
|
existing = next((item for item in extract_items(existing_data) if item.get("name") == POOL_GROUP_NAME), None)
|
|
payload = group_payload()
|
|
if existing is None:
|
|
created = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=payload), "create group")
|
|
return created, "created"
|
|
group_id = existing.get("id")
|
|
if group_id is None:
|
|
raise RuntimeError("existing group has no id")
|
|
payload["status"] = "active"
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/groups/{group_id}", bearer=token, payload=payload), "update group")
|
|
return updated if isinstance(updated, dict) else existing, "updated"
|
|
|
|
def list_accounts(token):
|
|
path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-codex-")
|
|
data = ensure_success(curl_api("GET", path, bearer=token), "list accounts")
|
|
return extract_items(data)
|
|
|
|
def list_probe_accounts(token):
|
|
path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-probe-")
|
|
data = ensure_success(curl_api("GET", path, bearer=token), "list probe accounts")
|
|
return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")]
|
|
|
|
def list_probe_groups(token):
|
|
data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")
|
|
return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")]
|
|
|
|
def cleanup_probe_resources(token):
|
|
api_key_results = []
|
|
account_results = []
|
|
group_results = []
|
|
for item in list_user_keys(token):
|
|
name = item.get("name")
|
|
key_id = item.get("id")
|
|
if not isinstance(name, str) or not name.startswith("unidesk-probe-") or key_id is None:
|
|
continue
|
|
api_key_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{key_id}", "api-key"))
|
|
for item in list_probe_accounts(token):
|
|
account_id = item.get("id")
|
|
if account_id is None:
|
|
continue
|
|
account_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account"))
|
|
for item in list_probe_groups(token):
|
|
group_id = item.get("id")
|
|
if group_id is None:
|
|
continue
|
|
group_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group"))
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in api_key_results + account_results + group_results),
|
|
"apiKeysDeleted": len(api_key_results),
|
|
"accountsDeleted": len(account_results),
|
|
"groupsDeleted": len(group_results),
|
|
"items": {
|
|
"apiKeys": api_key_results,
|
|
"accounts": account_results,
|
|
"groups": group_results,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def temp_unschedulable_credentials(profile):
|
|
credentials = profile.get("tempUnschedulableCredentials")
|
|
if not isinstance(credentials, dict):
|
|
credentials = {}
|
|
return normalize_temp_unschedulable_credentials(credentials)
|
|
|
|
def account_payload(profile, group_id):
|
|
extra = {
|
|
"openai_responses_mode": "force_responses",
|
|
"unidesk_codex_profile": profile["profile"],
|
|
"unidesk_managed": True,
|
|
}
|
|
ws_mode = profile.get("openaiResponsesWebSocketsV2Mode")
|
|
if ws_mode:
|
|
extra["openai_apikey_responses_websockets_v2_mode"] = ws_mode
|
|
extra["openai_apikey_responses_websockets_v2_enabled"] = ws_mode != "off"
|
|
credentials = {
|
|
"api_key": profile["apiKey"],
|
|
"base_url": profile["baseUrl"],
|
|
}
|
|
upstream_user_agent = profile.get("upstreamUserAgent")
|
|
if upstream_user_agent:
|
|
credentials["user_agent"] = upstream_user_agent
|
|
temp_unschedulable = temp_unschedulable_credentials(profile)
|
|
credentials["temp_unschedulable_enabled"] = temp_unschedulable["enabled"]
|
|
credentials["temp_unschedulable_rules"] = temp_unschedulable["rules"]
|
|
return {
|
|
"name": profile["accountName"],
|
|
"notes": f"UniDesk-managed Codex profile {profile['profile']} from {profile['configFile']} and {profile['authFile']}; secret source={profile['apiKeySource']}; fingerprint={profile['apiKeyFingerprint']}.",
|
|
"platform": "openai",
|
|
"type": "apikey",
|
|
"credentials": credentials,
|
|
"extra": extra,
|
|
"concurrency": int(profile.get("capacity", 5) or 5),
|
|
"priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
|
|
"rate_multiplier": 1,
|
|
"load_factor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR),
|
|
"group_ids": [group_id],
|
|
"confirm_mixed_channel_risk": True,
|
|
}
|
|
|
|
def ensure_account_schedulable(token, account_id, account_name):
|
|
data = ensure_success(
|
|
curl_api("POST", f"/api/v1/admin/accounts/{account_id}/schedulable", bearer=token, payload={"schedulable": True}),
|
|
f"set account {account_name} schedulable",
|
|
)
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
def ensure_accounts(token, profiles, group_id, prune_removed=False):
|
|
existing_accounts = list_accounts(token)
|
|
existing = {item.get("name"): item for item in existing_accounts}
|
|
desired_names = {profile["accountName"] for profile in profiles}
|
|
results = []
|
|
for profile in profiles:
|
|
payload = account_payload(profile, group_id)
|
|
current = existing.get(profile["accountName"])
|
|
if current and current.get("id") is not None:
|
|
account_id = current["id"]
|
|
update_payload = dict(payload)
|
|
update_payload.pop("platform", None)
|
|
update_payload["status"] = "active"
|
|
data = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account_id}", bearer=token, payload=update_payload), f"update account {profile['accountName']}")
|
|
action = "updated"
|
|
else:
|
|
data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}")
|
|
action = "created"
|
|
if isinstance(data, dict) and data.get("id") is not None:
|
|
schedulable_data = ensure_account_schedulable(token, data["id"], profile["accountName"])
|
|
if schedulable_data:
|
|
data = schedulable_data
|
|
results.append({
|
|
"profile": profile["profile"],
|
|
"accountName": profile["accountName"],
|
|
"accountId": data.get("id") if isinstance(data, dict) else None,
|
|
"action": action,
|
|
"baseUrl": profile["baseUrl"],
|
|
"apiKeySource": profile["apiKeySource"],
|
|
"apiKeyFingerprint": profile["apiKeyFingerprint"],
|
|
"openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"),
|
|
"priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
|
|
"capacity": int(profile.get("capacity", 5) or 5),
|
|
"loadFactor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR),
|
|
"runtimeConcurrency": data.get("concurrency") if isinstance(data, dict) else None,
|
|
"runtimeLoadFactor": data.get("load_factor") if isinstance(data, dict) else None,
|
|
"runtimeSchedulable": data.get("schedulable") if isinstance(data, dict) else None,
|
|
"tempUnschedulableConfigured": bool(payload["credentials"].get("temp_unschedulable_enabled")),
|
|
"tempUnschedulableRuleCount": len(payload["credentials"].get("temp_unschedulable_rules") or []),
|
|
"upstreamUserAgentConfigured": bool(profile.get("upstreamUserAgent")),
|
|
"valuesPrinted": False,
|
|
})
|
|
prune_results = prune_removed_accounts(token, existing_accounts, desired_names) if prune_removed else []
|
|
return results, prune_results
|
|
|
|
def prune_removed_accounts(token, existing_accounts, desired_names):
|
|
results = []
|
|
for account in existing_accounts:
|
|
name = account.get("name")
|
|
account_id = account.get("id")
|
|
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
|
if not isinstance(name, str) or not name.startswith("unidesk-codex-"):
|
|
continue
|
|
if extra.get("unidesk_managed") is not True:
|
|
continue
|
|
if name in desired_names:
|
|
continue
|
|
if account_id is None:
|
|
raise RuntimeError(f"removed account {name} has no id")
|
|
ensure_success(curl_api("DELETE", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"delete removed account {name}")
|
|
results.append({
|
|
"accountName": name,
|
|
"accountId": account_id,
|
|
"profile": extra.get("unidesk_codex_profile"),
|
|
"action": "deleted",
|
|
"reason": "removed-from-yaml",
|
|
"valuesPrinted": False,
|
|
})
|
|
return results
|
|
|
|
def generate_api_key():
|
|
alphabet = string.ascii_letters + string.digits
|
|
return "sk-unidesk-codex-" + "".join(secrets.choice(alphabet) for _ in range(48))
|
|
|
|
def ensure_api_key_secret(group_id):
|
|
existing = None
|
|
try:
|
|
existing = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY)
|
|
except Exception:
|
|
existing = None
|
|
api_key = existing if existing else generate_api_key()
|
|
secret_action = "kept-existing" if existing else "created"
|
|
manifest = {
|
|
"apiVersion": "v1",
|
|
"kind": "Secret",
|
|
"metadata": {
|
|
"name": POOL_API_KEY_SECRET_NAME,
|
|
"namespace": NAMESPACE,
|
|
"labels": {
|
|
"app.kubernetes.io/name": "sub2api",
|
|
"app.kubernetes.io/part-of": "platform-infra",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/secret-purpose": "sub2api-codex-pool-api-key",
|
|
},
|
|
},
|
|
"type": "Opaque",
|
|
"stringData": {
|
|
POOL_API_KEY_SECRET_KEY: api_key,
|
|
"GROUP_ID": str(group_id),
|
|
"GROUP_NAME": POOL_GROUP_NAME,
|
|
"SERVICE_DNS": SERVICE_DNS,
|
|
},
|
|
}
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"apply API key secret failed: {text(proc.stderr, 1000)}")
|
|
return api_key, secret_action, text(proc.stdout, 1000)
|
|
|
|
def apply_sentinel_manifest(manifest):
|
|
if not isinstance(manifest, str) or not manifest.strip():
|
|
return {
|
|
"ok": False,
|
|
"action": "missing-manifest",
|
|
"valuesPrinted": False,
|
|
}
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest)
|
|
if proc.returncode != 0:
|
|
return {
|
|
"ok": False,
|
|
"action": "apply-failed",
|
|
"stdoutTail": text(proc.stdout, 2000),
|
|
"stderrTail": text(proc.stderr, 4000),
|
|
"valuesPrinted": False,
|
|
}
|
|
status = sentinel_runtime_status()
|
|
return {
|
|
"ok": status.get("ok") is True,
|
|
"action": "applied",
|
|
"stdoutTail": text(proc.stdout, 2000),
|
|
"runtime": status,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def safe_kube_json(args, label):
|
|
proc = kubectl([*args, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)}
|
|
try:
|
|
return json.loads(proc.stdout.decode("utf-8")), None
|
|
except Exception as exc:
|
|
return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)}
|
|
|
|
def sentinel_runtime_status():
|
|
cfg = SENTINEL_CONFIG
|
|
cronjob_name = cfg.get("cronJobName")
|
|
secret_name = cfg.get("credentialsSecretName")
|
|
configmap_name = cfg.get("configMapName")
|
|
state_name = cfg.get("stateConfigMapName")
|
|
cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
|
|
secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}")
|
|
configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}")
|
|
state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
|
|
state = None
|
|
if isinstance(state_cm, dict):
|
|
raw_state = (state_cm.get("data") or {}).get("state.json")
|
|
if isinstance(raw_state, str) and raw_state:
|
|
try:
|
|
state = json.loads(raw_state)
|
|
except Exception as exc:
|
|
state = {"parseError": str(exc)}
|
|
accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {}
|
|
quarantined = []
|
|
recent_accounts = []
|
|
for name, account_state in accounts.items():
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
|
quarantined.append({
|
|
"accountName": name,
|
|
"until": quarantine.get("until"),
|
|
"applied": quarantine.get("applied"),
|
|
"reason": quarantine.get("reason"),
|
|
"failureKind": quarantine.get("failureKind"),
|
|
"errorDetails": quarantine.get("errorDetails"),
|
|
"intervalMinutes": quarantine.get("intervalMinutes"),
|
|
})
|
|
last_probe = account_state.get("lastProbe")
|
|
if isinstance(last_probe, dict):
|
|
recent_accounts.append({
|
|
"accountName": name,
|
|
"lastProbeAt": account_state.get("lastProbeAt"),
|
|
"lastStatus": account_state.get("lastStatus"),
|
|
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
|
"ok": last_probe.get("ok"),
|
|
"purpose": last_probe.get("purpose"),
|
|
"httpStatus": last_probe.get("httpStatus"),
|
|
"durationMs": last_probe.get("durationMs"),
|
|
"markerMatched": last_probe.get("markerMatched"),
|
|
"outputHash": last_probe.get("outputHash"),
|
|
"outputPreview": last_probe.get("outputPreview"),
|
|
"responseBodyHash": last_probe.get("responseBodyHash"),
|
|
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
|
"error": last_probe.get("error"),
|
|
"errorDetails": last_probe.get("errorDetails"),
|
|
"usage": last_probe.get("usage"),
|
|
"failureKind": last_probe.get("failureKind"),
|
|
"requestShape": last_probe.get("requestShape"),
|
|
"action": last_probe.get("action"),
|
|
})
|
|
recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
|
|
last_run = state.get("lastRun") if isinstance(state, dict) else None
|
|
cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
|
|
secret_data = secret.get("data") if isinstance(secret, dict) else {}
|
|
configmap_data = configmap.get("data") if isinstance(configmap, dict) else {}
|
|
ok = cronjob is not None and secret is not None and configmap is not None
|
|
return {
|
|
"ok": ok,
|
|
"desired": {
|
|
"monitorEnabled": cfg.get("monitor", {}).get("enabled"),
|
|
"actionsEnabled": cfg.get("actions", {}).get("enabled"),
|
|
"schedule": cfg.get("schedule"),
|
|
"cronJobName": cronjob_name,
|
|
"configMapName": configmap_name,
|
|
"credentialsSecretName": secret_name,
|
|
"stateConfigMapName": state_name,
|
|
},
|
|
"cronJob": {
|
|
"exists": cronjob is not None,
|
|
"schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None,
|
|
"suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None,
|
|
"lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None,
|
|
"active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None,
|
|
"error": cronjob_error,
|
|
},
|
|
"secret": {
|
|
"exists": secret is not None,
|
|
"profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data,
|
|
"valuesPrinted": False,
|
|
"error": secret_error,
|
|
},
|
|
"configMap": {
|
|
"exists": configmap is not None,
|
|
"configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data,
|
|
"runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data,
|
|
"error": configmap_error,
|
|
},
|
|
"state": {
|
|
"exists": state_cm is not None,
|
|
"accountCount": len(accounts),
|
|
"quarantinedCount": len(quarantined),
|
|
"quarantined": quarantined[-10:],
|
|
"recentAccounts": recent_accounts[-12:],
|
|
"lastRun": last_run,
|
|
"error": state_error,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def parse_epoch_z(value):
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
|
except Exception:
|
|
return None
|
|
|
|
def sentinel_state_object():
|
|
state_name = SENTINEL_CONFIG.get("stateConfigMapName")
|
|
if not state_name:
|
|
return None
|
|
obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
|
|
if not isinstance(obj, dict):
|
|
return None
|
|
raw_state = (obj.get("data") or {}).get("state.json")
|
|
if not isinstance(raw_state, str) or not raw_state:
|
|
return None
|
|
try:
|
|
return json.loads(raw_state)
|
|
except Exception:
|
|
return None
|
|
|
|
def sentinel_state_summary():
|
|
state = sentinel_state_object()
|
|
if not isinstance(state, dict):
|
|
return {"exists": False, "valuesPrinted": False}
|
|
accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
quarantined = []
|
|
recent_accounts = []
|
|
for name, account_state in accounts.items():
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
|
quarantined.append({
|
|
"accountName": name,
|
|
"until": quarantine.get("until"),
|
|
"applied": quarantine.get("applied"),
|
|
"reason": quarantine.get("reason"),
|
|
"failureKind": quarantine.get("failureKind"),
|
|
"errorDetails": quarantine.get("errorDetails"),
|
|
"intervalMinutes": quarantine.get("intervalMinutes"),
|
|
})
|
|
last_probe = account_state.get("lastProbe")
|
|
if isinstance(last_probe, dict):
|
|
recent_accounts.append({
|
|
"accountName": name,
|
|
"lastProbeAt": account_state.get("lastProbeAt"),
|
|
"lastStatus": account_state.get("lastStatus"),
|
|
"nextProbeAfter": account_state.get("nextProbeAfter"),
|
|
"ok": last_probe.get("ok"),
|
|
"purpose": last_probe.get("purpose"),
|
|
"httpStatus": last_probe.get("httpStatus"),
|
|
"durationMs": last_probe.get("durationMs"),
|
|
"markerMatched": last_probe.get("markerMatched"),
|
|
"usage": last_probe.get("usage"),
|
|
"responseBodyHash": last_probe.get("responseBodyHash"),
|
|
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
|
"error": last_probe.get("error"),
|
|
"errorDetails": last_probe.get("errorDetails"),
|
|
"failureKind": last_probe.get("failureKind"),
|
|
"requestShape": last_probe.get("requestShape"),
|
|
"action": last_probe.get("action"),
|
|
})
|
|
recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
|
|
return {
|
|
"exists": True,
|
|
"accountCount": len(accounts),
|
|
"quarantinedCount": len(quarantined),
|
|
"quarantined": quarantined[-10:],
|
|
"recentAccounts": recent_accounts[-12:],
|
|
"lastRun": state.get("lastRun"),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def reassert_sentinel_freezes_after_sync(token):
|
|
if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True:
|
|
return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False}
|
|
state = sentinel_state_object()
|
|
if not isinstance(state, dict):
|
|
return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False}
|
|
accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
now_epoch = time.time()
|
|
items = []
|
|
for name, account_state in accounts_state.items():
|
|
if not isinstance(account_state, dict):
|
|
continue
|
|
quarantine = account_state.get("quarantine")
|
|
if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
|
|
continue
|
|
until_epoch = parse_epoch_z(quarantine.get("until"))
|
|
if until_epoch is not None and until_epoch <= now_epoch:
|
|
continue
|
|
account = by_name.get(name)
|
|
if not account or account.get("id") is None:
|
|
items.append({"accountName": name, "ok": False, "reason": "account-not-found"})
|
|
continue
|
|
try:
|
|
ensure_success(
|
|
curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}),
|
|
f"reassert sentinel freeze for {name}",
|
|
)
|
|
items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")})
|
|
except Exception as exc:
|
|
items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)})
|
|
return {
|
|
"ok": all(item.get("ok") is True for item in items),
|
|
"skipped": False,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def list_user_keys(token):
|
|
data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys")
|
|
return extract_items(data)
|
|
|
|
def ensure_sub2api_api_key(token, api_key, group_id):
|
|
keys = list_user_keys(token)
|
|
existing = next((item for item in keys if item.get("key") == api_key), None)
|
|
action = "kept-existing"
|
|
if existing is None:
|
|
existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None)
|
|
if existing is None or existing.get("key") != api_key:
|
|
payload = {
|
|
"name": POOL_API_KEY_NAME,
|
|
"group_id": group_id,
|
|
"custom_key": api_key,
|
|
"quota": 0,
|
|
"rate_limit_5h": 0,
|
|
"rate_limit_1d": 0,
|
|
"rate_limit_7d": 0,
|
|
}
|
|
created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key")
|
|
existing = created if isinstance(created, dict) else existing
|
|
action = "created"
|
|
elif existing.get("id") is not None and existing.get("group_id") != group_id:
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group")
|
|
existing = updated if isinstance(updated, dict) else existing
|
|
action = "updated-group"
|
|
return {
|
|
"action": action,
|
|
"id": existing.get("id") if isinstance(existing, dict) else None,
|
|
"name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME,
|
|
"groupId": existing.get("group_id") if isinstance(existing, dict) else group_id,
|
|
"userId": existing.get("user_id") if isinstance(existing, dict) else None,
|
|
}
|
|
|
|
def get_admin_user(token, user_id):
|
|
data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner")
|
|
if not isinstance(data, dict):
|
|
raise RuntimeError("API key owner response is not an object")
|
|
return data
|
|
|
|
def ensure_pool_owner_balance(token, user_id):
|
|
user = get_admin_user(token, user_id)
|
|
current = float(user.get("balance") or 0)
|
|
if current >= MIN_OWNER_BALANCE_USD:
|
|
return {
|
|
"action": "kept-existing",
|
|
"userId": user_id,
|
|
"balanceBefore": current,
|
|
"balanceAfter": current,
|
|
"minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
|
|
}
|
|
updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={
|
|
"balance": MIN_OWNER_BALANCE_USD,
|
|
"operation": "set",
|
|
"notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.",
|
|
}), "set API key owner balance")
|
|
after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD
|
|
return {
|
|
"action": "set",
|
|
"userId": user_id,
|
|
"balanceBefore": current,
|
|
"balanceAfter": after,
|
|
"minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
|
|
}
|
|
|
|
def ensure_pool_owner_concurrency(token, user_id):
|
|
user = get_admin_user(token, user_id)
|
|
try:
|
|
current = int(user.get("concurrency") or 0)
|
|
except Exception:
|
|
current = 0
|
|
if current >= MIN_OWNER_CONCURRENCY:
|
|
return {
|
|
"ok": True,
|
|
"action": "kept-existing",
|
|
"userId": user_id,
|
|
"concurrencyBefore": current,
|
|
"concurrencyAfter": current,
|
|
"minimumConcurrency": MIN_OWNER_CONCURRENCY,
|
|
"minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
|
}
|
|
updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={
|
|
"concurrency": MIN_OWNER_CONCURRENCY,
|
|
}), "set API key owner concurrency")
|
|
try:
|
|
after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY
|
|
except Exception:
|
|
after = MIN_OWNER_CONCURRENCY
|
|
return {
|
|
"ok": after >= MIN_OWNER_CONCURRENCY,
|
|
"action": "set",
|
|
"userId": user_id,
|
|
"concurrencyBefore": current,
|
|
"concurrencyAfter": after,
|
|
"minimumConcurrency": MIN_OWNER_CONCURRENCY,
|
|
"minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
|
}
|
|
|
|
def validate_gateway(api_key):
|
|
resp = curl_api("GET", "/v1/models", bearer=api_key)
|
|
parsed = resp.get("json")
|
|
model_count = None
|
|
if isinstance(parsed, dict) and isinstance(parsed.get("data"), list):
|
|
model_count = len(parsed["data"])
|
|
return {
|
|
"ok": resp.get("ok"),
|
|
"httpStatus": resp.get("httpStatus"),
|
|
"transportExitCode": resp.get("transportExitCode"),
|
|
"modelCount": model_count,
|
|
"bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "",
|
|
"stderr": resp.get("stderr", ""),
|
|
"method": "GET /v1/models",
|
|
"serviceDns": SERVICE_DNS,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def response_output_preview(parsed):
|
|
if not isinstance(parsed, dict):
|
|
return ""
|
|
if isinstance(parsed.get("output_text"), str):
|
|
return parsed["output_text"][:240]
|
|
output = parsed.get("output")
|
|
if not isinstance(output, list):
|
|
return ""
|
|
parts = []
|
|
for item in output:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
for block in content:
|
|
if not isinstance(block, dict):
|
|
continue
|
|
text_value = block.get("text")
|
|
if isinstance(text_value, str) and text_value:
|
|
parts.append(text_value)
|
|
return "\\n".join(parts)[:240]
|
|
|
|
def request_log_evidence(request_id):
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=5m", "--tail=800"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
lines = [line for line in stdout.splitlines() if request_id in line]
|
|
failovers = []
|
|
final = None
|
|
for line in lines:
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
continue
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
continue
|
|
if "upstream_failover_switching" in line:
|
|
failovers.append({
|
|
"accountId": item.get("account_id"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
if "http request completed" in line:
|
|
final = {
|
|
"accountId": item.get("account_id"),
|
|
"statusCode": item.get("status_code"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
"path": item.get("path"),
|
|
}
|
|
return {
|
|
"requestId": request_id,
|
|
"matchedLogLineCount": len(lines),
|
|
"failovers": failovers,
|
|
"final": final,
|
|
"logsExitCode": proc.returncode,
|
|
"logsStderr": text(proc.stderr, 1000),
|
|
}
|
|
|
|
def recent_compact_gateway_evidence():
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
failures = []
|
|
successes = []
|
|
failovers = []
|
|
final_errors = []
|
|
context_canceled = []
|
|
for line in stdout.splitlines():
|
|
if "/responses/compact" not in line and "remote_compact" not in line:
|
|
continue
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
continue
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
continue
|
|
path = item.get("path")
|
|
entry = {
|
|
"requestId": item.get("request_id"),
|
|
"clientRequestId": item.get("client_request_id"),
|
|
"accountId": item.get("account_id"),
|
|
"statusCode": item.get("status_code"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
"path": path,
|
|
}
|
|
if "codex.remote_compact.failed" in line:
|
|
failures.append(entry)
|
|
elif "codex.remote_compact.succeeded" in line:
|
|
successes.append(entry)
|
|
elif "upstream_failover_switching" in line and path == "/responses/compact":
|
|
failovers.append({
|
|
**entry,
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
|
final_errors.append(entry)
|
|
if "context canceled" in line and path == "/responses/compact":
|
|
context_canceled.append(entry)
|
|
return {
|
|
"ok": True,
|
|
"degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
|
|
"window": "6h",
|
|
"tailLines": 2500,
|
|
"failureCount": len(failures),
|
|
"successCount": len(successes),
|
|
"failoverCount": len(failovers),
|
|
"finalErrorCount": len(final_errors),
|
|
"contextCanceledCount": len(context_canceled),
|
|
"recentFailures": failures[-5:],
|
|
"recentSuccesses": successes[-5:],
|
|
"recentFailovers": failovers[-8:],
|
|
"recentFinalErrors": final_errors[-5:],
|
|
"recentContextCanceled": context_canceled[-5:],
|
|
"logsExitCode": proc.returncode,
|
|
"logsStderr": text(proc.stderr, 1000),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def is_ignored_probe_noise(entry):
|
|
for value in (entry.get("requestId"), entry.get("clientRequestId")):
|
|
if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"):
|
|
return True
|
|
return False
|
|
|
|
def filter_ignored_probe_noise(items):
|
|
return [item for item in items if not is_ignored_probe_noise(item)]
|
|
|
|
def ignored_probe_noise(items, section):
|
|
result = []
|
|
for item in items:
|
|
if is_ignored_probe_noise(item):
|
|
probe = dict(item)
|
|
probe["section"] = section
|
|
result.append(probe)
|
|
return result
|
|
|
|
def recent_responses_gateway_evidence():
|
|
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
|
|
stdout = proc.stdout.decode("utf-8", errors="replace")
|
|
failovers = []
|
|
forward_failures = []
|
|
final_errors = []
|
|
context_canceled = []
|
|
slow_final_errors = []
|
|
for line in stdout.splitlines():
|
|
if '"/responses"' not in line and '"/v1/responses"' not in line:
|
|
continue
|
|
json_start = line.find("{")
|
|
if json_start < 0:
|
|
continue
|
|
try:
|
|
item = json.loads(line[json_start:])
|
|
except Exception:
|
|
continue
|
|
path = item.get("path")
|
|
if path not in ("/responses", "/v1/responses"):
|
|
continue
|
|
entry = {
|
|
"requestId": item.get("request_id"),
|
|
"clientRequestId": item.get("client_request_id"),
|
|
"accountId": item.get("account_id"),
|
|
"statusCode": item.get("status_code"),
|
|
"upstreamStatus": item.get("upstream_status"),
|
|
"latencyMs": item.get("latency_ms"),
|
|
"path": path,
|
|
}
|
|
if "upstream_failover_switching" in line:
|
|
failovers.append({
|
|
**entry,
|
|
"switchCount": item.get("switch_count"),
|
|
"maxSwitches": item.get("max_switches"),
|
|
})
|
|
elif "openai.forward_failed" in line:
|
|
forward_failures.append({
|
|
**entry,
|
|
"errorPreview": text(str(item.get("error") or ""), 500),
|
|
"fallbackErrorResponseWritten": item.get("fallback_error_response_written"),
|
|
"upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"),
|
|
})
|
|
elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
|
final_errors.append(entry)
|
|
latency_ms = item.get("latency_ms")
|
|
if isinstance(latency_ms, int) and latency_ms >= 30000:
|
|
slow_final_errors.append(entry)
|
|
if "context canceled" in line:
|
|
context_canceled.append(entry)
|
|
visible_failovers = filter_ignored_probe_noise(failovers)
|
|
visible_forward_failures = filter_ignored_probe_noise(forward_failures)
|
|
visible_final_errors = filter_ignored_probe_noise(final_errors)
|
|
visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors)
|
|
visible_context_canceled = filter_ignored_probe_noise(context_canceled)
|
|
probe_noise = (
|
|
ignored_probe_noise(failovers, "failovers")
|
|
+ ignored_probe_noise(forward_failures, "forwardFailures")
|
|
+ ignored_probe_noise(final_errors, "finalErrors")
|
|
+ ignored_probe_noise(slow_final_errors, "slowFinalErrors")
|
|
+ ignored_probe_noise(context_canceled, "contextCanceled")
|
|
)
|
|
return {
|
|
"ok": True,
|
|
"degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0,
|
|
"window": "6h",
|
|
"tailLines": 2500,
|
|
"failoverCount": len(visible_failovers),
|
|
"forwardFailureCount": len(visible_forward_failures),
|
|
"finalErrorCount": len(visible_final_errors),
|
|
"slowFinalErrorCount": len(visible_slow_final_errors),
|
|
"contextCanceledCount": len(visible_context_canceled),
|
|
"ignoredProbeNoiseCount": len(probe_noise),
|
|
"rawCounts": {
|
|
"failoverCount": len(failovers),
|
|
"forwardFailureCount": len(forward_failures),
|
|
"finalErrorCount": len(final_errors),
|
|
"slowFinalErrorCount": len(slow_final_errors),
|
|
"contextCanceledCount": len(context_canceled),
|
|
},
|
|
"recentFailovers": visible_failovers[-8:],
|
|
"recentForwardFailures": visible_forward_failures[-8:],
|
|
"recentFinalErrors": visible_final_errors[-8:],
|
|
"recentSlowFinalErrors": visible_slow_final_errors[-5:],
|
|
"recentContextCanceled": visible_context_canceled[-5:],
|
|
"recentProbeNoise": probe_noise[-5:],
|
|
"logsExitCode": proc.returncode,
|
|
"logsStderr": text(proc.stderr, 1000),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def validate_gateway_responses(api_key):
|
|
request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
|
|
payload = {
|
|
"model": RESPONSES_SMOKE_MODEL,
|
|
"input": "Reply exactly: unidesk-sub2api-validate-ok",
|
|
"stream": False,
|
|
"store": False,
|
|
"max_output_tokens": 32,
|
|
}
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
script = r'''
|
|
set -eu
|
|
token="$1"
|
|
request_id="$2"
|
|
tmp="$(mktemp)"
|
|
trap 'rm -f "$tmp"' EXIT
|
|
cat > "$tmp"
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Content-Type: application/json' \
|
|
-H "X-Request-ID: $request_id" \
|
|
-H "OpenAI-Client-Request-ID: $request_id" \
|
|
--data-binary @"$tmp" \
|
|
http://127.0.0.1:8080/v1/responses
|
|
'''
|
|
started = time.time()
|
|
proc = run([
|
|
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
|
"--", "sh", "-c", script, "sh", api_key, request_id,
|
|
], body)
|
|
resp = parse_curl_output(proc)
|
|
evidence = request_log_evidence(request_id)
|
|
parsed = resp.get("json")
|
|
failover_count = len(evidence.get("failovers") or [])
|
|
return {
|
|
"ok": resp.get("ok"),
|
|
"degraded": failover_count > 0,
|
|
"outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"),
|
|
"httpStatus": resp.get("httpStatus"),
|
|
"transportExitCode": resp.get("transportExitCode"),
|
|
"method": "POST /v1/responses",
|
|
"model": RESPONSES_SMOKE_MODEL,
|
|
"requestId": request_id,
|
|
"durationMs": int((time.time() - started) * 1000),
|
|
"outputTextPreview": response_output_preview(parsed),
|
|
"bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800),
|
|
"stderr": resp.get("stderr", ""),
|
|
"evidence": evidence,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def bool_value(value):
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
if value.lower() == "true":
|
|
return True
|
|
if value.lower() == "false":
|
|
return False
|
|
return False
|
|
|
|
def normalize_temp_unschedulable_credentials(credentials):
|
|
if not isinstance(credentials, dict):
|
|
credentials = {}
|
|
enabled = bool_value(credentials.get("temp_unschedulable_enabled"))
|
|
raw_rules = credentials.get("temp_unschedulable_rules")
|
|
if isinstance(raw_rules, str):
|
|
try:
|
|
raw_rules = json.loads(raw_rules)
|
|
except json.JSONDecodeError:
|
|
raw_rules = []
|
|
rules = []
|
|
if isinstance(raw_rules, list):
|
|
for rule in raw_rules:
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
error_code = rule.get("error_code", rule.get("statusCode"))
|
|
duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes"))
|
|
keywords = rule.get("keywords")
|
|
if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list):
|
|
continue
|
|
clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()]
|
|
if not clean_keywords:
|
|
continue
|
|
description = rule.get("description") if isinstance(rule.get("description"), str) else ""
|
|
rules.append({
|
|
"error_code": error_code,
|
|
"keywords": clean_keywords,
|
|
"duration_minutes": duration_minutes,
|
|
"description": description,
|
|
})
|
|
return {
|
|
"enabled": enabled and len(rules) > 0,
|
|
"rules": rules if enabled else [],
|
|
}
|
|
|
|
def summarize_temp_unschedulable_rules(rules):
|
|
return [{
|
|
"errorCode": rule.get("error_code"),
|
|
"durationMinutes": rule.get("duration_minutes"),
|
|
"keywordCount": len(rule.get("keywords") or []),
|
|
"keywords": rule.get("keywords") or [],
|
|
"hasDescription": bool(rule.get("description")),
|
|
} for rule in rules]
|
|
|
|
def success_body_reclassification_requirement():
|
|
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
|
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
|
for rule in expected["rules"]:
|
|
error_code = rule.get("error_code")
|
|
keywords = rule.get("keywords") or []
|
|
if isinstance(error_code, int) and 200 <= error_code < 300 and keywords:
|
|
return {
|
|
"required": True,
|
|
"sourceAccountName": name,
|
|
"statusCode": error_code,
|
|
"keywords": keywords,
|
|
"probeKeyword": keywords[0],
|
|
"durationMinutes": rule.get("duration_minutes"),
|
|
}
|
|
return {
|
|
"required": False,
|
|
"sourceAccountName": None,
|
|
"statusCode": None,
|
|
"keywords": [],
|
|
"probeKeyword": None,
|
|
"durationMinutes": None,
|
|
}
|
|
|
|
def model_routing_400_failover_requirement():
|
|
preferred = ["暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"]
|
|
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
|
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
|
for rule in expected["rules"]:
|
|
error_code = rule.get("error_code")
|
|
keywords = rule.get("keywords") or []
|
|
if error_code != 400 or not keywords:
|
|
continue
|
|
probe_keyword = next((item for item in preferred if item in keywords), keywords[0])
|
|
return {
|
|
"required": True,
|
|
"sourceAccountName": name,
|
|
"statusCode": error_code,
|
|
"keywords": keywords,
|
|
"probeKeyword": probe_keyword,
|
|
"durationMinutes": rule.get("duration_minutes"),
|
|
}
|
|
return {
|
|
"required": False,
|
|
"sourceAccountName": None,
|
|
"statusCode": None,
|
|
"keywords": [],
|
|
"probeKeyword": None,
|
|
"durationMinutes": None,
|
|
}
|
|
|
|
def delete_probe_resource(token, method, path, label):
|
|
if not path:
|
|
return {"label": label, "ok": True, "skipped": True}
|
|
resp = curl_api(method, path, bearer=token)
|
|
ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410)
|
|
return {
|
|
"label": label,
|
|
"ok": ok,
|
|
"method": method,
|
|
"path": path,
|
|
"httpStatus": resp.get("httpStatus"),
|
|
"transportExitCode": resp.get("transportExitCode"),
|
|
"bodyPreview": "" if ok else text(resp.get("body", ""), 500),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def launch_success_body_mock_upstream(status_code, body_text):
|
|
port = 28000 + secrets.randbelow(2000)
|
|
body_b64 = base64.b64encode(body_text.encode("utf-8")).decode("ascii")
|
|
script = r'''
|
|
set -eu
|
|
port="$1"
|
|
status="$2"
|
|
body_b64="$3"
|
|
body="$(printf "%s" "$body_b64" | base64 -d)"
|
|
length="$(printf "%s" "$body" | wc -c | tr -d " ")"
|
|
{ printf "HTTP/1.1 %s OK\r\nContent-Type: application/json\r\nContent-Length: %s\r\nConnection: close\r\n\r\n" "$status" "$length"; printf "%s" "$body"; } | nc -l -p "$port" -w 5
|
|
'''
|
|
proc = subprocess.Popen([
|
|
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
|
"--", "sh", "-c", script, "sh", str(port), str(status_code), body_b64,
|
|
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
time.sleep(0.35)
|
|
if proc.poll() is None:
|
|
return port, proc, None
|
|
stdout, stderr = proc.communicate(timeout=2)
|
|
return None, None, {
|
|
"ok": False,
|
|
"error": "mock-upstream-exited-before-probe",
|
|
"exitCode": proc.returncode,
|
|
"stdoutTail": text(stdout, 1000),
|
|
"stderrTail": text(stderr, 1000),
|
|
}
|
|
|
|
def finish_mock_upstream(proc):
|
|
if proc is None:
|
|
return None
|
|
timed_out = False
|
|
try:
|
|
stdout, stderr = proc.communicate(timeout=2)
|
|
except subprocess.TimeoutExpired:
|
|
timed_out = True
|
|
proc.kill()
|
|
stdout, stderr = proc.communicate(timeout=2)
|
|
return {
|
|
"exitCode": proc.returncode,
|
|
"timedOut": timed_out,
|
|
"stdoutTail": text(stdout, 1000),
|
|
"stderrTail": text(stderr, 1000),
|
|
}
|
|
|
|
def create_success_body_probe_resources(token, base_url, requirement):
|
|
stamp = str(int(time.time() * 1000))
|
|
suffix = stamp + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(6))
|
|
group_payload_obj = group_payload()
|
|
group_payload_obj["name"] = "unidesk-probe-2xx-body-" + suffix
|
|
group_payload_obj["description"] = "UniDesk validate probe for OpenAI 2xx success-body reclassification."
|
|
group_id = None
|
|
account_id = None
|
|
api_key_id = None
|
|
try:
|
|
group = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=group_payload_obj), "create 2xx success-body probe group")
|
|
group_id = group.get("id") if isinstance(group, dict) else None
|
|
if group_id is None:
|
|
raise RuntimeError("2xx success-body probe group id missing")
|
|
|
|
account_payload_obj = {
|
|
"name": "unidesk-probe-2xx-body-" + suffix,
|
|
"notes": "Temporary UniDesk validate probe account; safe to delete.",
|
|
"platform": "openai",
|
|
"type": "apikey",
|
|
"credentials": {
|
|
"api_key": "sk-unidesk-probe-upstream",
|
|
"base_url": base_url,
|
|
"temp_unschedulable_enabled": True,
|
|
"temp_unschedulable_rules": [{
|
|
"error_code": requirement["statusCode"],
|
|
"keywords": [requirement["probeKeyword"]],
|
|
"duration_minutes": 1,
|
|
"description": "UniDesk runtime capability probe for OpenAI 2xx success-body reclassification.",
|
|
}],
|
|
},
|
|
"extra": {
|
|
"openai_responses_mode": "force_responses",
|
|
"unidesk_probe": "success_body_reclassification",
|
|
},
|
|
"concurrency": 1,
|
|
"priority": 0,
|
|
"rate_multiplier": 1,
|
|
"load_factor": 1,
|
|
"group_ids": [group_id],
|
|
"confirm_mixed_channel_risk": True,
|
|
}
|
|
account = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=account_payload_obj), "create 2xx success-body probe account")
|
|
account_id = account.get("id") if isinstance(account, dict) else None
|
|
if account_id is None:
|
|
raise RuntimeError("2xx success-body probe account id missing")
|
|
|
|
api_key = "sk-unidesk-probe-" + "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(36))
|
|
api_key_obj = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload={
|
|
"name": "unidesk-probe-2xx-body-" + suffix,
|
|
"group_id": group_id,
|
|
"custom_key": api_key,
|
|
"quota": 0,
|
|
"rate_limit_5h": 0,
|
|
"rate_limit_1d": 0,
|
|
"rate_limit_7d": 0,
|
|
}), "create 2xx success-body probe API key")
|
|
api_key_id = api_key_obj.get("id") if isinstance(api_key_obj, dict) else None
|
|
return {
|
|
"groupId": group_id,
|
|
"groupName": group_payload_obj["name"],
|
|
"accountId": account_id,
|
|
"accountName": account_payload_obj["name"],
|
|
"apiKeyId": api_key_id,
|
|
"apiKey": api_key,
|
|
"keyPreview": api_key_preview(api_key),
|
|
"valuesPrinted": False,
|
|
}
|
|
except Exception:
|
|
if api_key_id is not None:
|
|
delete_probe_resource(token, "DELETE", f"/api/v1/keys/{api_key_id}", "api-key")
|
|
if account_id is not None:
|
|
delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account")
|
|
if group_id is not None:
|
|
delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group")
|
|
raise
|
|
|
|
def gateway_success_body_probe_request(api_key, request_id):
|
|
payload = {
|
|
"model": RESPONSES_SMOKE_MODEL,
|
|
"input": "Reply exactly: success-body probe",
|
|
"stream": False,
|
|
"store": False,
|
|
"max_output_tokens": 8,
|
|
}
|
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
|
script = r'''
|
|
set -eu
|
|
token="$1"
|
|
request_id="$2"
|
|
tmp="$(mktemp)"
|
|
trap 'rm -f "$tmp"' EXIT
|
|
cat > "$tmp"
|
|
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Content-Type: application/json' \
|
|
-H "X-Request-ID: $request_id" \
|
|
-H "OpenAI-Client-Request-ID: $request_id" \
|
|
--data-binary @"$tmp" \
|
|
http://127.0.0.1:8080/v1/responses
|
|
'''
|
|
proc = run([
|
|
"kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
|
|
"--", "sh", "-c", script, "sh", api_key, request_id,
|
|
], body)
|
|
return parse_curl_output(proc)
|
|
|
|
def account_temp_unschedulable_probe_state(token, account_id):
|
|
detail = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), "get temp-unschedulable probe account")
|
|
if not isinstance(detail, dict):
|
|
return {
|
|
"accountId": account_id,
|
|
"status": None,
|
|
"schedulable": None,
|
|
"tempUnschedulableUntil": None,
|
|
"tempUnschedulableReasonPreview": "",
|
|
"tempUnschedulableSet": False,
|
|
}
|
|
until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil")
|
|
reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or ""
|
|
return {
|
|
"accountId": account_id,
|
|
"status": detail.get("status"),
|
|
"schedulable": detail.get("schedulable"),
|
|
"tempUnschedulableUntil": until,
|
|
"tempUnschedulableReasonPreview": text(str(reason), 500) if reason else "",
|
|
"tempUnschedulableSet": until is not None or bool(reason),
|
|
}
|
|
|
|
def validate_success_body_reclassification(token):
|
|
requirement = success_body_reclassification_requirement()
|
|
if not requirement["required"]:
|
|
return {
|
|
"ok": True,
|
|
"required": False,
|
|
"capability": "openai-2xx-success-body-temp-unschedulable-failover",
|
|
"outcome": "not-required-by-yaml",
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
resources = None
|
|
cleanup = []
|
|
mock_proc = None
|
|
try:
|
|
keyword = requirement["probeKeyword"]
|
|
upstream_body = json.dumps({
|
|
"id": "resp_unidesk_success_body_probe",
|
|
"object": "response",
|
|
"created_at": int(time.time()),
|
|
"status": "completed",
|
|
"model": RESPONSES_SMOKE_MODEL,
|
|
"output": [{
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": keyword}],
|
|
}],
|
|
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
|
}, separators=(",", ":"))
|
|
port, mock_proc, mock_error = launch_success_body_mock_upstream(requirement["statusCode"], upstream_body)
|
|
if mock_error is not None:
|
|
return {
|
|
"ok": False,
|
|
"required": True,
|
|
"capability": "openai-2xx-success-body-temp-unschedulable-failover",
|
|
"outcome": "probe-infrastructure-failed",
|
|
"requirement": requirement,
|
|
"mock": mock_error,
|
|
"valuesPrinted": False,
|
|
}
|
|
resources = create_success_body_probe_resources(token, f"http://127.0.0.1:{port}", requirement)
|
|
request_id = "unidesk-2xx-body-probe-" + str(int(time.time() * 1000))
|
|
started = time.time()
|
|
response = gateway_success_body_probe_request(resources["apiKey"], request_id)
|
|
mock_result = finish_mock_upstream(mock_proc)
|
|
mock_proc = None
|
|
state = account_temp_unschedulable_probe_state(token, resources["accountId"])
|
|
evidence = request_log_evidence(request_id)
|
|
supported = response.get("ok") is not True and state.get("tempUnschedulableSet") is True
|
|
return {
|
|
"ok": supported,
|
|
"required": True,
|
|
"capability": "openai-2xx-success-body-temp-unschedulable-failover",
|
|
"outcome": "supported" if supported else "unsupported-runtime-image",
|
|
"requirement": requirement,
|
|
"probe": {
|
|
"requestId": request_id,
|
|
"durationMs": int((time.time() - started) * 1000),
|
|
"httpStatus": response.get("httpStatus"),
|
|
"transportExitCode": response.get("transportExitCode"),
|
|
"responseOk": response.get("ok"),
|
|
"bodyPreview": text(response.get("body", ""), 800),
|
|
"stderr": response.get("stderr", ""),
|
|
"accountState": state,
|
|
"logEvidence": evidence,
|
|
},
|
|
"mock": mock_result,
|
|
"resources": {
|
|
"groupId": resources["groupId"],
|
|
"accountId": resources["accountId"],
|
|
"apiKeyId": resources["apiKeyId"],
|
|
"keyPreview": resources["keyPreview"],
|
|
"valuesPrinted": False,
|
|
},
|
|
"cleanup": cleanup,
|
|
"message": "Sub2API image must reclassify matching 2xx OpenAI bodies into failover/temp-unschedulable before statusCode=200 YAML rules are effective." if not supported else "Matching 2xx OpenAI bodies are reclassified into account failover/temp-unschedulable.",
|
|
"valuesPrinted": False,
|
|
}
|
|
except Exception as exc:
|
|
return {
|
|
"ok": False,
|
|
"required": True,
|
|
"capability": "openai-2xx-success-body-temp-unschedulable-failover",
|
|
"outcome": "probe-failed",
|
|
"requirement": requirement,
|
|
"error": str(exc),
|
|
"cleanup": cleanup,
|
|
"valuesPrinted": False,
|
|
}
|
|
finally:
|
|
if mock_proc is not None:
|
|
_ = finish_mock_upstream(mock_proc)
|
|
if resources is not None:
|
|
cleanup.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{resources['apiKeyId']}" if resources.get("apiKeyId") is not None else "", "api-key"))
|
|
cleanup.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{resources['accountId']}", "account"))
|
|
cleanup.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{resources['groupId']}", "group"))
|
|
|
|
def validate_runtime_capabilities(token):
|
|
success_body = validate_success_body_reclassification(token)
|
|
model_routing_400 = model_routing_400_failover_requirement()
|
|
return {
|
|
"ok": success_body.get("ok") is True,
|
|
"runtimeImage": app_pod_runtime_image(),
|
|
"successBodyReclassification": success_body,
|
|
"modelRouting400Failover": {
|
|
"ok": True,
|
|
"required": model_routing_400.get("required") is True,
|
|
"capability": "openai-400-model-routing-temp-unschedulable-failover",
|
|
"outcome": "declared-by-yaml-and-checked-by-runtime-rules",
|
|
"requirement": model_routing_400,
|
|
"message": "Default validate stays short; runtime proof comes from synced rules plus real request logs/failover evidence.",
|
|
"valuesPrinted": False,
|
|
},
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def app_pod_runtime_image():
|
|
try:
|
|
pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}")
|
|
spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else []
|
|
status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else []
|
|
spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {})
|
|
status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {})
|
|
return {
|
|
"pod": APP_POD,
|
|
"image": spec.get("image"),
|
|
"imageID": status.get("imageID"),
|
|
"ready": status.get("ready"),
|
|
"restartCount": status.get("restartCount"),
|
|
}
|
|
except Exception as exc:
|
|
return {"pod": APP_POD, "error": str(exc)}
|
|
|
|
def get_account_detail(token, account):
|
|
account_id = account.get("id") if isinstance(account, dict) else None
|
|
if account_id is None:
|
|
return account
|
|
data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}")
|
|
return data if isinstance(data, dict) else account
|
|
|
|
def account_temp_unschedulable_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
enabled_names = []
|
|
for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
|
|
expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedEnabled": expected["enabled"],
|
|
"runtimeEnabled": None,
|
|
"expectedRuleCount": len(expected["rules"]),
|
|
"runtimeRuleCount": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
detail = get_account_detail(token, account)
|
|
credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {}
|
|
runtime = normalize_temp_unschedulable_credentials(credentials)
|
|
temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil")
|
|
temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or ""
|
|
ok = runtime == expected
|
|
if expected["enabled"]:
|
|
enabled_names.append(name)
|
|
if not ok:
|
|
mismatched.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedEnabled": expected["enabled"],
|
|
"runtimeEnabled": runtime["enabled"],
|
|
"expectedRuleCount": len(expected["rules"]),
|
|
"runtimeRuleCount": len(runtime["rules"]),
|
|
"expectedRules": summarize_temp_unschedulable_rules(expected["rules"]),
|
|
"runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]),
|
|
"status": account.get("status"),
|
|
"schedulable": account.get("schedulable"),
|
|
"tempUnschedulableUntil": temp_until,
|
|
"tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "",
|
|
"tempUnschedulableSet": temp_until is not None or bool(temp_reason),
|
|
"ok": ok,
|
|
})
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0,
|
|
"desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE),
|
|
"enabled": enabled_names,
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def account_capacity_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
expected_capacity_total = 0
|
|
runtime_concurrency_total = 0
|
|
schedulable_runtime_concurrency_total = 0
|
|
available_runtime_concurrency_total = 0
|
|
temp_unschedulable_runtime_concurrency_total = 0
|
|
for name in sorted(EXPECTED_ACCOUNT_CAPACITIES):
|
|
expected = int(EXPECTED_ACCOUNT_CAPACITIES[name])
|
|
expected_capacity_total += expected
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedCapacity": expected,
|
|
"runtimeConcurrency": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
runtime = account.get("concurrency")
|
|
runtime_int = runtime if isinstance(runtime, int) else 0
|
|
runtime_concurrency_total += runtime_int
|
|
if account.get("schedulable") is True:
|
|
runtime_status = account.get("status")
|
|
if runtime_status is None or runtime_status == "active":
|
|
runtime_status_ok = True
|
|
else:
|
|
runtime_status_ok = False
|
|
if runtime_status_ok:
|
|
schedulable_runtime_concurrency_total += runtime_int
|
|
temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil")
|
|
temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason")
|
|
if temp_until is None and not bool(temp_reason):
|
|
available_runtime_concurrency_total += runtime_int
|
|
else:
|
|
temp_unschedulable_runtime_concurrency_total += runtime_int
|
|
ok = runtime == expected
|
|
if not ok:
|
|
mismatched.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedCapacity": expected,
|
|
"runtimeConcurrency": runtime,
|
|
"priority": account.get("priority"),
|
|
"status": account.get("status"),
|
|
"schedulable": account.get("schedulable"),
|
|
"ok": ok,
|
|
})
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0,
|
|
"defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY,
|
|
"desired": len(EXPECTED_ACCOUNT_CAPACITIES),
|
|
"totals": {
|
|
"expectedCapacityTotal": expected_capacity_total,
|
|
"runtimeConcurrencyTotal": runtime_concurrency_total,
|
|
"schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total,
|
|
"availableRuntimeConcurrencyTotal": available_runtime_concurrency_total,
|
|
"tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total,
|
|
"minOwnerConcurrency": MIN_OWNER_CONCURRENCY,
|
|
"minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
|
|
},
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def account_load_factor_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS):
|
|
expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name])
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedLoadFactor": expected,
|
|
"runtimeLoadFactor": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
runtime_raw = account.get("load_factor")
|
|
if runtime_raw is None:
|
|
detail = get_account_detail(token, account)
|
|
runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None
|
|
try:
|
|
runtime = int(runtime_raw)
|
|
except Exception:
|
|
runtime = runtime_raw
|
|
ok = runtime == expected
|
|
if not ok:
|
|
mismatched.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedLoadFactor": expected,
|
|
"runtimeLoadFactor": runtime,
|
|
"priority": account.get("priority"),
|
|
"status": account.get("status"),
|
|
"schedulable": account.get("schedulable"),
|
|
"ok": ok,
|
|
})
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0,
|
|
"defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR,
|
|
"desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS),
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def account_ws_v2_status(token):
|
|
accounts = list_accounts(token)
|
|
by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
|
|
items = []
|
|
missing = []
|
|
mismatched = []
|
|
enabled_names = []
|
|
unschedulable_enabled = []
|
|
schedulable_enabled = []
|
|
for name in sorted(EXPECTED_ACCOUNT_WS_MODES):
|
|
expected_mode = EXPECTED_ACCOUNT_WS_MODES[name]
|
|
expected_enabled = expected_mode not in (None, "", "off")
|
|
account = by_name.get(name)
|
|
if account is None:
|
|
missing.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": None,
|
|
"expectedMode": expected_mode,
|
|
"expectedEnabled": expected_enabled,
|
|
"runtimeMode": None,
|
|
"runtimeEnabled": None,
|
|
"ok": False,
|
|
})
|
|
continue
|
|
extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
|
|
runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode")
|
|
runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled")
|
|
schedulable = account.get("schedulable")
|
|
if expected_mode == "off":
|
|
ok = runtime_mode == "off" and runtime_enabled is False
|
|
elif expected_mode is None:
|
|
ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False)
|
|
else:
|
|
ok = runtime_mode == expected_mode and runtime_enabled is True
|
|
if not ok:
|
|
mismatched.append(name)
|
|
if expected_enabled:
|
|
enabled_names.append(name)
|
|
if schedulable is True:
|
|
schedulable_enabled.append(name)
|
|
else:
|
|
unschedulable_enabled.append(name)
|
|
items.append({
|
|
"accountName": name,
|
|
"accountId": account.get("id"),
|
|
"expectedMode": expected_mode,
|
|
"expectedEnabled": expected_enabled,
|
|
"runtimeMode": runtime_mode,
|
|
"runtimeEnabled": runtime_enabled,
|
|
"status": account.get("status"),
|
|
"schedulable": schedulable,
|
|
"ok": ok and ((not expected_enabled) or schedulable is True),
|
|
})
|
|
availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0
|
|
return {
|
|
"ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok,
|
|
"desired": len(EXPECTED_ACCOUNT_WS_MODES),
|
|
"enabled": enabled_names,
|
|
"schedulableEnabled": schedulable_enabled,
|
|
"unschedulableEnabled": unschedulable_enabled,
|
|
"missing": missing,
|
|
"mismatched": mismatched,
|
|
"items": items,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def api_key_preview(api_key):
|
|
if len(api_key) <= 14:
|
|
return "***"
|
|
return api_key[:10] + "..." + api_key[-4:]
|
|
|
|
def run_sync():
|
|
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8"))
|
|
profiles = payload.get("profiles") or []
|
|
prune_removed = bool(payload.get("pruneRemoved"))
|
|
sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {}
|
|
if not profiles:
|
|
raise RuntimeError("sync payload has no profiles")
|
|
admin_email, token, admin_compliance = login()
|
|
group, group_action = ensure_group(token)
|
|
group_id = group.get("id") if isinstance(group, dict) else None
|
|
if group_id is None:
|
|
raise RuntimeError("pool group id missing after ensure")
|
|
account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed)
|
|
capacity_status = account_capacity_status(token)
|
|
load_factor_status = account_load_factor_status(token)
|
|
ws_v2_status = account_ws_v2_status(token)
|
|
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
|
api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id)
|
|
api_key_result = ensure_sub2api_api_key(token, api_key, group_id)
|
|
owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"])
|
|
owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"])
|
|
gateway = validate_gateway(api_key)
|
|
responses_smoke = validate_gateway_responses(api_key)
|
|
compact_evidence = recent_compact_gateway_evidence()
|
|
responses_evidence = recent_responses_gateway_evidence()
|
|
runtime_capabilities = validate_runtime_capabilities(token)
|
|
sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest"))
|
|
sentinel_reassert = reassert_sentinel_freezes_after_sync(token)
|
|
return {
|
|
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and sentinel.get("ok") is True and sentinel_reassert.get("ok") is True,
|
|
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
|
|
"mode": "sync",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"},
|
|
"accounts": {
|
|
"desired": len(profiles),
|
|
"created": sum(1 for item in account_results if item["action"] == "created"),
|
|
"updated": sum(1 for item in account_results if item["action"] == "updated"),
|
|
"pruned": len(pruned_account_results),
|
|
"pruneMode": "explicit" if prune_removed else "disabled-by-default",
|
|
"items": account_results,
|
|
"prunedItems": pruned_account_results,
|
|
"processControl": {"schedulableRestore": "POST /api/v1/admin/accounts/:id/schedulable", "durableConfig": False},
|
|
"valuesPrinted": False,
|
|
},
|
|
"capacity": capacity_status,
|
|
"loadFactor": load_factor_status,
|
|
"webSocketsV2": ws_v2_status,
|
|
"tempUnschedulable": temp_unschedulable_status,
|
|
"apiKey": {
|
|
"name": POOL_API_KEY_NAME,
|
|
"secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}",
|
|
"secretAction": secret_action,
|
|
"secretApply": secret_apply_stdout,
|
|
"sub2apiAction": api_key_result["action"],
|
|
"sub2apiId": api_key_result["id"],
|
|
"groupId": api_key_result["groupId"],
|
|
"userId": api_key_result["userId"],
|
|
"keyPreview": api_key_preview(api_key),
|
|
"valuesPrinted": False,
|
|
},
|
|
"ownerBalance": owner_balance,
|
|
"ownerConcurrency": owner_concurrency,
|
|
"sentinel": {**sentinel, "freezeReassert": sentinel_reassert},
|
|
"runtimeCapabilities": runtime_capabilities,
|
|
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
|
|
}
|
|
|
|
def run_validate():
|
|
api_key = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY)
|
|
if not api_key:
|
|
raise RuntimeError(f"{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY} missing")
|
|
admin_email, token, admin_compliance = login()
|
|
key_item = next((item for item in list_user_keys(token) if item.get("key") == api_key), None)
|
|
owner_balance = None
|
|
owner_concurrency = None
|
|
if key_item is not None and key_item.get("user_id") is not None:
|
|
owner_balance = ensure_pool_owner_balance(token, key_item["user_id"])
|
|
owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"])
|
|
capacity_status = account_capacity_status(token)
|
|
load_factor_status = account_load_factor_status(token)
|
|
ws_v2_status = account_ws_v2_status(token)
|
|
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
|
gateway = validate_gateway(api_key)
|
|
responses_smoke = validate_gateway_responses(api_key)
|
|
compact_evidence = recent_compact_gateway_evidence()
|
|
responses_evidence = recent_responses_gateway_evidence()
|
|
runtime_capabilities = validate_runtime_capabilities(token)
|
|
sentinel = sentinel_runtime_status()
|
|
return {
|
|
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and sentinel.get("ok") is True,
|
|
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
|
|
"mode": "validate",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"apiKey": {
|
|
"secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}",
|
|
"sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None,
|
|
"userId": key_item.get("user_id") if isinstance(key_item, dict) else None,
|
|
"keyPreview": api_key_preview(api_key),
|
|
"valuesPrinted": False,
|
|
},
|
|
"ownerBalance": owner_balance,
|
|
"ownerConcurrency": owner_concurrency,
|
|
"capacity": capacity_status,
|
|
"loadFactor": load_factor_status,
|
|
"webSocketsV2": ws_v2_status,
|
|
"tempUnschedulable": temp_unschedulable_status,
|
|
"sentinel": sentinel,
|
|
"runtimeCapabilities": runtime_capabilities,
|
|
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
|
|
}
|
|
|
|
def parse_embedded_json(stdout):
|
|
if not isinstance(stdout, str) or not stdout.strip():
|
|
return None
|
|
start = stdout.find("{")
|
|
end = stdout.rfind("}")
|
|
if start < 0 or end <= start:
|
|
return None
|
|
try:
|
|
return json.loads(stdout[start:end + 1])
|
|
except Exception:
|
|
return None
|
|
|
|
def inject_env(template, name, value):
|
|
spec = template.setdefault("spec", {})
|
|
containers = spec.setdefault("containers", [])
|
|
if not containers:
|
|
raise RuntimeError("sentinel job template has no containers")
|
|
container = containers[0]
|
|
env = container.setdefault("env", [])
|
|
env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)]
|
|
env.append({"name": name, "value": value})
|
|
|
|
def sentinel_probe_job_manifest(accounts):
|
|
cronjob_name = SENTINEL_CONFIG.get("cronJobName")
|
|
if not isinstance(cronjob_name, str) or not cronjob_name:
|
|
raise RuntimeError("sentinel cronJobName missing from config")
|
|
cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
|
|
job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {}
|
|
job_spec = copy_json(job_template.get("spec") or {})
|
|
if not isinstance(job_spec, dict) or not job_spec:
|
|
raise RuntimeError("sentinel CronJob jobTemplate.spec missing")
|
|
template = job_spec.get("template")
|
|
if not isinstance(template, dict):
|
|
raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing")
|
|
inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts))
|
|
job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600)
|
|
suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5))
|
|
job_name = ("sub2api-sentinel-probe-" + suffix)[:63]
|
|
return job_name, {
|
|
"apiVersion": "batch/v1",
|
|
"kind": "Job",
|
|
"metadata": {
|
|
"name": job_name,
|
|
"namespace": NAMESPACE,
|
|
"labels": {
|
|
"app.kubernetes.io/name": cronjob_name,
|
|
"app.kubernetes.io/part-of": "platform-infra",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
"unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe",
|
|
},
|
|
},
|
|
"spec": job_spec,
|
|
}
|
|
|
|
def copy_json(value):
|
|
return json.loads(json.dumps(value))
|
|
|
|
def job_condition(job, cond_type):
|
|
for item in ((job.get("status") or {}).get("conditions") or []):
|
|
if item.get("type") == cond_type and item.get("status") == "True":
|
|
return item
|
|
return None
|
|
|
|
def wait_sentinel_probe_job(job_name, timeout_seconds):
|
|
deadline = time.time() + timeout_seconds
|
|
latest = None
|
|
while time.time() < deadline:
|
|
latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}")
|
|
if isinstance(latest, dict):
|
|
complete = job_condition(latest, "Complete")
|
|
failed = job_condition(latest, "Failed")
|
|
if complete is not None:
|
|
return "succeeded", latest, complete
|
|
if failed is not None:
|
|
return "failed", latest, failed
|
|
time.sleep(2)
|
|
return "timeout", latest, None
|
|
|
|
def job_logs(job_name):
|
|
proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"])
|
|
return {
|
|
"exitCode": proc.returncode,
|
|
"stdout": proc.stdout.decode("utf-8", errors="replace"),
|
|
"stderr": text(proc.stderr, 4000),
|
|
}
|
|
|
|
def run_sentinel_probe():
|
|
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
|
|
accounts = payload.get("accounts") if isinstance(payload, dict) else None
|
|
if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts):
|
|
raise RuntimeError("sentinel-probe payload requires non-empty accounts")
|
|
job_name, manifest = sentinel_probe_job_manifest(accounts)
|
|
proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}")
|
|
timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240)
|
|
status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds)
|
|
logs = job_logs(job_name)
|
|
parsed = parse_embedded_json(logs.get("stdout") or "")
|
|
results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else []
|
|
requested = set(accounts)
|
|
measured = {item.get("accountName") for item in results if isinstance(item, dict)}
|
|
missing = sorted(name for name in requested if name not in measured)
|
|
marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested)
|
|
job_ok = status == "succeeded" and isinstance(parsed, dict) and parsed.get("ok") is True
|
|
return {
|
|
"ok": job_ok and marker_ok,
|
|
"jobExecutionOk": job_ok,
|
|
"markerOk": marker_ok,
|
|
"mode": "sentinel-probe",
|
|
"namespace": NAMESPACE,
|
|
"requestedAccounts": accounts,
|
|
"missingAccounts": missing,
|
|
"job": {
|
|
"name": job_name,
|
|
"status": status,
|
|
"condition": condition,
|
|
"timeoutSeconds": timeout_seconds,
|
|
"applyStdoutTail": text(proc.stdout, 1200),
|
|
"logsExitCode": logs.get("exitCode"),
|
|
"logsStderrTail": logs.get("stderr"),
|
|
},
|
|
"probe": parsed,
|
|
"sentinelState": sentinel_state_summary(),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
def run_cleanup_probes():
|
|
admin_email, token, admin_compliance = login()
|
|
cleanup = cleanup_probe_resources(token)
|
|
return {
|
|
"ok": cleanup.get("ok") is True,
|
|
"mode": "cleanup-probes",
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": APP_POD,
|
|
"admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
|
|
"cleanup": cleanup,
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
try:
|
|
if MODE == "sync":
|
|
result = run_sync()
|
|
elif MODE == "cleanup-probes":
|
|
result = run_cleanup_probes()
|
|
elif MODE == "sentinel-probe":
|
|
result = run_sentinel_probe()
|
|
else:
|
|
result = run_validate()
|
|
except Exception as exc:
|
|
result = {
|
|
"ok": False,
|
|
"mode": MODE,
|
|
"namespace": NAMESPACE,
|
|
"serviceDns": SERVICE_DNS,
|
|
"appPod": globals().get("APP_POD"),
|
|
"error": str(exc),
|
|
"valuesPrinted": False,
|
|
}
|
|
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
sys.exit(0 if result.get("ok") else 1)
|
|
PY
|
|
`;
|
|
}
|
|
|
|
async function capture(config: UniDeskConfig, target: string, args: string[], input?: string): Promise<SshCaptureResult> {
|
|
return await runSshCommandCapture(config, target, args, input);
|
|
}
|
|
|
|
type RemoteCodexPoolMode = "sync" | "validate" | "sentinel-probe" | "sentinel-image-status" | "sentinel-image-build";
|
|
|
|
async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: RemoteCodexPoolMode, script: string): Promise<SshCaptureResult> {
|
|
const jobName = `codex-pool-${mode}-${Date.now().toString(36)}`.slice(0, 63);
|
|
const startedAtMs = Date.now();
|
|
const start = await capture(config, g14K3sRoute, ["script"], remoteJobStartScript(jobName, script));
|
|
const started = parseJsonOutput(start.stdout);
|
|
if (start.exitCode !== 0 || boolField(started, "ok", false) !== true) return start;
|
|
|
|
let latest: RemoteCodexPoolJobStatus | null = null;
|
|
while (Date.now() - startedAtMs <= remoteJobTimeoutMs) {
|
|
await sleep(remoteJobPollMs);
|
|
const probe = await capture(config, g14K3sRoute, ["script"], remoteJobStatusScript(jobName));
|
|
const parsed = parseJsonOutput(probe.stdout);
|
|
latest = normalizeRemoteJobStatus(parsed);
|
|
process.stderr.write(`${JSON.stringify({
|
|
event: "platform-infra.sub2api.codex-pool.progress",
|
|
at: new Date().toISOString(),
|
|
mode,
|
|
jobName,
|
|
status: latest?.status ?? "unknown",
|
|
exitCode: latest?.exitCode ?? null,
|
|
elapsedMs: Date.now() - startedAtMs,
|
|
stdoutBytes: latest?.stdoutBytes ?? null,
|
|
stderrBytes: latest?.stderrBytes ?? null,
|
|
})}\n`);
|
|
if (probe.exitCode !== 0) {
|
|
return {
|
|
exitCode: probe.exitCode,
|
|
stdout: probe.stdout,
|
|
stderr: `${start.stderr}${probe.stderr}`,
|
|
};
|
|
}
|
|
if (latest?.status === "succeeded" || latest?.status === "failed") {
|
|
return {
|
|
exitCode: latest.exitCode ?? (latest.status === "succeeded" ? 0 : 1),
|
|
stdout: latest.stdout ?? "",
|
|
stderr: latest.stderr ?? "",
|
|
};
|
|
}
|
|
}
|
|
|
|
return {
|
|
exitCode: 124,
|
|
stdout: latest?.stdout ?? "",
|
|
stderr: [
|
|
latest?.stderr ?? "",
|
|
`remote codex-pool ${mode} job ${jobName} did not finish within ${remoteJobTimeoutMs}ms`,
|
|
`status command: ${codexPoolModeCommand(mode)}`,
|
|
].filter(Boolean).join("\n"),
|
|
};
|
|
}
|
|
|
|
function codexPoolModeCommand(mode: RemoteCodexPoolMode): string {
|
|
if (mode === "sync") return "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm";
|
|
if (mode === "sentinel-probe") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account <accountName> --confirm";
|
|
return "bun scripts/cli.ts platform-infra sub2api codex-pool validate";
|
|
}
|
|
|
|
interface RemoteCodexPoolJobStatus {
|
|
status: "running" | "succeeded" | "failed" | "unknown";
|
|
exitCode: number | null;
|
|
stdout: string | null;
|
|
stderr: string | null;
|
|
stdoutBytes: number | null;
|
|
stderrBytes: number | null;
|
|
}
|
|
|
|
function normalizeRemoteJobStatus(parsed: Record<string, unknown> | null): RemoteCodexPoolJobStatus | null {
|
|
if (parsed === null) return null;
|
|
const status = parsed.status === "running" || parsed.status === "succeeded" || parsed.status === "failed" || parsed.status === "unknown"
|
|
? parsed.status
|
|
: "unknown";
|
|
return {
|
|
status,
|
|
exitCode: typeof parsed.exitCode === "number" ? parsed.exitCode : null,
|
|
stdout: typeof parsed.stdout === "string" ? parsed.stdout : null,
|
|
stderr: typeof parsed.stderr === "string" ? parsed.stderr : null,
|
|
stdoutBytes: typeof parsed.stdoutBytes === "number" ? parsed.stdoutBytes : null,
|
|
stderrBytes: typeof parsed.stderrBytes === "number" ? parsed.stderrBytes : null,
|
|
};
|
|
}
|
|
|
|
function remoteJobStartScript(jobName: string, script: string): string {
|
|
const scriptB64 = Buffer.from(script, "utf8").toString("base64");
|
|
return `
|
|
set -eu
|
|
job=${shQuote(jobName)}
|
|
dir=${shQuote(remoteJobDir)}
|
|
mkdir -p "$dir"
|
|
chmod 700 "$dir"
|
|
script_path="$dir/$job.sh"
|
|
stdout_path="$dir/$job.stdout"
|
|
stderr_path="$dir/$job.stderr"
|
|
exit_path="$dir/$job.exit"
|
|
done_path="$dir/$job.done"
|
|
pid_path="$dir/$job.pid"
|
|
rm -f "$script_path" "$stdout_path" "$stderr_path" "$exit_path" "$done_path" "$pid_path"
|
|
base64 -d > "$script_path" <<'UNIDESK_REMOTE_SCRIPT_B64'
|
|
${scriptB64}
|
|
UNIDESK_REMOTE_SCRIPT_B64
|
|
chmod 700 "$script_path"
|
|
(
|
|
set +e
|
|
trap '' HUP
|
|
sh "$script_path" > "$stdout_path" 2> "$stderr_path"
|
|
code=$?
|
|
printf '%s' "$code" > "$exit_path"
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$done_path"
|
|
rm -f "$script_path"
|
|
) >/dev/null 2>&1 &
|
|
pid=$!
|
|
printf '%s' "$pid" > "$pid_path"
|
|
python3 - <<PY
|
|
import json
|
|
print(json.dumps({"ok": True, "jobName": ${JSON.stringify(jobName)}, "pid": int("${"${pid}"}"), "dir": ${JSON.stringify(remoteJobDir)}}))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function remoteJobStatusScript(jobName: string): string {
|
|
return `
|
|
set -eu
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import signal
|
|
|
|
job = ${JSON.stringify(jobName)}
|
|
directory = ${JSON.stringify(remoteJobDir)}
|
|
stdout_path = os.path.join(directory, job + ".stdout")
|
|
stderr_path = os.path.join(directory, job + ".stderr")
|
|
exit_path = os.path.join(directory, job + ".exit")
|
|
done_path = os.path.join(directory, job + ".done")
|
|
pid_path = os.path.join(directory, job + ".pid")
|
|
|
|
def read_text(path, limit=None):
|
|
try:
|
|
with open(path, "rb") as handle:
|
|
data = handle.read()
|
|
except FileNotFoundError:
|
|
return None
|
|
if limit is not None and len(data) > limit:
|
|
data = data[-limit:]
|
|
return data.decode("utf-8", errors="replace")
|
|
|
|
def file_size(path):
|
|
try:
|
|
return os.path.getsize(path)
|
|
except FileNotFoundError:
|
|
return 0
|
|
|
|
pid_text = read_text(pid_path)
|
|
done = os.path.exists(done_path)
|
|
exit_text = read_text(exit_path)
|
|
exit_code = None
|
|
if exit_text is not None:
|
|
try:
|
|
exit_code = int(exit_text.strip())
|
|
except ValueError:
|
|
exit_code = None
|
|
running = False
|
|
if pid_text is not None and not done:
|
|
try:
|
|
os.kill(int(pid_text.strip()), 0)
|
|
running = True
|
|
except Exception:
|
|
running = False
|
|
status = "succeeded" if done and exit_code == 0 else "failed" if done else "running" if running else "unknown"
|
|
print(json.dumps({
|
|
"ok": True,
|
|
"jobName": job,
|
|
"status": status,
|
|
"pid": int(pid_text.strip()) if pid_text and pid_text.strip().isdigit() else None,
|
|
"exitCode": exit_code,
|
|
"stdoutBytes": file_size(stdout_path),
|
|
"stderrBytes": file_size(stderr_path),
|
|
"stdout": read_text(stdout_path, 500000) if done else read_text(stdout_path, 12000),
|
|
"stderr": read_text(stderr_path, 120000) if done else read_text(stderr_path, 12000),
|
|
"doneAt": read_text(done_path),
|
|
}, ensure_ascii=False))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
function shQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'\\''`)}'`;
|
|
}
|
|
|
|
function sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function parseJsonOutput(stdout: string): Record<string, unknown> | null {
|
|
const trimmed = stdout.trim();
|
|
if (trimmed.length === 0) return null;
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start === -1 || end === -1 || end <= start) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function boolField(value: Record<string, unknown> | null, key: string, defaultValue: boolean): boolean {
|
|
if (value === null) return defaultValue;
|
|
const field = value[key];
|
|
return typeof field === "boolean" ? field : defaultValue;
|
|
}
|
|
|
|
function compactCapture(result: SshCaptureResult, options: { full?: boolean } = {}): Record<string, unknown> {
|
|
const full = options.full ?? false;
|
|
return {
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
|
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
|
stdoutTail: full || result.exitCode !== 0 ? result.stdout.slice(-8000) : "",
|
|
stderrTail: full || result.exitCode !== 0 ? result.stderr.slice(-4000) : "",
|
|
};
|
|
}
|