import { createHash } from "node:crypto";
import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import type { UniDeskConfig } from "./config";
import { rootPath } from "./config";
import type { RenderedCliResult } from "./output";
import {
  codexPoolSentinelSummary,
  codexPoolSentinelRuntimeImage,
  defaultCodexPoolSentinelConfig,
  readCodexPoolSentinelConfig,
  renderCodexPoolSentinelManifest,
  type CodexPoolSentinelConfig,
  type CodexPoolSentinelProfileSecret,
} from "./platform-infra-sub2api-codex-sentinel";
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";

const g14K3sRoute = "G14:k3s";
const namespace = "platform-infra";
const serviceName = "sub2api";
const serviceDns = `${serviceName}.${namespace}.svc.cluster.local:8080`;
const fieldManager = "unidesk-platform-infra";
const appSecretName = "sub2api-secrets";
const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml");
const sentinelImageDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile");
const defaultPoolGroupName = "unidesk-codex-pool";
const defaultPoolApiKeyName = "unidesk-codex-pool-api-key";
const defaultPoolApiKeySecretName = "sub2api-codex-pool-api-key";
const defaultPoolApiKeySecretKey = "API_KEY";
const defaultMinOwnerBalanceUsd = 1000;
const defaultAccountPriority = 10;
const remoteJobDir = "/tmp/unidesk-platform-infra-sub2api-codex-pool";
const remoteJobTimeoutMs = 15 * 60_000;
const remoteJobPollMs = 5_000;

interface DisclosureOptions {
  full: boolean;
  raw: boolean;
}

interface SyncOptions extends DisclosureOptions {
  confirm: boolean;
  pruneRemoved: boolean;
}

interface ConfirmOptions extends DisclosureOptions {
  confirm: boolean;
}

interface SentinelProbeOptions extends ConfirmOptions {
  accounts: string[];
}

interface SentinelReportOptions extends DisclosureOptions {
  events: number;
}

interface SentinelImageOptions extends DisclosureOptions {
  action: "status" | "build";
  confirm: boolean;
  dryRun: boolean;
}

interface CodexProfile {
  profile: string;
  accountName: string;
  configFile: string;
  authFile: string;
  provider: string;
  baseUrl: string;
  wireApi: string | null;
  model: string | null;
  envKey: string | null;
  apiKey: string | null;
  apiKeySource: "auth-json" | "env" | null;
  openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
  upstreamUserAgent: string | null;
  priority: number;
  capacity: number;
  loadFactor: number;
  tempUnschedulable: CodexTempUnschedulablePolicy;
  authOpenAIKeyShape: string;
  ok: boolean;
  error: string | null;
}

type OpenAIResponsesWebSocketsV2Mode = "off" | "ctx_pool" | "passthrough";

export interface CodexTempUnschedulableRule {
  statusCode: number;
  keywords: string[];
  durationMinutes: number;
  description: string | null;
}

export interface CodexTempUnschedulablePolicy {
  enabled: boolean;
  rules: CodexTempUnschedulableRule[];
}

interface CodexPoolConfig {
  groupName: string;
  apiKeyName: string;
  apiKeySecretName: string;
  apiKeySecretKey: string;
  minOwnerBalanceUsd: number;
  minOwnerConcurrency: number;
  minOwnerConcurrencySource: "auto" | "yaml";
  defaultAccountPriority: number;
  defaultAccountCapacity: number;
  defaultAccountLoadFactor: number;
  defaultTempUnschedulable: CodexTempUnschedulablePolicy;
  profiles: CodexPoolProfileConfig[];
  publicExposure: CodexPoolPublicExposureConfig;
  localCodex: CodexPoolLocalCodexConfig;
  sentinel: CodexPoolSentinelConfig;
}

interface CodexPoolProfileConfig {
  profile: string;
  accountName: string | null;
  configFile: string;
  authFile: string;
  fallbackConfigFile: string | null;
  fallbackAuthFile: string | null;
  openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
  upstreamUserAgent: string | null;
  priority: number;
  capacity: number | null;
  loadFactor: number | null;
  tempUnschedulable: CodexTempUnschedulablePolicy;
}

interface CodexPoolPublicExposureConfig {
  enabled: boolean;
  proxyName: string;
  configMapName: string;
  deploymentName: string;
  frpcImage: string;
  serverAddr: string;
  serverPort: number;
  remotePort: number;
  localIP: string;
  localPort: number;
  publicBaseUrl: string;
  masterBaseUrl: string;
  masterFrps: {
    configPath: string;
    containerName: string;
  };
  masterCaddy: {
    enabled: boolean;
    domain: string;
    configPath: string;
    serviceName: string;
    upstreamBaseUrl: string;
    responseHeaderTimeoutSeconds: number;
  };
}

interface CodexPoolLocalCodexConfig {
  backupSuffix: string;
  providerName: string;
  wireApi: string;
  supportsWebSockets: boolean;
  responsesWebSocketsV2: boolean;
  responsesSmokeModel: string;
}

interface CodexLocalConsumerTomlOptions {
  providerName: string;
  baseUrl: string;
  wireApi: string;
  supportsWebSockets: boolean;
  responsesWebSocketsV2: boolean;
}

export function codexPoolHelp(): unknown {
  const pool = readCodexPoolConfig();
  return {
    command: "platform-infra sub2api codex-pool plan|sync|validate|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
    output: "json, except sentinel-report defaults to a ps-like text table",
    usage: [
      "bun scripts/cli.ts platform-infra sub2api codex-pool plan",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm [--prune-removed]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--full|--raw]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
      "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report [--events 20|--full|--raw]",
      "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm",
      "bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm",
      "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
    ],
    description: "Import YAML-selected ~/.codex API-key profiles into one Sub2API OpenAI pool, expose one unified API_KEY, and optionally configure this master server's ~/.codex consumer endpoint.",
    target: {
      route: g14K3sRoute,
      namespace,
      serviceDns,
      configPath: codexPoolConfigPath,
      poolGroupName: pool.groupName,
      poolApiKeySecretName: pool.apiKeySecretName,
      poolApiKeySecretKey: pool.apiKeySecretKey,
      publicBaseUrl: pool.publicExposure.enabled ? pool.publicExposure.publicBaseUrl : null,
      masterBaseUrl: pool.publicExposure.enabled ? pool.publicExposure.masterBaseUrl : null,
      sentinelMonitorEnabled: pool.sentinel.monitor.enabled,
      sentinelActionsEnabled: pool.sentinel.actions.enabled,
      secretValuesPrinted: false,
    },
  };
}

export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
  const [action = "plan"] = args;
  if (action === "plan") return codexPoolPlan(parseDisclosureOptions(args.slice(1)));
  if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1)));
  if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1)));
  if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1)));
  if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1)));
  if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1)));
  if (action === "cleanup-probes") return await codexPoolCleanupProbes(config, parseConfirmOptions(args.slice(1)));
  if (action === "expose") return await codexPoolExpose(config, parseConfirmOptions(args.slice(1)));
  if (action === "configure-local") return await codexPoolConfigureLocal(config, parseConfirmOptions(args.slice(1)));
  return {
    ok: false,
    error: "unsupported-platform-infra-sub2api-codex-pool-command",
    args,
    help: codexPoolHelp(),
  };
}

function parseSyncOptions(args: string[]): SyncOptions {
  validateOptions(args, new Set(["--confirm", "--prune-removed", "--full", "--raw"]));
  const disclosure = parseDisclosureOptions(args.filter((arg) => arg !== "--confirm" && arg !== "--prune-removed"));
  return { ...disclosure, confirm: args.includes("--confirm"), pruneRemoved: args.includes("--prune-removed") };
}

function parseConfirmOptions(args: string[]): ConfirmOptions {
  validateOptions(args, new Set(["--confirm", "--full", "--raw"]));
  const disclosure = parseDisclosureOptions(args.filter((arg) => arg !== "--confirm"));
  return { ...disclosure, confirm: args.includes("--confirm") };
}

function parseSentinelImageOptions(args: string[]): SentinelImageOptions {
  const [actionRaw = "status", ...rest] = args;
  if (actionRaw !== "status" && actionRaw !== "build") throw new Error("sentinel-image usage: status|build [--dry-run|--confirm] [--full|--raw]");
  let confirm = false;
  let explicitDryRun = false;
  const disclosureArgs: string[] = [];
  for (const arg of rest) {
    if (arg === "--confirm") {
      confirm = true;
      continue;
    }
    if (arg === "--dry-run") {
      explicitDryRun = true;
      continue;
    }
    if (arg === "--full" || arg === "--raw") {
      disclosureArgs.push(arg);
      continue;
    }
    throw new Error(`unsupported option: ${arg}`);
  }
  if (confirm && explicitDryRun) throw new Error("sentinel-image accepts only one of --confirm or --dry-run");
  const disclosure = parseDisclosureOptions(disclosureArgs);
  return {
    ...disclosure,
    action: actionRaw,
    confirm,
    dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm,
  };
}

function parseSentinelProbeOptions(args: string[]): SentinelProbeOptions {
  const accounts: string[] = [];
  const disclosureArgs: string[] = [];
  let confirm = false;
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index]!;
    if (arg === "--confirm") {
      confirm = true;
      continue;
    }
    if (arg === "--full" || arg === "--raw") {
      disclosureArgs.push(arg);
      continue;
    }
    if (arg === "--account") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--account requires an account name");
      accounts.push(...splitAccountNames(value));
      index += 1;
      continue;
    }
    if (arg.startsWith("--account=")) {
      accounts.push(...splitAccountNames(arg.slice("--account=".length)));
      continue;
    }
    throw new Error(`unsupported option: ${arg}`);
  }
  const uniqueAccounts = [...new Set(accounts)];
  if (uniqueAccounts.length === 0) throw new Error("sentinel-probe requires --account <accountName>");
  for (const account of uniqueAccounts) validateKubernetesName(account, "--account", false);
  const disclosure = parseDisclosureOptions(disclosureArgs);
  return { ...disclosure, confirm, accounts: uniqueAccounts };
}

function parseSentinelReportOptions(args: string[]): SentinelReportOptions {
  let events = 20;
  let explicitEvents = false;
  const disclosureArgs: string[] = [];
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index]!;
    if (arg === "--full" || arg === "--raw") {
      disclosureArgs.push(arg);
      continue;
    }
    if (arg === "--events") {
      const value = args[index + 1];
      if (value === undefined || value.startsWith("--")) throw new Error("--events requires a positive integer");
      events = readReportEventLimit(value, "--events");
      explicitEvents = true;
      index += 1;
      continue;
    }
    if (arg.startsWith("--events=")) {
      events = readReportEventLimit(arg.slice("--events=".length), "--events");
      explicitEvents = true;
      continue;
    }
    throw new Error(`unsupported option: ${arg}`);
  }
  const disclosure = parseDisclosureOptions(disclosureArgs);
  if (disclosure.full && !explicitEvents) events = 80;
  return { ...disclosure, events };
}

function readReportEventLimit(raw: string, option: string): number {
  const value = Number(raw);
  if (!Number.isInteger(value) || value < 1 || value > 200) throw new Error(`${option} must be an integer from 1 to 200`);
  return value;
}

function parseDisclosureOptions(args: string[]): DisclosureOptions {
  validateOptions(args, new Set(["--full", "--raw"]));
  const raw = args.includes("--raw");
  return { full: raw || args.includes("--full"), raw };
}

function splitAccountNames(value: string): string[] {
  return value.split(",").map((item) => item.trim()).filter(Boolean);
}

function validateOptions(args: string[], booleanOptions: Set<string>): void {
  for (const arg of args) {
    if (booleanOptions.has(arg)) continue;
    throw new Error(`unsupported option: ${arg}`);
  }
}

function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false }): Record<string, unknown> {
  const pool = readCodexPoolConfig();
  const profiles = collectCodexProfiles();
  const ok = profiles.length > 0 && profiles.every((profile) => profile.ok);
  return {
    ok,
    action: "platform-infra-sub2api-codex-pool-plan",
    source: {
      directory: join(homedir(), ".codex"),
      configPattern: "YAML-selected config files under ~/.codex",
      authPattern: "YAML-selected auth files under ~/.codex",
      valuesPrinted: false,
    },
    target: poolTarget(),
    config: {
      path: codexPoolConfigPath,
      pool: options.full ? pool : codexPoolConfigSummary(pool),
    },
    profiles: options.full ? profiles.map(redactProfile) : profiles.map(compactProfile),
    decision: {
      accountType: "openai/apikey",
      grouping: `All discovered Codex profiles are bound to one Sub2API group named ${pool.groupName}.`,
      unifiedApiKey: `The client-facing API_KEY is controlled by k3s Secret ${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}.`,
      sentinel: pool.sentinel.monitor.enabled
        ? `Account sentinel is enabled as k8s CronJob ${namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.`
        : "Account sentinel monitoring is disabled by YAML.",
      publicExposure: pool.publicExposure.enabled
        ? `Default Codex consumers use ${codexConsumerBaseUrl(pool)}; bounded master-local probes may use ${pool.publicExposure.masterBaseUrl}. FRP proxy ${pool.publicExposure.proxyName} maps public ${pool.publicExposure.publicBaseUrl} to ${pool.publicExposure.localIP}:${pool.publicExposure.localPort}.`
        : "Public FRP exposure is disabled by YAML.",
      idempotency: "sync reuses the group, account names, and k3s Secret when they already exist; credentials are updated from the current local Codex files; managed accounts missing from YAML are preserved unless --prune-removed is explicitly provided.",
      configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.",
    },
    next: ok
      ? { sync: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" }
      : { fix: "Ensure every discovered config.toml profile has a base_url and either auth.json OPENAI_API_KEY or the configured env_key present in this shell." },
  };
}

async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promise<Record<string, unknown>> {
  const pool = readCodexPoolConfig();
  const profiles = collectCodexProfiles();
  const planOk = profiles.length > 0 && profiles.every((profile) => profile.ok);
  if (!options.confirm || !planOk) {
    return {
      ...codexPoolPlan(options),
      ok: !options.confirm ? planOk : false,
      mode: options.confirm ? "blocked-invalid-local-profile" : "dry-run",
      next: options.confirm
        ? { fix: "Repair invalid local Codex profiles, then rerun sync --confirm." }
        : { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" },
    };
  }

  const sentinelImage = pool.sentinel.monitor.enabled
    ? await runCodexPoolSentinelImage(config, pool, { action: "build", confirm: true, dryRun: false, full: options.full, raw: false })
    : { ok: true, mode: "skipped-monitor-disabled" };
  if (sentinelImage.ok !== true) {
    return {
      ok: false,
      action: "platform-infra-sub2api-codex-pool-sync",
      mode: "blocked-sentinel-image",
      sentinelImage,
      next: {
        image: "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm",
      },
    };
  }

  const payload = {
    pruneRemoved: options.pruneRemoved,
    sentinel: {
      manifest: renderCodexPoolSentinelManifest(pool.sentinel, sentinelProfileSecrets(profiles), {
        namespace,
        serviceName,
        serviceDns,
        appSecretName,
      }),
      summary: codexPoolSentinelSummary(pool.sentinel),
    },
    pool: {
      groupName: pool.groupName,
      apiKeyName: pool.apiKeyName,
      apiKeySecretName: pool.apiKeySecretName,
      apiKeySecretKey: pool.apiKeySecretKey,
      minOwnerBalanceUsd: pool.minOwnerBalanceUsd,
      minOwnerConcurrency: pool.minOwnerConcurrency,
      defaultAccountPriority: pool.defaultAccountPriority,
      defaultAccountCapacity: pool.defaultAccountCapacity,
      defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
    },
    profiles: profiles.map((profile) => ({
      profile: profile.profile,
      accountName: profile.accountName,
      configFile: profile.configFile,
      authFile: profile.authFile,
      provider: profile.provider,
      baseUrl: profile.baseUrl,
      wireApi: profile.wireApi,
      model: profile.model,
      apiKey: profile.apiKey,
      apiKeySource: profile.apiKeySource,
      apiKeyFingerprint: fingerprint(profile.apiKey ?? ""),
      openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
      upstreamUserAgent: profile.upstreamUserAgent,
      priority: profile.priority,
      capacity: profile.capacity,
      loadFactor: profile.loadFactor,
      tempUnschedulable: profile.tempUnschedulable,
      tempUnschedulableCredentials: renderSub2ApiTempUnschedulableCredentials(profile.tempUnschedulable),
    })),
  };
  const result = await runRemoteCodexPoolScript(config, "sync", syncScript(payload, pool));
  const parsed = parseJsonOutput(result.stdout);
  if (options.raw) {
    return {
      ok: result.exitCode === 0 && boolField(parsed, "ok", false),
      action: "platform-infra-sub2api-codex-pool-sync",
      remote: compactCapture(result, { full: true }),
      parsed,
    };
  }
  return {
    ok: result.exitCode === 0 && boolField(parsed, "ok", false),
    action: "platform-infra-sub2api-codex-pool-sync",
    local: {
      profileCount: profiles.length,
      pruneRemoved: options.pruneRemoved,
      invalidProfiles: profiles.filter((profile) => !profile.ok).map(compactProfile),
      profiles: options.full ? profiles.map(redactProfile) : undefined,
      valuesPrinted: false,
    },
    sentinelImage,
    remote: parsed === null
      ? compactCapture(result, { full: options.full || result.exitCode !== 0 })
      : options.full ? parsed : codexPoolSyncSummary(parsed),
    next: {
      validate: "bun scripts/cli.ts platform-infra sub2api codex-pool validate",
    },
  };
}

async function codexPoolSentinelImage(config: UniDeskConfig, options: SentinelImageOptions): Promise<Record<string, unknown>> {
  const pool = readCodexPoolConfig();
  return await runCodexPoolSentinelImage(config, pool, options);
}

async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolConfig, options: SentinelImageOptions): Promise<Record<string, unknown>> {
  const target = codexPoolSentinelRuntimeImage(pool.sentinel);
  if (options.action === "build" && options.dryRun) {
    return {
      ok: true,
      action: "platform-infra-sub2api-codex-pool-sentinel-image",
      mode: "dry-run",
      image: target,
      dockerfile: sentinelImageDockerfilePath,
      mutation: false,
      next: {
        confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm",
      },
    };
  }
  const mode: RemoteCodexPoolMode = options.action === "status" ? "sentinel-image-status" : "sentinel-image-build";
  const script = options.action === "status" ? sentinelImageStatusScript(pool) : sentinelImageBuildScript(pool);
  const result = await runRemoteCodexPoolScript(config, mode, script);
  const parsed = parseJsonOutput(result.stdout);
  if (options.raw) {
    return {
      ok: result.exitCode === 0 && boolField(parsed, "ok", false),
      action: "platform-infra-sub2api-codex-pool-sentinel-image",
      mode: options.action,
      image: target,
      remote: compactCapture(result, { full: true }),
      parsed,
    };
  }
  return {
    ok: result.exitCode === 0 && boolField(parsed, "ok", false),
    action: "platform-infra-sub2api-codex-pool-sentinel-image",
    mode: options.action,
    image: target,
    summary: parsed,
    remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
  };
}

async function codexPoolValidate(config: UniDeskConfig, options: DisclosureOptions): Promise<Record<string, unknown>> {
  const pool = readCodexPoolConfig();
  const result = await runRemoteCodexPoolScript(config, "validate", validateScript(pool));
  const parsed = parseJsonOutput(result.stdout);
  if (options.raw) {
    return {
      ok: result.exitCode === 0 && boolField(parsed, "ok", false),
      action: "platform-infra-sub2api-codex-pool-validate",
      remote: compactCapture(result, { full: true }),
      parsed,
    };
  }
  return {
    ok: result.exitCode === 0 && boolField(parsed, "ok", false),
    action: "platform-infra-sub2api-codex-pool-validate",
    summary: options.full ? parsed : codexPoolValidationSummary(parsed),
    remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
  };
}

async function codexPoolSentinelReport(config: UniDeskConfig, options: SentinelReportOptions): Promise<Record<string, unknown> | RenderedCliResult> {
  const pool = readCodexPoolConfig();
  const result = await capture(config, g14K3sRoute, ["script"], sentinelReportScript(pool, options.events));
  const parsed = parseJsonOutput(result.stdout);
  const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
  if (options.raw) {
    return {
      ok,
      action: "platform-infra-sub2api-codex-pool-sentinel-report",
      remote: compactCapture(result, { full: true }),
      report: parsed,
      valuesPrinted: false,
    };
  }
  const text = renderSentinelReport(parsed, {
    events: options.events,
    full: options.full,
    remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }),
  });
  return renderedCliResult(ok, "platform-infra sub2api codex-pool sentinel-report", text);
}

async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelProbeOptions): Promise<Record<string, unknown>> {
  const pool = readCodexPoolConfig();
  const configuredAccounts = desiredAccountNames(pool);
  const missing = options.accounts.filter((account) => !configuredAccounts.includes(account));
  if (missing.length > 0) {
    return {
      ok: false,
      action: "platform-infra-sub2api-codex-pool-sentinel-probe",
      error: "account-not-in-yaml",
      missing,
      configuredAccounts,
      valuesPrinted: false,
    };
  }
  if (!options.confirm) {
    return {
      ok: true,
      action: "platform-infra-sub2api-codex-pool-sentinel-probe",
      mode: "dry-run",
      target: poolTarget(pool),
      accounts: options.accounts,
      effect: "Would create one Kubernetes Job from the managed sentinel CronJob and force an immediate marker probe for the requested account(s).",
      next: {
        confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account ${options.accounts.join(",")} --confirm`,
      },
      valuesPrinted: false,
    };
  }
  const payload = {
    accounts: options.accounts,
  };
  const result = await runRemoteCodexPoolScript(config, "sentinel-probe", sentinelProbeScript(payload, pool));
  const parsed = parseJsonOutput(result.stdout);
  if (options.raw) {
    return {
      ok: result.exitCode === 0 && boolField(parsed, "ok", false),
      action: "platform-infra-sub2api-codex-pool-sentinel-probe",
      remote: compactCapture(result, { full: true }),
      parsed,
    };
  }
  return {
    ok: result.exitCode === 0 && boolField(parsed, "ok", false),
    action: "platform-infra-sub2api-codex-pool-sentinel-probe",
    summary: options.full ? parsed : compactSentinelProbeResult(parsed),
    remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
  };
}

async function codexPoolCleanupProbes(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
  if (!options.confirm) {
    return {
      ok: true,
      action: "platform-infra-sub2api-codex-pool-cleanup-probes",
      mode: "dry-run",
      target: poolTarget(),
      scope: "Only deletes temporary resources whose names start with unidesk-probe-.",
      next: { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm" },
      valuesPrinted: false,
    };
  }
  const pool = readCodexPoolConfig();
  const result = await capture(config, g14K3sRoute, ["script"], cleanupProbesScript(pool));
  const parsed = parseJsonOutput(result.stdout);
  if (options.raw) {
    return {
      ok: result.exitCode === 0 && boolField(parsed, "ok", false),
      action: "platform-infra-sub2api-codex-pool-cleanup-probes",
      remote: compactCapture(result, { full: true }),
      parsed,
    };
  }
  return {
    ok: result.exitCode === 0 && boolField(parsed, "ok", false),
    action: "platform-infra-sub2api-codex-pool-cleanup-probes",
    summary: parsed,
    remote: compactCapture(result, { full: options.full || result.exitCode !== 0 }),
  };
}

async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
  const pool = readCodexPoolConfig();
  if (!pool.publicExposure.enabled) {
    return {
      ok: true,
      action: "platform-infra-sub2api-codex-pool-expose",
      mode: "disabled-by-yaml",
      target: poolTarget(pool),
    };
  }
  if (!options.confirm) {
    return {
      ok: true,
      action: "platform-infra-sub2api-codex-pool-expose",
      mode: "dry-run",
      target: poolTarget(pool),
      publicExposure: publicExposureSummary(pool),
      next: {
        confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm",
      },
    };
  }
  const masterResult = await applyMasterFrpsAllowPort(pool);
  const caddyResult = await applyMasterCaddySite(pool);
  const remoteResult = await capture(config, g14K3sRoute, ["script"], exposeScript(pool));
  const parsed = parseJsonOutput(remoteResult.stdout);
  const publicProbe = await probePublicModels(pool, "without-api-key", undefined, "public");
  const ok = masterResult.ok && caddyResult.ok && remoteResult.exitCode === 0 && boolField(parsed, "ok", false) && publicProbe.httpStatus === 401;
  if (options.raw) {
    return {
      ok,
      action: "platform-infra-sub2api-codex-pool-expose",
      masterFrps: masterResult,
      masterCaddy: caddyResult,
      remote: compactCapture(remoteResult, { full: true }),
      parsed,
      publicProbe,
    };
  }
  return {
    ok,
    action: "platform-infra-sub2api-codex-pool-expose",
    mode: "confirmed",
    publicExposure: publicExposureSummary(pool),
    masterFrps: masterResult,
    masterCaddy: caddyResult,
    remote: parsed ?? compactCapture(remoteResult, { full: options.full || remoteResult.exitCode !== 0 }),
    publicProbe,
    next: {
      configureLocal: "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
      validate: "bun scripts/cli.ts platform-infra sub2api codex-pool validate",
    },
  };
}

async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOptions): Promise<Record<string, unknown>> {
  const pool = readCodexPoolConfig();
  const codexDir = join(homedir(), ".codex");
  const configPath = join(codexDir, "config.toml");
  const authPath = join(codexDir, "auth.json");
  const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`);
  const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`);
  if (!options.confirm) {
    return {
      ok: true,
      action: "platform-infra-sub2api-codex-pool-configure-local",
      mode: "dry-run",
      target: {
        codexDir,
        configPath,
        authPath,
        backupConfigPath,
        backupAuthPath,
        baseUrl: codexConsumerBaseUrl(pool),
        providerName: pool.localCodex.providerName,
        wireApi: pool.localCodex.wireApi,
        supportsWebSockets: pool.localCodex.supportsWebSockets,
        responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
        responsesSmokeModel: pool.localCodex.responsesSmokeModel,
        valuesPrinted: false,
      },
      next: {
        confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm",
      },
    };
  }
  if (!pool.publicExposure.enabled) throw new Error(`${codexPoolConfigPath}.publicExposure.enabled must be true before configure-local`);
  const keyResult = await fetchPoolApiKey(config, pool);
  if (keyResult.apiKey === null) {
    return {
      ok: false,
      action: "platform-infra-sub2api-codex-pool-configure-local",
      error: keyResult.error ?? "pool API key missing",
      valuesPrinted: false,
    };
  }
  const writeResult = writeLocalCodexConfig(pool, keyResult.apiKey);
  const validateResult = await validatePublicGatewayWithKey(pool, keyResult.apiKey);
  return {
    ok: writeResult.ok && validateResult.ok,
    action: "platform-infra-sub2api-codex-pool-configure-local",
    mode: "confirmed",
    local: writeResult,
    validation: validateResult,
    apiKey: {
      secret: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`,
      keyPreview: apiKeyPreview(keyResult.apiKey),
      apiKeyFingerprint: fingerprint(keyResult.apiKey),
      valuesPrinted: false,
    },
  };
}

function collectCodexProfiles(): CodexProfile[] {
  const codexDir = join(homedir(), ".codex");
  const pool = readCodexPoolConfig();
  if (!existsSync(codexDir)) return [];
  const seenAccountNames = new Set<string>();
  const configs = pool.profiles.length > 0
    ? pool.profiles
    : discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
  return configs.map((entry) => {
    const resolved = resolveProfileFiles(codexDir, entry);
    const profile = entry.profile;
    const accountName = entry.accountName ?? uniqueAccountName(profile, seenAccountNames);
    seenAccountNames.add(accountName);
    const configFile = resolved.configFile;
    const authFile = resolved.authFile;
    const configPath = join(codexDir, configFile);
    const authPath = join(codexDir, authFile);
    const base: CodexProfile = {
      profile,
      accountName,
      configFile,
      authFile,
      provider: "",
      baseUrl: "",
      wireApi: null,
      model: null,
      envKey: null,
      apiKey: null,
      apiKeySource: null,
      openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode,
      upstreamUserAgent: entry.upstreamUserAgent,
      priority: entry.priority,
      capacity: entry.capacity ?? pool.defaultAccountCapacity,
      loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor,
      tempUnschedulable: entry.tempUnschedulable,
      authOpenAIKeyShape: existsSync(authPath) ? "unknown" : "missing",
      ok: false,
      error: null,
    };

    try {
      if (!existsSync(configPath)) throw new Error(`config file ${configFile} is missing`);
      const parsed = Bun.TOML.parse(readFileSync(configPath, "utf8")) as unknown;
      if (!isRecord(parsed)) throw new Error("config is not a TOML object");
      const providers = parsed.model_providers;
      if (!isRecord(providers)) throw new Error("model_providers is missing");
      const provider = stringValue(parsed.model_provider) ?? Object.keys(providers)[0] ?? "";
      if (provider === "") throw new Error("model_provider is missing");
      const providerConfig = providers[provider];
      if (!isRecord(providerConfig)) throw new Error(`model provider ${provider} is missing`);
      const baseUrl = normalizeBaseUrl(stringValue(providerConfig.base_url));
      if (baseUrl === null) throw new Error(`model provider ${provider} base_url is missing or invalid`);
      base.provider = provider;
      base.baseUrl = baseUrl;
      base.envKey = stringValue(providerConfig.env_key);
      base.wireApi = stringValue(providerConfig.wire_api);
      base.model = stringValue(parsed.model);

      const auth = readAuthAPIKey(authPath);
      base.authOpenAIKeyShape = auth.shape;
      if (auth.apiKey !== null) {
        base.apiKey = auth.apiKey;
        base.apiKeySource = "auth-json";
      } else if (base.envKey !== null && typeof process.env[base.envKey] === "string" && process.env[base.envKey]!.length > 0) {
        base.apiKey = process.env[base.envKey]!;
        base.apiKeySource = "env";
      }
      if (base.apiKey === null || base.apiKey.length === 0) {
        throw new Error(base.envKey === null ? "auth OPENAI_API_KEY is missing or empty" : `auth OPENAI_API_KEY is missing and env ${base.envKey} is not present`);
      }
      base.ok = true;
      return base;
    } catch (error) {
      base.error = error instanceof Error ? error.message : String(error);
      return base;
    }
  });
}

function discoverCodexProfileConfigs(
  codexDir: string,
  defaultTempUnschedulable = defaultCodexTempUnschedulablePolicy(),
  defaultPriority = defaultAccountPriority,
): CodexPoolProfileConfig[] {
  return readdirSync(codexDir)
    .filter((file) => file === "config.toml" || file.startsWith("config.toml."))
    .sort((a, b) => {
      if (a === "config.toml") return -1;
      if (b === "config.toml") return 1;
      return a.localeCompare(b);
    })
    .map((configFile) => {
      const suffix = configFile === "config.toml" ? "" : configFile.slice("config.toml.".length);
      const profile = suffix === "" ? "default" : suffix;
      return {
        profile,
        accountName: null,
        configFile,
        authFile: suffix === "" ? "auth.json" : `auth.json.${suffix}`,
        fallbackConfigFile: null,
        fallbackAuthFile: null,
        openaiResponsesWebSocketsV2Mode: null,
        upstreamUserAgent: null,
        priority: defaultPriority,
        capacity: null,
        loadFactor: null,
        tempUnschedulable: defaultTempUnschedulable,
      };
    });
}

function resolveProfileFiles(codexDir: string, profile: CodexPoolProfileConfig): { configFile: string; authFile: string } {
  const configFile = existsSync(join(codexDir, profile.configFile)) || profile.fallbackConfigFile === null
    ? profile.configFile
    : profile.fallbackConfigFile;
  const authFile = existsSync(join(codexDir, profile.authFile)) || profile.fallbackAuthFile === null
    ? profile.authFile
    : profile.fallbackAuthFile;
  return { configFile, authFile };
}

function readCodexPoolConfig(): CodexPoolConfig {
  const defaults = defaultCodexPoolConfig();
  if (!existsSync(codexPoolConfigPath)) return defaults;
  const parsed = Bun.YAML.parse(readFileSync(codexPoolConfigPath, "utf8")) as unknown;
  if (!isRecord(parsed)) throw new Error(`${codexPoolConfigPath} must contain a YAML object`);
  const pool = parsed.pool;
  if (!isRecord(pool)) throw new Error(`${codexPoolConfigPath}.pool must be a YAML object`);
  rejectSchedulableYamlField(pool, "pool");
  const defaultTempUnschedulable = readTempUnschedulablePolicy(pool.defaultTempUnschedulable, "pool.defaultTempUnschedulable", defaults.defaultTempUnschedulable);
  const defaultAccountPriorityValue = readAccountPriority(pool.defaultAccountPriority, "pool.defaultAccountPriority");
  const defaultAccountCapacityValue = readAccountCapacity(pool.defaultAccountCapacity, "pool.defaultAccountCapacity");
  const defaultAccountLoadFactorValue = readAccountLoadFactor(pool.defaultAccountLoadFactor, "pool.defaultAccountLoadFactor");
  const profiles = readProfileConfig(
    parsed.profiles,
    defaults.profiles,
    defaultTempUnschedulable,
    defaultAccountPriorityValue,
  );
  const declaredAccountCapacity = desiredProfileCapacityTotal(profiles, defaultAccountCapacityValue);
  const minOwnerConcurrencySource = pool.minOwnerConcurrency === undefined || pool.minOwnerConcurrency === null ? "auto" : "yaml";
  const minOwnerConcurrency = minOwnerConcurrencySource === "auto"
    ? Math.max(1, declaredAccountCapacity)
    : readOwnerConcurrency(pool.minOwnerConcurrency, "pool.minOwnerConcurrency");
  const config: CodexPoolConfig = {
    groupName: stringValue(pool.groupName) ?? defaults.groupName,
    apiKeyName: stringValue(pool.apiKeyName) ?? defaults.apiKeyName,
    apiKeySecretName: stringValue(pool.apiKeySecretName) ?? defaults.apiKeySecretName,
    apiKeySecretKey: stringValue(pool.apiKeySecretKey) ?? defaults.apiKeySecretKey,
    minOwnerBalanceUsd: numberValue(pool.minOwnerBalanceUsd) ?? defaults.minOwnerBalanceUsd,
    minOwnerConcurrency,
    minOwnerConcurrencySource,
    defaultAccountPriority: defaultAccountPriorityValue,
    defaultAccountCapacity: defaultAccountCapacityValue,
    defaultAccountLoadFactor: defaultAccountLoadFactorValue,
    defaultTempUnschedulable,
    profiles,
    publicExposure: readPublicExposureConfig(parsed.publicExposure, defaults.publicExposure),
    localCodex: readLocalCodexConfig(parsed.localCodex, defaults.localCodex),
    sentinel: readCodexPoolSentinelConfig(parsed.sentinel, defaults.sentinel, codexPoolConfigPath),
  };
  validateKubernetesName(config.groupName, "pool.groupName", false);
  validateKubernetesName(config.apiKeySecretName, "pool.apiKeySecretName", true);
  if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(config.apiKeySecretKey)) {
    throw new Error(`${codexPoolConfigPath}.pool.apiKeySecretKey must be a valid secret key name`);
  }
  if (config.minOwnerBalanceUsd <= 0) throw new Error(`${codexPoolConfigPath}.pool.minOwnerBalanceUsd must be > 0`);
  if (declaredAccountCapacity > 0 && config.minOwnerConcurrency < declaredAccountCapacity) {
    throw new Error(`${codexPoolConfigPath}.pool.minOwnerConcurrency must be >= the sum of declared account capacities (${declaredAccountCapacity})`);
  }
  return config;
}

function defaultCodexPoolConfig(): CodexPoolConfig {
  return {
    groupName: defaultPoolGroupName,
    apiKeyName: defaultPoolApiKeyName,
    apiKeySecretName: defaultPoolApiKeySecretName,
    apiKeySecretKey: defaultPoolApiKeySecretKey,
    minOwnerBalanceUsd: defaultMinOwnerBalanceUsd,
    minOwnerConcurrency: 1,
    minOwnerConcurrencySource: "auto",
    defaultAccountPriority,
    defaultAccountCapacity: 5,
    defaultAccountLoadFactor: 1,
    defaultTempUnschedulable: defaultCodexTempUnschedulablePolicy(),
    profiles: [],
    publicExposure: {
      enabled: false,
      proxyName: "platform-infra-sub2api",
      configMapName: "sub2api-frpc-config",
      deploymentName: "sub2api-frpc",
      frpcImage: "fatedier/frpc:v0.68.1",
      serverAddr: "74.48.78.17",
      serverPort: 7000,
      remotePort: 21880,
      localIP: serviceDns.split(":")[0],
      localPort: 8080,
      publicBaseUrl: "http://74.48.78.17:21880",
      masterBaseUrl: "http://127.0.0.1:21880",
      masterFrps: {
        configPath: "/opt/hwlab-frp/frps.dev.toml",
        containerName: "hwlab-frps-dev",
      },
      masterCaddy: {
        enabled: true,
        domain: "sub2api.74-48-78-17.nip.io",
        configPath: "/etc/caddy/Caddyfile",
        serviceName: "caddy",
        upstreamBaseUrl: "http://127.0.0.1:21880",
        responseHeaderTimeoutSeconds: 180,
      },
    },
    localCodex: {
      backupSuffix: "pre-sub2api",
      providerName: "OpenAI",
      wireApi: "responses",
      supportsWebSockets: true,
      responsesWebSocketsV2: true,
      responsesSmokeModel: "gpt-5.5",
    },
    sentinel: defaultCodexPoolSentinelConfig(),
  };
}

export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePolicy {
  return {
    enabled: true,
    rules: [
      {
        statusCode: 400,
        keywords: ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "bad_response_status_code", "model_not_found", "no available channel for model", "unsupported", "not supported", "not support", "暂不支持", "可用模型"],
        durationMinutes: 120,
        description: "Stable upstream 400 model-routing or Responses encrypted-content compatibility failures should use another account.",
      },
      {
        statusCode: 401,
        keywords: ["unauthorized", "invalid api key", "invalid_api_key", "authentication", "recovered upstream error"],
        durationMinutes: 120,
        description: "Credential/auth failures should use the longest cooldown.",
      },
      {
        statusCode: 403,
        keywords: ["forbidden", "access denied", "quota", "billing", "capacity", "weekly limit", "less than 10% of your weekly limit left", "run /status for a breakdown", "recovered upstream error"],
        durationMinutes: 120,
        description: "Permission, quota, or account-state failures should use the longest cooldown.",
      },
      {
        statusCode: 429,
        keywords: ["capacity", "rate limit", "rate_limit", "quota", "weekly limit", "less than 10% of your weekly limit left", "run /status for a breakdown", "too many requests", "overloaded", "resource_exhausted", "recovered upstream error"],
        durationMinutes: 10,
        description: "Capacity and rate-limit responses are often temporary; start with a ten-minute cooldown and use another account.",
      },
      {
        statusCode: 500,
        keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "recovered upstream error"],
        durationMinutes: 10,
        description: "Transient upstream server failures should start with a ten-minute cooldown and prefer another account.",
      },
      {
        statusCode: 502,
        keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "websocket dial", "handshake response", "recovered upstream error"],
        durationMinutes: 30,
        description: "Gateway upstream failures, including recovered upstream error wrappers, should cool down longer.",
      },
      {
        statusCode: 413,
        keywords: ["openai_error", "payload too large", "request too large", "context length", "context window", "maximum context"],
        durationMinutes: 30,
        description: "Large-context upstream failures should cool down the selected account so a larger-context channel can handle the request.",
      },
      {
        statusCode: 503,
        keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "recovered upstream error", "model_not_found", "no available channel for model"],
        durationMinutes: 30,
        description: "Service unavailable and upstream model-routing failures should cool down longer than one-off transient failures.",
      },
      {
        statusCode: 504,
        keywords: ["gateway timeout", "timeout", "upstream", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "recovered upstream error"],
        durationMinutes: 30,
        description: "Gateway timeout responses should cool down the selected account so another account can handle the next request.",
      },
      {
        statusCode: 524,
        keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "recovered upstream error"],
        durationMinutes: 30,
        description: "Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.",
      },
      {
        statusCode: 529,
        keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "recovered upstream error"],
        durationMinutes: 30,
        description: "Provider overloaded responses should cool down longer than generic transient failures and use another account.",
      },
    ],
  };
}

function readProfileConfig(
  value: unknown,
  defaults: CodexPoolProfileConfig[],
  defaultTempUnschedulable: CodexTempUnschedulablePolicy,
  defaultPriority: number,
): CodexPoolProfileConfig[] {
  if (!isRecord(value)) return defaults;
  const entries = value.entries;
  if (!Array.isArray(entries)) throw new Error(`${codexPoolConfigPath}.profiles.entries must be a YAML array`);
  return entries.map((entry, index) => {
    if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] must be an object`);
    rejectSchedulableYamlField(entry, `profiles.entries[${index}]`);
    const profile = stringValue(entry.profile);
    const configFile = stringValue(entry.configFile);
    const authFile = stringValue(entry.authFile);
    if (profile === null || configFile === null || authFile === null) {
      throw new Error(`${codexPoolConfigPath}.profiles.entries[${index}] requires profile, configFile, and authFile`);
    }
    const accountName = stringValue(entry.accountName);
    if (accountName !== null) validateKubernetesName(accountName, `profiles.entries[${index}].accountName`, false);
    validateCodexFileName(configFile, `profiles.entries[${index}].configFile`);
    validateCodexFileName(authFile, `profiles.entries[${index}].authFile`);
    const fallbackConfigFile = stringValue(entry.fallbackConfigFile);
    const fallbackAuthFile = stringValue(entry.fallbackAuthFile);
    if (fallbackConfigFile !== null) validateCodexFileName(fallbackConfigFile, `profiles.entries[${index}].fallbackConfigFile`);
    if (fallbackAuthFile !== null) validateCodexFileName(fallbackAuthFile, `profiles.entries[${index}].fallbackAuthFile`);
    const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`);
    const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`);
    const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority);
    const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`);
    const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`);
    const tempUnschedulable = readTempUnschedulablePolicy(entry.tempUnschedulable, `profiles.entries[${index}].tempUnschedulable`, defaultTempUnschedulable);
    return {
      profile,
      accountName,
      configFile,
      authFile,
      fallbackConfigFile,
      fallbackAuthFile,
      openaiResponsesWebSocketsV2Mode,
      upstreamUserAgent,
      priority,
      capacity,
      loadFactor,
      tempUnschedulable,
    };
  });
}

function desiredProfileCapacityTotal(profiles: CodexPoolProfileConfig[], defaultCapacity: number): number {
  return profiles.reduce((total, profile) => total + (profile.capacity ?? defaultCapacity), 0);
}

function rejectSchedulableYamlField(value: Record<string, unknown>, key: string): void {
  if (Object.prototype.hasOwnProperty.call(value, "schedulable")) {
    throw new Error(`${codexPoolConfigPath}.${key}.schedulable is process control, not durable YAML config; use codex-pool sync actions instead`);
  }
}

function readOpenAIResponsesWebSocketsV2Mode(value: unknown, key: string): OpenAIResponsesWebSocketsV2Mode | null {
  if (value === undefined || value === null) return null;
  const text = stringValue(value);
  if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`);
  if (text === "off" || text === "ctx_pool" || text === "passthrough") return text;
  throw new Error(`${codexPoolConfigPath}.${key} must be one of off, ctx_pool, passthrough`);
}

function readUpstreamUserAgent(value: unknown, key: string): string | null {
  if (value === undefined || value === null) return null;
  const text = stringValue(value);
  if (text === null) throw new Error(`${codexPoolConfigPath}.${key} must be a string`);
  if (text.length > 512) throw new Error(`${codexPoolConfigPath}.${key} must be at most 512 characters`);
  if (/[\r\n]/u.test(text)) throw new Error(`${codexPoolConfigPath}.${key} must not contain newlines`);
  return text;
}

function readAccountPriority(value: unknown, key: string, fallback = defaultAccountPriority): number {
  if (value === undefined || value === null) return fallback;
  const priority = numberValue(value);
  if (priority === null || !Number.isInteger(priority) || priority < 0 || priority > 1000) {
    throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 0 to 1000`);
  }
  return priority;
}

function readAccountCapacity(value: unknown, key: string): number {
  const capacity = numberValue(value);
  if (capacity === null || !Number.isInteger(capacity) || capacity < 1 || capacity > 1000) {
    throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`);
  }
  return capacity;
}

function readOwnerConcurrency(value: unknown, key: string): number {
  const concurrency = numberValue(value);
  if (concurrency === null || !Number.isInteger(concurrency) || concurrency < 1 || concurrency > 100000) {
    throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 100000`);
  }
  return concurrency;
}

function readAccountLoadFactor(value: unknown, key: string): number {
  const loadFactor = numberValue(value);
  if (loadFactor === null || !Number.isInteger(loadFactor) || loadFactor < 1 || loadFactor > 1000) {
    throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 1 to 1000`);
  }
  return loadFactor;
}

function readCaddyTimeoutSeconds(value: unknown, key: string, fallback: number): number {
  if (value === undefined || value === null) return fallback;
  const seconds = numberValue(value);
  if (seconds === null || !Number.isInteger(seconds) || seconds < 30 || seconds > 900) {
    throw new Error(`${codexPoolConfigPath}.${key} must be an integer from 30 to 900`);
  }
  return seconds;
}

function readTempUnschedulablePolicy(value: unknown, key: string, fallback: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
  if (value === undefined || value === null) return cloneTempUnschedulablePolicy(fallback);
  if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
  const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled);
  const rules = value.rules === undefined || value.rules === null
    ? cloneTempUnschedulablePolicy(fallback).rules
    : readTempUnschedulableRules(value.rules, `${key}.rules`);
  if (enabled && rules.length === 0) throw new Error(`${codexPoolConfigPath}.${key}.rules must not be empty when enabled=true`);
  return { enabled, rules };
}

function readTempUnschedulableRules(value: unknown, key: string): CodexTempUnschedulableRule[] {
  if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
  return value.map((entry, index) => {
    if (!isRecord(entry)) throw new Error(`${codexPoolConfigPath}.${key}[${index}] must be an object`);
    const statusCode = numberValue(entry.statusCode ?? entry.errorCode);
    if (statusCode === null || !Number.isInteger(statusCode) || statusCode < 100 || statusCode > 599) {
      throw new Error(`${codexPoolConfigPath}.${key}[${index}].statusCode must be an HTTP status code from 100 to 599`);
    }
    const durationMinutes = numberValue(entry.durationMinutes);
    if (durationMinutes === null || !Number.isInteger(durationMinutes) || durationMinutes < 1 || durationMinutes > 1440) {
      throw new Error(`${codexPoolConfigPath}.${key}[${index}].durationMinutes must be an integer from 1 to 1440`);
    }
    const keywords = readTempUnschedulableKeywords(entry.keywords, `${key}[${index}].keywords`);
    const description = stringValue(entry.description);
    if (description !== null && description.length > 240) throw new Error(`${codexPoolConfigPath}.${key}[${index}].description must be at most 240 characters`);
    return { statusCode, keywords, durationMinutes, description };
  });
}

function readTempUnschedulableKeywords(value: unknown, key: string): string[] {
  if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
  const seen = new Set<string>();
  const keywords: string[] = [];
  for (const item of value) {
    const keyword = stringValue(item);
    if (keyword === null) throw new Error(`${codexPoolConfigPath}.${key} entries must be non-empty strings`);
    if (keyword.length > 120) throw new Error(`${codexPoolConfigPath}.${key} entries must be at most 120 characters`);
    if (/[\r\n]/u.test(keyword)) throw new Error(`${codexPoolConfigPath}.${key} entries must not contain newlines`);
    const normalized = keyword.toLowerCase();
    if (seen.has(normalized)) continue;
    seen.add(normalized);
    keywords.push(keyword);
  }
  if (keywords.length === 0) throw new Error(`${codexPoolConfigPath}.${key} must not be empty`);
  return keywords;
}

function cloneTempUnschedulablePolicy(policy: CodexTempUnschedulablePolicy): CodexTempUnschedulablePolicy {
  return {
    enabled: policy.enabled,
    rules: policy.rules.map((rule) => ({
      statusCode: rule.statusCode,
      keywords: [...rule.keywords],
      durationMinutes: rule.durationMinutes,
      description: rule.description,
    })),
  };
}

function readBooleanConfig(value: unknown, key: string, fallback: boolean): boolean {
  if (value === undefined || value === null) return fallback;
  const parsed = booleanValue(value);
  if (parsed === null) throw new Error(`${codexPoolConfigPath}.${key} must be a boolean`);
  return parsed;
}

function readPublicExposureConfig(value: unknown, defaults: CodexPoolPublicExposureConfig): CodexPoolPublicExposureConfig {
  if (!isRecord(value)) return defaults;
  const masterFrpsValue = isRecord(value.masterFrps) ? value.masterFrps : {};
  const masterCaddyValue = isRecord(value.masterCaddy) ? value.masterCaddy : {};
  const config: CodexPoolPublicExposureConfig = {
    enabled: booleanValue(value.enabled) ?? defaults.enabled,
    proxyName: stringValue(value.proxyName) ?? defaults.proxyName,
    configMapName: stringValue(value.configMapName) ?? defaults.configMapName,
    deploymentName: stringValue(value.deploymentName) ?? defaults.deploymentName,
    frpcImage: stringValue(value.frpcImage) ?? defaults.frpcImage,
    serverAddr: stringValue(value.serverAddr) ?? defaults.serverAddr,
    serverPort: numberValue(value.serverPort) ?? defaults.serverPort,
    remotePort: numberValue(value.remotePort) ?? defaults.remotePort,
    localIP: stringValue(value.localIP) ?? defaults.localIP,
    localPort: numberValue(value.localPort) ?? defaults.localPort,
    publicBaseUrl: normalizeBaseUrl(stringValue(value.publicBaseUrl)) ?? defaults.publicBaseUrl,
    masterBaseUrl: normalizeBaseUrl(stringValue(value.masterBaseUrl)) ?? defaults.masterBaseUrl,
    masterFrps: {
      configPath: stringValue(masterFrpsValue.configPath) ?? defaults.masterFrps.configPath,
      containerName: stringValue(masterFrpsValue.containerName) ?? defaults.masterFrps.containerName,
    },
    masterCaddy: {
      enabled: booleanValue(masterCaddyValue.enabled) ?? defaults.masterCaddy.enabled,
      domain: stringValue(masterCaddyValue.domain) ?? defaults.masterCaddy.domain,
      configPath: stringValue(masterCaddyValue.configPath) ?? defaults.masterCaddy.configPath,
      serviceName: stringValue(masterCaddyValue.serviceName) ?? defaults.masterCaddy.serviceName,
      upstreamBaseUrl: normalizeBaseUrl(stringValue(masterCaddyValue.upstreamBaseUrl)) ?? defaults.masterCaddy.upstreamBaseUrl,
      responseHeaderTimeoutSeconds: readCaddyTimeoutSeconds(
        masterCaddyValue.responseHeaderTimeoutSeconds,
        "publicExposure.masterCaddy.responseHeaderTimeoutSeconds",
        defaults.masterCaddy.responseHeaderTimeoutSeconds,
      ),
    },
  };
  validateKubernetesName(config.configMapName, "publicExposure.configMapName", true);
  validateKubernetesName(config.deploymentName, "publicExposure.deploymentName", true);
  validateProxyName(config.proxyName, "publicExposure.proxyName");
  validatePort(config.serverPort, "publicExposure.serverPort");
  validatePort(config.remotePort, "publicExposure.remotePort");
  validatePort(config.localPort, "publicExposure.localPort");
  if (!/^[A-Za-z0-9._:/-]+$/u.test(config.frpcImage)) throw new Error(`${codexPoolConfigPath}.publicExposure.frpcImage has an unsupported format`);
  if (!/^[A-Za-z0-9._:-]+$/u.test(config.serverAddr)) throw new Error(`${codexPoolConfigPath}.publicExposure.serverAddr has an unsupported format`);
  if (!/^[A-Za-z0-9._:-]+$/u.test(config.localIP)) throw new Error(`${codexPoolConfigPath}.publicExposure.localIP has an unsupported format`);
  if (!config.masterFrps.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterFrps.configPath must be absolute`);
  validateProxyName(config.masterFrps.containerName, "publicExposure.masterFrps.containerName");
  validatePublicHostname(config.masterCaddy.domain, "publicExposure.masterCaddy.domain");
  if (!config.masterCaddy.configPath.startsWith("/")) throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.configPath must be absolute`);
  validateProxyName(config.masterCaddy.serviceName, "publicExposure.masterCaddy.serviceName");
  const upstream = new URL(config.masterCaddy.upstreamBaseUrl);
  if (upstream.protocol !== "http:") throw new Error(`${codexPoolConfigPath}.publicExposure.masterCaddy.upstreamBaseUrl must use http:// for the local upstream`);
  return config;
}

function readLocalCodexConfig(value: unknown, defaults: CodexPoolLocalCodexConfig): CodexPoolLocalCodexConfig {
  if (!isRecord(value)) return defaults;
  const config = {
    backupSuffix: stringValue(value.backupSuffix) ?? defaults.backupSuffix,
    providerName: stringValue(value.providerName) ?? defaults.providerName,
    wireApi: stringValue(value.wireApi) ?? defaults.wireApi,
    supportsWebSockets: readBooleanConfig(value.supportsWebSockets, "localCodex.supportsWebSockets", defaults.supportsWebSockets),
    responsesWebSocketsV2: readBooleanConfig(value.responsesWebSocketsV2, "localCodex.responsesWebSocketsV2", defaults.responsesWebSocketsV2),
    responsesSmokeModel: stringValue(value.responsesSmokeModel) ?? defaults.responsesSmokeModel,
  };
  if (!/^[A-Za-z0-9._-]+$/u.test(config.backupSuffix)) throw new Error(`${codexPoolConfigPath}.localCodex.backupSuffix has an unsupported format`);
  validateProxyName(config.providerName, "localCodex.providerName");
  validateProxyName(config.wireApi, "localCodex.wireApi");
  validateModelName(config.responsesSmokeModel, "localCodex.responsesSmokeModel");
  return config;
}

function validateCodexFileName(value: string, key: string): void {
  if (!/^(config\.toml|config\.toml\.[A-Za-z0-9._-]+|auth\.json|auth\.json\.[A-Za-z0-9._-]+)$/u.test(value)) {
    throw new Error(`${codexPoolConfigPath}.${key} has an unsupported file name`);
  }
}

function validateProxyName(value: string, key: string): void {
  if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`);
}

function validateModelName(value: string, key: string): void {
  if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported model name`);
}

function validatePublicHostname(value: string, key: string): void {
  if (value.length > 253 || !/^[A-Za-z0-9.-]+$/u.test(value) || value.startsWith(".") || value.endsWith(".")) {
    throw new Error(`${codexPoolConfigPath}.${key} has an unsupported hostname format`);
  }
}

function validatePort(value: number, key: string): void {
  if (!Number.isInteger(value) || value < 1 || value > 65535) throw new Error(`${codexPoolConfigPath}.${key} must be an integer port`);
}

function readAuthAPIKey(authPath: string): { apiKey: string | null; shape: string } {
  if (!existsSync(authPath)) return { apiKey: null, shape: "missing" };
  const parsed = JSON.parse(readFileSync(authPath, "utf8")) as unknown;
  if (!isRecord(parsed)) return { apiKey: null, shape: "non-object" };
  const value = parsed.OPENAI_API_KEY;
  const shape = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
  if (typeof value === "string" && value.length > 0) return { apiKey: value, shape };
  return { apiKey: null, shape };
}

function uniqueAccountName(profile: string, seen: Set<string>): string {
  const normalized = profile
    .toLowerCase()
    .replace(/[^a-z0-9._-]+/gu, "-")
    .replace(/^-+|-+$/gu, "") || "default";
  let candidate = `unidesk-codex-${normalized}`;
  let counter = 2;
  while (seen.has(candidate)) {
    candidate = `unidesk-codex-${normalized}-${counter}`;
    counter += 1;
  }
  seen.add(candidate);
  return candidate;
}

function redactProfile(profile: CodexProfile): Record<string, unknown> {
  return {
    profile: profile.profile,
    accountName: profile.accountName,
    configFile: profile.configFile,
    authFile: profile.authFile,
    provider: profile.provider || null,
    baseUrl: profile.baseUrl || null,
    wireApi: profile.wireApi,
    model: profile.model,
    envKey: profile.envKey,
    apiKeySource: profile.apiKeySource,
    openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
    upstreamUserAgent: profile.upstreamUserAgent,
    priority: profile.priority,
    capacity: profile.capacity,
    loadFactor: profile.loadFactor,
    tempUnschedulable: tempUnschedulableSummary(profile.tempUnschedulable),
    apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0,
    apiKeyBytes: profile.apiKey === null ? 0 : Buffer.byteLength(profile.apiKey, "utf8"),
    apiKeyFingerprint: profile.apiKey === null ? null : fingerprint(profile.apiKey),
    authOpenAIKeyShape: profile.authOpenAIKeyShape,
    ok: profile.ok,
    error: profile.error,
    valuesPrinted: false,
  };
}

function compactProfile(profile: CodexProfile): Record<string, unknown> {
  return {
    profile: profile.profile,
    accountName: profile.accountName,
    provider: profile.provider || null,
    model: profile.model,
    priority: profile.priority,
    capacity: profile.capacity,
    loadFactor: profile.loadFactor,
    tempUnschedulableEnabled: profile.tempUnschedulable.enabled && profile.tempUnschedulable.rules.length > 0,
    tempUnschedulableRuleCount: profile.tempUnschedulable.enabled ? profile.tempUnschedulable.rules.length : 0,
    apiKeyPresent: profile.apiKey !== null && profile.apiKey.length > 0,
    ok: profile.ok,
    error: profile.error,
    valuesPrinted: false,
  };
}

export function renderSub2ApiTempUnschedulableCredentials(policy: CodexTempUnschedulablePolicy): Record<string, unknown> {
  return {
    temp_unschedulable_enabled: policy.enabled && policy.rules.length > 0,
    temp_unschedulable_rules: policy.enabled
      ? policy.rules.map((rule) => ({
        error_code: rule.statusCode,
        keywords: [...rule.keywords],
        duration_minutes: rule.durationMinutes,
        description: rule.description ?? "",
      }))
      : [],
  };
}

function tempUnschedulableSummary(policy: CodexTempUnschedulablePolicy): Record<string, unknown> {
  return {
    enabled: policy.enabled && policy.rules.length > 0,
    ruleCount: policy.enabled ? policy.rules.length : 0,
    statusCodes: policy.enabled ? policy.rules.map((rule) => rule.statusCode) : [],
  };
}

function codexPoolConfigSummary(pool: CodexPoolConfig): Record<string, unknown> {
  const accountCapacityTotal = desiredAccountCapacityTotal(pool);
  return {
    groupName: pool.groupName,
    apiKeyName: pool.apiKeyName,
    apiKeySecretName: pool.apiKeySecretName,
    apiKeySecretKey: pool.apiKeySecretKey,
    minOwnerBalanceUsd: pool.minOwnerBalanceUsd,
    minOwnerConcurrency: pool.minOwnerConcurrency,
    minOwnerConcurrencySource: pool.minOwnerConcurrencySource,
    accountCapacityTotal,
    defaultAccountPriority: pool.defaultAccountPriority,
    defaultAccountCapacity: pool.defaultAccountCapacity,
    defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
    defaultTempUnschedulable: tempUnschedulableSummary(pool.defaultTempUnschedulable),
    profileCount: pool.profiles.length,
    publicExposure: publicExposureSummary(pool),
    localCodex: pool.localCodex,
    sentinel: codexPoolSentinelSummary(pool.sentinel),
    disclosure: {
      full: "bun scripts/cli.ts platform-infra sub2api codex-pool plan --full",
    },
    valuesPrinted: false,
  };
}

function compactText(value: unknown, limit = 260): string {
  if (typeof value !== "string") return "";
  return value.length > limit ? `${value.slice(0, limit)}…` : value;
}

function recordArray(value: unknown): Record<string, unknown>[] {
  return Array.isArray(value) ? value.filter(isRecord) : [];
}

function pickSummaryFields(item: Record<string, unknown>, keys: string[]): Record<string, unknown> {
  const result: Record<string, unknown> = {};
  for (const key of keys) {
    if (item[key] !== undefined) result[key] = item[key];
  }
  return result;
}

function compactStatusBlock(block: unknown, keys: string[]): Record<string, unknown> | null {
  if (!isRecord(block)) return null;
  const items = recordArray(block.items);
  const attentionItems = items
    .filter((item) => item.ok === false)
    .map((item) => pickSummaryFields(item, keys));
  return {
    ok: block.ok,
    desired: block.desired,
    missing: block.missing,
    mismatched: block.mismatched,
    totals: block.totals,
    itemCount: items.length,
    attentionItems,
    valuesPrinted: false,
  };
}

function compactTempUnschedulableStatus(block: unknown): Record<string, unknown> | null {
  if (!isRecord(block)) return null;
  const items = recordArray(block.items);
  const compactItems = items.map((item) => {
    const result = pickSummaryFields(item, [
      "accountName",
      "accountId",
      "expectedEnabled",
      "runtimeEnabled",
      "expectedRuleCount",
      "runtimeRuleCount",
      "status",
      "schedulable",
      "tempUnschedulableUntil",
      "tempUnschedulableSet",
      "ok",
    ]);
    if (item.tempUnschedulableReasonPreview !== undefined) {
      result.tempUnschedulableReason = tempUnschedulableReasonSummary(item.tempUnschedulableReasonPreview);
    }
    return result;
  });
  const frozenItems = compactItems.filter((item) => item.tempUnschedulableSet === true);
  const focusedFrozenItems = uniqueByAccountName([
    ...frozenItems.filter((item) => isRecord(item.tempUnschedulableReason) && item.tempUnschedulableReason.statusCode === 400),
    ...frozenItems.filter((item) => item.schedulable === false),
    ...frozenItems,
  ]).slice(0, 6);
  return {
    ok: block.ok,
    desired: block.desired,
    enabledCount: Array.isArray(block.enabled) ? block.enabled.length : undefined,
    missing: block.missing,
    mismatched: block.mismatched,
    itemCount: compactItems.length,
    frozenCount: frozenItems.length,
    frozen: focusedFrozenItems,
    manuallyUnschedulable: compactItems.filter((item) => item.schedulable === false && item.tempUnschedulableSet !== true),
    valuesPrinted: false,
  };
}

function tempUnschedulableReasonSummary(value: unknown): Record<string, unknown> {
  const reason = compactText(value, 180);
  const statusCodeMatch = reason.match(/"status_code":(\d{3})/u) ?? reason.match(/OpenAI\s+(\d{3})/u) ?? reason.match(/\((\d{3})\)/u);
  const keywordMatch = reason.match(/"matched_keyword":"([^"]+)"/u);
  const summary: Record<string, unknown> = {
    statusCode: statusCodeMatch ? Number(statusCodeMatch[1]) : undefined,
    matchedKeyword: keywordMatch ? keywordMatch[1] : undefined,
  };
  if (!statusCodeMatch || !keywordMatch) summary.preview = compactText(reason, 120);
  return summary;
}

function uniqueByAccountName(items: Record<string, unknown>[]): Record<string, unknown>[] {
  const seen = new Set<unknown>();
  const result: Record<string, unknown>[] = [];
  for (const item of items) {
    const key = item.accountName ?? item.accountId;
    if (seen.has(key)) continue;
    seen.add(key);
    result.push(item);
  }
  return result;
}

function compactRuntimeCapability(block: unknown): unknown {
  if (!isRecord(block)) return block;
  const probe = isRecord(block.probe) ? block.probe : {};
  const logEvidence = isRecord(probe.logEvidence) ? probe.logEvidence : {};
  const accountState = isRecord(probe.accountState) ? probe.accountState : {};
  const resources = isRecord(block.resources) ? block.resources : {};
  const requirement = isRecord(block.requirement) ? block.requirement : {};
  return {
    ok: block.ok,
    required: block.required,
    capability: block.capability,
    outcome: block.outcome,
    requirement: Object.keys(requirement).length === 0 ? undefined : {
      statusCode: requirement.statusCode,
      probeKeyword: requirement.probeKeyword,
      durationMinutes: requirement.durationMinutes,
      sourceAccountName: requirement.sourceAccountName,
    },
    probe: Object.keys(probe).length === 0 ? undefined : {
      requestId: probe.requestId,
      durationMs: probe.durationMs,
      httpStatus: probe.httpStatus,
      responseOk: probe.responseOk,
      accountState: Object.keys(accountState).length === 0 ? undefined : {
        accountId: accountState.accountId,
        status: accountState.status,
        schedulable: accountState.schedulable,
        tempUnschedulableUntil: accountState.tempUnschedulableUntil,
        tempUnschedulableSet: accountState.tempUnschedulableSet,
        tempUnschedulableReasonPreview: compactText(accountState.tempUnschedulableReasonPreview, 180),
      },
      logEvidence: Object.keys(logEvidence).length === 0 ? undefined : {
        matchedLogLineCount: logEvidence.matchedLogLineCount,
        failovers: logEvidence.failovers,
        final: logEvidence.final,
      },
      bodyPreview: probe.responseOk === false ? compactText(probe.bodyPreview, 240) : undefined,
    },
    resources: Object.keys(resources).length === 0 ? undefined : {
      groupId: resources.groupId,
      accountId: resources.accountId,
      badAccountId: resources.badAccountId,
      goodAccountId: resources.goodAccountId,
      apiKeyId: resources.apiKeyId,
      valuesPrinted: false,
    },
    message: block.message,
    valuesPrinted: false,
  };
}

function compactGatewayModels(block: unknown): unknown {
  if (!isRecord(block)) return block;
  return pickSummaryFields(block, [
    "ok",
    "httpStatus",
    "modelCount",
    "method",
    "serviceDns",
    "valuesPrinted",
  ]);
}

function compactGatewayResponses(block: unknown): unknown {
  if (!isRecord(block)) return block;
  const evidence = isRecord(block.evidence) ? block.evidence : {};
  return {
    ok: block.ok,
    degraded: block.degraded,
    outcome: block.outcome,
    httpStatus: block.httpStatus,
    method: block.method,
    model: block.model,
    requestId: block.requestId,
    durationMs: block.durationMs,
    outputTextPreview: block.outputTextPreview,
    evidence: Object.keys(evidence).length === 0 ? undefined : {
      matchedLogLineCount: evidence.matchedLogLineCount,
      failoverCount: Array.isArray(evidence.failovers) ? evidence.failovers.length : undefined,
      failovers: Array.isArray(evidence.failovers) ? evidence.failovers.slice(-5) : undefined,
      final: evidence.final,
    },
    valuesPrinted: false,
  };
}

function compactGatewayResponsesRecent(block: unknown): unknown {
  if (!isRecord(block)) return block;
  return {
    ok: block.ok,
    degraded: block.degraded,
    window: block.window,
    tailLines: block.tailLines,
    failoverCount: block.failoverCount,
    forwardFailureCount: block.forwardFailureCount,
    finalErrorCount: block.finalErrorCount,
    slowFinalErrorCount: block.slowFinalErrorCount,
    contextCanceledCount: block.contextCanceledCount,
    ignoredProbeNoiseCount: block.ignoredProbeNoiseCount,
    recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined,
    recentForwardFailures: Array.isArray(block.recentForwardFailures) ? block.recentForwardFailures.slice(-4).reverse().map((item) => ({
      ...item,
      errorPreview: compactText(item.errorPreview, 220),
    })) : undefined,
    recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-3).reverse() : undefined,
    recentSlowFinalErrors: Array.isArray(block.recentSlowFinalErrors) ? block.recentSlowFinalErrors.slice(-3).reverse() : undefined,
    recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-3).reverse() : undefined,
    logsExitCode: block.logsExitCode,
    valuesPrinted: false,
  };
}

function compactGatewayCompactRecent(block: unknown): unknown {
  if (!isRecord(block)) return block;
  return {
    ok: block.ok,
    degraded: block.degraded,
    window: block.window,
    tailLines: block.tailLines,
    failureCount: block.failureCount,
    successCount: block.successCount,
    failoverCount: block.failoverCount,
    finalErrorCount: block.finalErrorCount,
    contextCanceledCount: block.contextCanceledCount,
    recentFailures: Array.isArray(block.recentFailures) ? block.recentFailures.slice(-2).reverse() : undefined,
    recentSuccesses: Array.isArray(block.recentSuccesses) ? block.recentSuccesses.slice(-2).reverse() : undefined,
    recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined,
    recentFinalErrors: Array.isArray(block.recentFinalErrors) ? block.recentFinalErrors.slice(-2).reverse() : undefined,
    recentContextCanceled: Array.isArray(block.recentContextCanceled) ? block.recentContextCanceled.slice(-2).reverse() : undefined,
    logsExitCode: block.logsExitCode,
    valuesPrinted: false,
  };
}

function compactSentinelStatus(block: unknown): unknown {
  if (!isRecord(block)) return block;
  const runtime = isRecord(block.runtime) ? block.runtime : block;
  const desired = isRecord(runtime.desired) ? runtime.desired : {};
  const cronJob = isRecord(runtime.cronJob) ? runtime.cronJob : {};
  const secret = isRecord(runtime.secret) ? runtime.secret : {};
  const configMap = isRecord(runtime.configMap) ? runtime.configMap : {};
  const state = isRecord(runtime.state) ? runtime.state : {};
  const freezeReassert = isRecord(block.freezeReassert) ? block.freezeReassert : {};
  return {
    ok: block.ok,
    action: block.action,
    desired: {
      monitorEnabled: desired.monitorEnabled,
      actionsEnabled: desired.actionsEnabled,
      schedule: desired.schedule,
      cronJobName: desired.cronJobName,
      configMapName: desired.configMapName,
      credentialsSecretName: desired.credentialsSecretName,
      stateConfigMapName: desired.stateConfigMapName,
    },
    cronJob: {
      exists: cronJob.exists,
      schedule: cronJob.schedule,
      suspend: cronJob.suspend,
      lastScheduleTime: cronJob.lastScheduleTime,
      active: cronJob.active,
      error: cronJob.error,
    },
    secret: {
      exists: secret.exists,
      profileSecretPresent: secret.profileSecretPresent,
      valuesPrinted: false,
      error: secret.error,
    },
    configMap: {
      exists: configMap.exists,
      configPresent: configMap.configPresent,
      runnerPresent: configMap.runnerPresent,
      error: configMap.error,
    },
    state: {
      exists: state.exists,
      accountCount: state.accountCount,
      quarantinedCount: state.quarantinedCount,
      quarantined: state.quarantined,
      recentAccounts: state.recentAccounts,
      lastRun: state.lastRun,
      error: state.error,
    },
    freezeReassert: Object.keys(freezeReassert).length > 0 ? {
      ok: freezeReassert.ok,
      skipped: freezeReassert.skipped,
      reason: freezeReassert.reason,
      itemCount: freezeReassert.itemCount,
      attentionItems: freezeReassert.attentionItems,
    } : undefined,
    valuesPrinted: false,
  };
}

function compactSentinelProbeResult(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
  if (parsed === null) return null;
  const probe = isRecord(parsed.probe) ? parsed.probe : {};
  const summary = isRecord(probe.summary) ? probe.summary : {};
  const state = isRecord(parsed.sentinelState) ? parsed.sentinelState : {};
  return {
    ok: parsed.ok,
    mode: parsed.mode,
    namespace: parsed.namespace,
    job: parsed.job,
    requestedAccounts: parsed.requestedAccounts,
    summary: {
      at: summary.at,
      monitorEnabled: summary.monitorEnabled,
      actionsEnabled: summary.actionsEnabled,
      selected: summary.selected,
      okCount: summary.okCount,
      mismatchCount: summary.mismatchCount,
      markerMismatchCount: summary.markerMismatchCount,
      transportFailureCount: summary.transportFailureCount,
      actionsTaken: summary.actionsTaken,
      selection: summary.selection,
    },
    results: recordArray(probe.results).map((item) => pickSummaryFields(item, [
      "accountName",
      "purpose",
      "ok",
      "markerMatched",
      "httpStatus",
      "durationMs",
      "usage",
      "outputHash",
      "outputPreview",
      "responseBodyHash",
      "responseBodyPreview",
      "error",
      "errorDetails",
      "failureKind",
      "sdk",
      "requestShape",
    ])),
    actions: recordArray(probe.actions).map((item) => pickSummaryFields(item, [
      "accountName",
      "taken",
      "type",
      "error",
    ])),
    sentinelState: {
      quarantined: state.quarantined,
      recentAccounts: state.recentAccounts,
      lastRun: state.lastRun,
    },
    valuesPrinted: false,
  };
}

function renderedCliResult(ok: boolean, command: string, renderedText: string): RenderedCliResult {
  return { ok, command, renderedText, contentType: "text/plain" };
}

function renderSentinelReport(
  parsed: Record<string, unknown> | null,
  context: { events: number; full: boolean; remote: Record<string, unknown> },
): string {
  if (parsed === null) {
    return [
      "SUB2API SENTINEL REPORT unavailable",
      `remote_exit=${context.remote.exitCode ?? "?"} stdout_bytes=${context.remote.stdoutBytes ?? "?"} stderr_bytes=${context.remote.stderrBytes ?? "?"}`,
      stringValue(context.remote.stderrTail) ?? stringValue(context.remote.stdoutTail) ?? "",
    ].filter(Boolean).join("\n");
  }
  const metadata = isRecord(parsed.metadata) ? parsed.metadata : {};
  const cronJob = isRecord(parsed.cronJob) ? parsed.cronJob : {};
  const summary = isRecord(parsed.summary) ? parsed.summary : {};
  const accounts = recordArray(parsed.accounts);
  const runs = recordArray(parsed.runs);
  const globalLedger = isRecord(parsed.globalLedger) ? parsed.globalLedger : {};
  const lines: string[] = [];
  lines.push([
    "SUB2API SENTINEL",
    `ok=${parsed.ok === true ? "true" : "false"}`,
    `accounts=${summary.accountCount ?? accounts.length}`,
    `quarantined=${summary.quarantinedCount ?? "?"}`,
    `history=${summary.historyCount ?? runs.length}`,
    `window=${formatWindow(summary.historyFrom, summary.historyTo)}`,
  ].join(" "));
  lines.push([
    "CRON",
    `schedule=${cronJob.schedule ?? "-"}`,
    `last=${shortIso(cronJob.lastScheduleTime)}`,
    `active=${cronJob.active ?? "-"}`,
    `state=${metadata.namespace ?? namespace}/${metadata.stateConfigMapName ?? "-"}`,
    `ledger=req:${globalLedger.requestCount ?? 0} tok:${formatNumber(globalLedger.totalTokens)} cost:$${formatCost(globalLedger.estimatedCostUsd)}`,
  ].join(" "));
  lines.push("");
  lines.push("ACCOUNTS");
  lines.push(renderTable([
    ["ACCOUNT", "STATE", "Q", "F_MIN", "S_MIN", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
    ...accounts.map((account) => [
      stringValue(account.account) ?? "-",
      stringValue(account.status) ?? "-",
      account.quarantineActive === true ? "Y" : "-",
      textValue(account.freezeIntervalMin),
      textValue(account.successIntervalMin),
      textValue(account.probeCount),
      shortIso(account.lastProbeAt),
      textValue(account.lastHttp),
      account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-",
      shorten(stringValue(account.lastFailureKind) ?? "-", 20),
      shorten(stringValue(account.lastAction) ?? "-", 16),
      shortIso(account.nextProbeAfter),
      textValue(account.observedLastToNextMin),
    ]),
  ]));
  if (runs.length > 0 || context.full) {
    lines.push("");
    lines.push(`RUNS last=${Math.min(context.events, runs.length)}`);
    lines.push(renderTable([
      ["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "REASSERT"],
      ...runs.slice(-context.events).map((run) => [
        shortIso(run.at),
        textValue(run.selected),
        textValue(run.due),
        textValue(run.ok),
        textValue(run.mismatch),
        textValue(run.transportFailures),
        textValue(run.actionsTaken),
        textValue(run.reasserts),
      ]),
    ]));
  }
  lines.push("");
  lines.push("LEGEND Q=quarantined M=marker matched F_MIN=freeze interval S_MIN=success interval OBS_MIN=last probe to next probe minutes TF=transport failures");
  lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
  return lines.join("\n");
}

function renderTable(rows: string[][]): string {
  if (rows.length === 0) return "";
  const widths: number[] = [];
  for (const row of rows) {
    row.forEach((cell, index) => {
      widths[index] = Math.max(widths[index] ?? 0, displayWidth(cell));
    });
  }
  return rows.map((row) => row.map((cell, index) => padRight(cell, widths[index] ?? 0)).join("  ").trimEnd()).join("\n");
}

function padRight(value: string, width: number): string {
  const pad = width - displayWidth(value);
  return pad <= 0 ? value : `${value}${" ".repeat(pad)}`;
}

function displayWidth(value: string): number {
  return [...value].reduce((width, char) => width + (char.charCodeAt(0) > 0x7f ? 2 : 1), 0);
}

function formatWindow(from: unknown, to: unknown): string {
  const left = shortIso(from);
  const right = shortIso(to);
  return left === "-" && right === "-" ? "-" : `${left}..${right}`;
}

function shortIso(value: unknown): string {
  const text = stringValue(value);
  if (text === null) return "-";
  return text.replace(/^\d{4}-/u, "").replace(/:00Z$/u, "Z").replace("T", " ");
}

function textValue(value: unknown): string {
  if (value === null || value === undefined || value === "") return "-";
  if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Math.round(value * 10) / 10);
  if (typeof value === "boolean") return value ? "true" : "false";
  return String(value);
}

function shorten(value: string, maxChars: number): string {
  return value.length <= maxChars ? value : `${value.slice(0, Math.max(0, maxChars - 1))}…`;
}

function formatNumber(value: unknown): string {
  const num = numberValue(value);
  if (num === null) return "0";
  if (Math.abs(num) >= 1_000_000) return `${(num / 1_000_000).toFixed(1)}M`;
  if (Math.abs(num) >= 1_000) return `${(num / 1_000).toFixed(1)}K`;
  return String(Math.round(num));
}

function formatCost(value: unknown): string {
  const num = numberValue(value);
  if (num === null) return "0.0000";
  return num.toFixed(4);
}

function codexPoolValidationSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
  if (parsed === null) return null;
  const validation = isRecord(parsed.validation) ? parsed.validation : {};
  const runtimeCapabilities = isRecord(parsed.runtimeCapabilities) ? parsed.runtimeCapabilities : {};
  return {
    ok: parsed.ok,
    degraded: parsed.degraded,
    mode: parsed.mode,
    namespace: parsed.namespace,
    serviceDns: parsed.serviceDns,
    appPod: parsed.appPod,
    admin: parsed.admin,
    apiKey: parsed.apiKey,
    ownerBalance: parsed.ownerBalance,
    ownerConcurrency: parsed.ownerConcurrency,
    capacity: compactStatusBlock(parsed.capacity, ["accountName", "accountId", "expectedCapacity", "runtimeConcurrency", "priority", "status", "schedulable", "ok"]),
    loadFactor: compactStatusBlock(parsed.loadFactor, ["accountName", "accountId", "expectedLoadFactor", "runtimeLoadFactor", "priority", "status", "schedulable", "ok"]),
    webSocketsV2: compactStatusBlock(parsed.webSocketsV2, ["accountName", "accountId", "expectedMode", "runtimeMode", "runtimeEnabled", "status", "schedulable", "ok"]),
    tempUnschedulable: compactTempUnschedulableStatus(parsed.tempUnschedulable),
    sentinel: compactSentinelStatus(parsed.sentinel),
    runtimeCapabilities: {
      ok: runtimeCapabilities.ok,
      runtimeImage: runtimeCapabilities.runtimeImage,
      successBodyReclassification: compactRuntimeCapability(runtimeCapabilities.successBodyReclassification),
      modelRouting400Failover: compactRuntimeCapability(runtimeCapabilities.modelRouting400Failover),
      valuesPrinted: false,
    },
    validation: {
      gatewayModels: compactGatewayModels(validation.gatewayModels),
      gatewayResponses: compactGatewayResponses(validation.gatewayResponses),
      gatewayResponsesRecent: compactGatewayResponsesRecent(validation.gatewayResponsesRecent),
      gatewayCompactRecent: compactGatewayCompactRecent(validation.gatewayCompactRecent),
    },
    disclosure: {
      full: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --full",
      raw: "bun scripts/cli.ts platform-infra sub2api codex-pool validate --raw",
    },
    valuesPrinted: false,
  };
}

function codexPoolSyncSummary(parsed: Record<string, unknown> | null): Record<string, unknown> | null {
  const base = codexPoolValidationSummary(parsed);
  if (parsed === null || base === null) return base;
  const accounts = isRecord(parsed.accounts) ? parsed.accounts : {};
  const compactAccounts = recordArray(accounts.items).map((item) => pickSummaryFields(item, [
    "accountName",
    "accountId",
    "action",
    "status",
    "runtimeSchedulable",
    "priority",
    "capacity",
    "loadFactor",
    "tempUnschedulableConfigured",
    "tempUnschedulableRuleCount",
    "ok",
  ]));
  return {
    ...base,
    pool: parsed.pool,
    accounts: {
      desired: accounts.desired,
      created: accounts.created,
      updated: accounts.updated,
      pruned: accounts.pruned,
      pruneMode: accounts.pruneMode,
      itemCount: compactAccounts.length,
      attentionItems: compactAccounts.filter((item) => item.ok === false || item.runtimeSchedulable === false),
      prunedItems: accounts.prunedItems,
      processControl: accounts.processControl,
      valuesPrinted: false,
    },
    disclosure: {
      full: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --full",
      raw: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --raw",
      pruneRemoved: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm --prune-removed",
    },
  };
}

function poolTarget(pool = readCodexPoolConfig()): Record<string, unknown> {
  return {
    route: g14K3sRoute,
    namespace,
    service: serviceName,
    serviceDns,
    configPath: codexPoolConfigPath,
    groupName: pool.groupName,
    apiKeyName: pool.apiKeyName,
    apiKeySecret: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`,
    minOwnerConcurrency: pool.minOwnerConcurrency,
    minOwnerConcurrencySource: pool.minOwnerConcurrencySource,
    accountCapacityTotal: desiredAccountCapacityTotal(pool),
    defaultAccountPriority: pool.defaultAccountPriority,
    defaultAccountCapacity: pool.defaultAccountCapacity,
    defaultAccountLoadFactor: pool.defaultAccountLoadFactor,
    sentinel: {
      monitorEnabled: pool.sentinel.monitor.enabled,
      actionsEnabled: pool.sentinel.actions.enabled,
      cronJobName: pool.sentinel.cronJobName,
      stateConfigMapName: pool.sentinel.stateConfigMapName,
    },
    valuesPrinted: false,
  };
}

function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProfileSecret[] {
  return profiles
    .filter((profile) => profile.ok && profile.apiKey !== null && profile.apiKey.length > 0)
    .map((profile) => ({
      accountName: profile.accountName,
      profile: profile.profile,
      baseUrl: profile.baseUrl,
      apiKey: profile.apiKey ?? "",
      upstreamUserAgent: profile.upstreamUserAgent,
    }));
}

function publicExposureSummary(pool: CodexPoolConfig): Record<string, unknown> {
  return {
    enabled: pool.publicExposure.enabled,
    proxyName: pool.publicExposure.proxyName,
    namespace,
    configMapName: pool.publicExposure.configMapName,
    deploymentName: pool.publicExposure.deploymentName,
    frpcImage: pool.publicExposure.frpcImage,
    frps: {
      serverAddr: pool.publicExposure.serverAddr,
      serverPort: pool.publicExposure.serverPort,
      remotePort: pool.publicExposure.remotePort,
      masterConfigPath: pool.publicExposure.masterFrps.configPath,
      masterContainerName: pool.publicExposure.masterFrps.containerName,
    },
    caddy: {
      enabled: pool.publicExposure.masterCaddy.enabled,
      domain: pool.publicExposure.masterCaddy.domain,
      configPath: pool.publicExposure.masterCaddy.configPath,
      serviceName: pool.publicExposure.masterCaddy.serviceName,
      upstreamBaseUrl: pool.publicExposure.masterCaddy.upstreamBaseUrl,
      responseHeaderTimeoutSeconds: pool.publicExposure.masterCaddy.responseHeaderTimeoutSeconds,
    },
    upstream: {
      localIP: pool.publicExposure.localIP,
      localPort: pool.publicExposure.localPort,
      serviceDns,
    },
    urls: {
      publicBaseUrl: pool.publicExposure.publicBaseUrl,
      masterBaseUrl: pool.publicExposure.masterBaseUrl,
    },
  };
}

async function applyMasterFrpsAllowPort(pool: CodexPoolConfig): Promise<Record<string, unknown>> {
  const path = pool.publicExposure.masterFrps.configPath;
  const port = pool.publicExposure.remotePort;
  const container = pool.publicExposure.masterFrps.containerName;
  if (!existsSync(path)) {
    return { ok: false, error: "master-frps-config-missing", path, valuesPrinted: false };
  }
  const before = readFileSync(path, "utf8");
  const alreadyAllowed = frpsAllowPortExists(before, port);
  let backupPath: string | null = null;
  let action = "kept-existing";
  if (!alreadyAllowed) {
    const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z");
    backupPath = `${path}.bak-sub2api-${stamp}`;
    copyFileSync(path, backupPath);
    const next = before.replace(/\s*$/u, "") + `\n\n[[allowPorts]]\nstart = ${port}\nend = ${port}\n`;
    writeFileSync(path, next, "utf8");
    chmodSync(path, statSync(backupPath).mode & 0o777);
    action = "added-allow-port";
  }
  const restart = alreadyAllowed ? null : Bun.spawnSync(["docker", "restart", container]);
  const inspect = Bun.spawnSync(["docker", "inspect", "-f", "{{.State.Running}}", container]);
  const ok = alreadyAllowed || (restart?.exitCode === 0 && inspect.exitCode === 0 && String(inspect.stdout).trim() === "true");
  return {
    ok,
    action,
    path,
    backupPath,
    remotePort: port,
    containerName: container,
    restart: restart === null ? null : {
      exitCode: restart.exitCode,
      stdoutTail: Buffer.from(restart.stdout).toString("utf8").slice(-1000),
      stderrTail: Buffer.from(restart.stderr).toString("utf8").slice(-1000),
    },
    inspect: {
      exitCode: inspect.exitCode,
      running: String(inspect.stdout).trim() === "true",
      stderrTail: Buffer.from(inspect.stderr).toString("utf8").slice(-1000),
    },
    valuesPrinted: false,
  };
}


async function applyMasterCaddySite(pool: CodexPoolConfig): Promise<Record<string, unknown>> {
  const caddy = pool.publicExposure.masterCaddy;
  if (!caddy.enabled) return { ok: true, action: "disabled-by-yaml", valuesPrinted: false };
  const path = caddy.configPath;
  if (!existsSync(path)) return { ok: false, error: "master-caddy-config-missing", path, valuesPrinted: false };
  const before = readFileSync(path, "utf8");
  const desiredBlock = renderCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds);
  const existing = caddySiteBlock(before, caddy.domain);
  const alreadyConfigured = existing === desiredBlock;
  let backupPath: string | null = null;
  let action = "kept-existing";
  if (!alreadyConfigured) {
    const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z");
    backupPath = `${path}.bak-sub2api-https-${stamp}`;
    copyFileSync(path, backupPath);
    const next = existing === null
      ? `${before.replace(/\s*$/u, "")}\n\n${desiredBlock}\n`
      : before.replace(existing, desiredBlock);
    writeFileSync(path, next, "utf8");
    chmodSync(path, statSync(backupPath).mode & 0o777);
    action = existing === null ? "added-site" : "updated-site";
  }
  const validate = Bun.spawnSync(["caddy", "validate", "--config", path, "--adapter", "caddyfile"]);
  let reload: ReturnType<typeof Bun.spawnSync> | null = null;
  if (validate.exitCode === 0 && !alreadyConfigured) {
    reload = Bun.spawnSync(["systemctl", "reload", caddy.serviceName]);
  }
  const active = Bun.spawnSync(["systemctl", "is-active", caddy.serviceName]);
  const ok = validate.exitCode === 0 && (reload === null || reload.exitCode === 0) && active.exitCode === 0 && String(active.stdout).trim() === "active";
  return {
    ok,
    action,
    path,
    backupPath,
    domain: caddy.domain,
    upstreamBaseUrl: caddy.upstreamBaseUrl,
    serviceName: caddy.serviceName,
    responseHeaderTimeoutSeconds: caddy.responseHeaderTimeoutSeconds,
    validate: {
      exitCode: validate.exitCode,
      stdoutTail: Buffer.from(validate.stdout).toString("utf8").slice(-1000),
      stderrTail: Buffer.from(validate.stderr).toString("utf8").slice(-2000),
    },
    reload: reload === null ? null : {
      exitCode: reload.exitCode,
      stdoutTail: Buffer.from(reload.stdout).toString("utf8").slice(-1000),
      stderrTail: Buffer.from(reload.stderr).toString("utf8").slice(-1000),
    },
    active: {
      exitCode: active.exitCode,
      stdoutTail: Buffer.from(active.stdout).toString("utf8").slice(-1000),
      stderrTail: Buffer.from(active.stderr).toString("utf8").slice(-1000),
    },
    valuesPrinted: false,
  };
}

export function renderCaddySiteBlock(domain: string, upstreamBaseUrl: string, responseHeaderTimeoutSeconds = 180): string {
  const upstream = new URL(upstreamBaseUrl);
  const upstreamHost = `${upstream.hostname}${upstream.port ? `:${upstream.port}` : ""}`;
  return `${domain} {
    encode zstd gzip
    reverse_proxy ${upstreamHost} {
        header_up Host {host}
        header_up X-Real-IP {remote_host}
        transport http {
            dial_timeout 5s
            response_header_timeout ${responseHeaderTimeoutSeconds}s
        }
    }
}`;
}

function caddySiteBlock(text: string, domain: string): string | null {
  const startMatch = new RegExp(`(^|\\n)${escapeRegExp(domain)}\\s*\\{`, "u").exec(text);
  if (startMatch === null) return null;
  const start = startMatch.index + (startMatch[1] === "\n" ? 1 : 0);
  const open = text.indexOf("{", start);
  if (open < 0) return null;
  let depth = 0;
  for (let index = open; index < text.length; index += 1) {
    const ch = text[index];
    if (ch === "{") depth += 1;
    if (ch === "}") {
      depth -= 1;
      if (depth === 0) return text.slice(start, index + 1);
    }
  }
  return null;
}

function frpsAllowPortExists(toml: string, port: number): boolean {
  const sections = toml.split(/(?=\[\[allowPorts\]\])/u);
  return sections.some((section) => {
    if (!section.includes("[[allowPorts]]")) return false;
    const start = section.match(/^\s*start\s*=\s*(\d+)\s*$/mu);
    const end = section.match(/^\s*end\s*=\s*(\d+)\s*$/mu);
    return Number(start?.[1]) === port && Number(end?.[1]) === port;
  });
}

function exposeScript(pool: CodexPoolConfig): string {
  const manifest = frpcManifest(pool);
  const encoded = Buffer.from(manifest, "utf8").toString("base64");
  return `
set -u
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
manifest="$tmp/sub2api-frpc.yaml"
printf '%s' '${encoded}' | base64 -d > "$manifest"
ns_out="$tmp/ns.out"
ns_err="$tmp/ns.err"
apply_out="$tmp/apply.out"
apply_err="$tmp/apply.err"
rollout_out="$tmp/rollout.out"
rollout_err="$tmp/rollout.err"
kubectl get namespace ${namespace} >"$ns_out" 2>"$ns_err"
ns_rc=$?
apply_rc=1
rollout_rc=1
if [ "$ns_rc" -eq 0 ]; then
  kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$apply_out" 2>"$apply_err"
  apply_rc=$?
  if [ "$apply_rc" -eq 0 ]; then
    kubectl -n ${namespace} rollout status deployment/${pool.publicExposure.deploymentName} --timeout=30s >"$rollout_out" 2>"$rollout_err"
    rollout_rc=$?
  else
    printf '%s\\n' 'skipped because apply failed' >"$rollout_err"
  fi
else
  : >"$apply_out"
  printf '%s\\n' 'skipped because namespace is missing' >"$apply_err"
  : >"$rollout_out"
  printf '%s\\n' 'skipped because namespace is missing' >"$rollout_err"
fi
pods_json="$tmp/pods.json"
pods_err="$tmp/pods.err"
kubectl -n ${namespace} get pods -l app.kubernetes.io/name=${pool.publicExposure.deploymentName} -o json >"$pods_json" 2>"$pods_err"
pods_rc=$?
logs_out="$tmp/logs.out"
logs_err="$tmp/logs.err"
kubectl -n ${namespace} logs deployment/${pool.publicExposure.deploymentName} --tail=80 >"$logs_out" 2>"$logs_err"
logs_rc=$?
python3 - "$tmp" "$ns_rc" "$apply_rc" "$rollout_rc" "$pods_rc" "$logs_rc" <<'PY'
import json
import os
import sys

tmp = sys.argv[1]
ns_rc, apply_rc, rollout_rc, pods_rc, logs_rc = [int(value) for value in sys.argv[2:]]

def text(name, limit=4000):
    path = os.path.join(tmp, name)
    try:
        return open(path, encoding="utf-8", errors="replace").read()[-limit:]
    except FileNotFoundError:
        return ""

def load_json(name):
    path = os.path.join(tmp, name)
    try:
        return json.load(open(path, encoding="utf-8"))
    except Exception:
        return None

pods_obj = load_json("pods.json")
pods = []
if isinstance(pods_obj, dict):
    for item in pods_obj.get("items") or []:
        status = item.get("status") or {}
        statuses = status.get("containerStatuses") or []
        pods.append({
            "name": item.get("metadata", {}).get("name"),
            "phase": status.get("phase"),
            "ready": all(cs.get("ready") is True for cs in statuses) if statuses else False,
            "restarts": sum(int(cs.get("restartCount") or 0) for cs in statuses),
        })

logs = text("logs.out", 4000)
payload = {
    "ok": ns_rc == 0 and apply_rc == 0 and rollout_rc == 0 and pods_rc == 0 and len(pods) > 0 and all(p["ready"] for p in pods),
    "namespace": "${namespace}",
    "proxy": {
        "name": "${pool.publicExposure.proxyName}",
        "remotePort": ${JSON.stringify(pool.publicExposure.remotePort)},
        "publicBaseUrl": "${pool.publicExposure.publicBaseUrl}",
        "masterBaseUrl": "${pool.publicExposure.masterBaseUrl}",
        "localIP": "${pool.publicExposure.localIP}",
        "localPort": ${JSON.stringify(pool.publicExposure.localPort)},
    },
    "resources": {
        "configMap": "${pool.publicExposure.configMapName}",
        "deployment": "${pool.publicExposure.deploymentName}",
        "image": "${pool.publicExposure.frpcImage}",
    },
    "steps": {
        "namespace": {"exitCode": ns_rc, "stdout": text("ns.out"), "stderr": text("ns.err")},
        "apply": {"exitCode": apply_rc, "stdout": text("apply.out"), "stderr": text("apply.err")},
        "rollout": {"exitCode": rollout_rc, "stdout": text("rollout.out"), "stderr": text("rollout.err")},
        "pods": {"exitCode": pods_rc, "items": pods, "stderr": text("pods.err")},
        "logs": {
            "exitCode": logs_rc,
            "startProxySuccess": "start proxy success" in logs,
            "stdoutTail": logs,
            "stderrTail": text("logs.err"),
        },
    },
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
sys.exit(0 if payload["ok"] else 1)
PY
`;
}

function frpcManifest(pool: CodexPoolConfig): string {
  return `apiVersion: v1
kind: ConfigMap
metadata:
  name: ${pool.publicExposure.configMapName}
  namespace: ${namespace}
  labels:
    app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
data:
  frpc.toml: |
    serverAddr = "${pool.publicExposure.serverAddr}"
    serverPort = ${pool.publicExposure.serverPort}
    loginFailExit = true

    [[proxies]]
    name = "${pool.publicExposure.proxyName}"
    type = "tcp"
    localIP = "${pool.publicExposure.localIP}"
    localPort = ${pool.publicExposure.localPort}
    remotePort = ${pool.publicExposure.remotePort}
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${pool.publicExposure.deploymentName}
  namespace: ${namespace}
  labels:
    app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
    app.kubernetes.io/part-of: platform-infra
    app.kubernetes.io/managed-by: unidesk
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ${pool.publicExposure.deploymentName}
        app.kubernetes.io/part-of: platform-infra
        app.kubernetes.io/managed-by: unidesk
    spec:
      containers:
        - name: frpc
          image: ${pool.publicExposure.frpcImage}
          imagePullPolicy: IfNotPresent
          args:
            - -c
            - /etc/frp/frpc.toml
          volumeMounts:
            - name: config
              mountPath: /etc/frp
              readOnly: true
      volumes:
        - name: config
          configMap:
            name: ${pool.publicExposure.configMapName}
`;
}

async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig): Promise<{ apiKey: string | null; error: string | null }> {
  const result = await capture(config, g14K3sRoute, ["script"], `
set -u
kubectl -n ${namespace} get secret ${pool.apiKeySecretName} -o json
`);
  if (result.exitCode !== 0) {
    return { apiKey: null, error: `read pool API key secret failed: ${result.stderr.slice(-1000)}` };
  }
  const parsed = parseJsonOutput(result.stdout);
  const data = isRecord(parsed?.data) ? parsed.data : null;
  const encoded = typeof data?.[pool.apiKeySecretKey] === "string" ? data[pool.apiKeySecretKey] : null;
  if (encoded === null) return { apiKey: null, error: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey} missing` };
  try {
    const apiKey = Buffer.from(encoded, "base64").toString("utf8");
    return apiKey.length > 0 ? { apiKey, error: null } : { apiKey: null, error: "decoded API key is empty" };
  } catch (error) {
    return { apiKey: null, error: error instanceof Error ? error.message : String(error) };
  }
}

function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string): Record<string, unknown> {
  const codexDir = join(homedir(), ".codex");
  mkdirSync(codexDir, { recursive: true, mode: 0o700 });
  const configPath = join(codexDir, "config.toml");
  const authPath = join(codexDir, "auth.json");
  const backupConfigPath = join(codexDir, `config.toml.${pool.localCodex.backupSuffix}`);
  const backupAuthPath = join(codexDir, `auth.json.${pool.localCodex.backupSuffix}`);
  if (!existsSync(configPath)) throw new Error(`${configPath} missing`);
  if (!existsSync(authPath)) throw new Error(`${authPath} missing`);
  const configBackupAction = copyIfMissing(configPath, backupConfigPath);
  const authBackupAction = copyIfMissing(authPath, backupAuthPath);
  const currentToml = readFileSync(configPath, "utf8");
  const nextToml = updateCodexConfigToml(currentToml, pool);
  writeFileSync(configPath, nextToml, { encoding: "utf8", mode: 0o600 });
  chmodSync(configPath, 0o600);
  writeFileSync(authPath, `${JSON.stringify({ OPENAI_API_KEY: apiKey }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
  chmodSync(authPath, 0o600);
  return {
    ok: true,
    codexDir,
    config: {
      path: configPath,
      bytes: statSync(configPath).size,
      backupPath: backupConfigPath,
      backupAction: configBackupAction,
      backupBytes: statSync(backupConfigPath).size,
    },
    auth: {
      path: authPath,
      bytes: statSync(authPath).size,
      backupPath: backupAuthPath,
      backupAction: authBackupAction,
      backupBytes: statSync(backupAuthPath).size,
      openaiApiKeyShape: "string",
      openaiApiKeyBytes: Buffer.byteLength(apiKey, "utf8"),
      openaiApiKeyFingerprint: fingerprint(apiKey),
      valuesPrinted: false,
    },
    provider: {
      name: pool.localCodex.providerName,
      baseUrl: codexConsumerBaseUrl(pool),
      wireApi: pool.localCodex.wireApi,
      supportsWebSockets: pool.localCodex.supportsWebSockets,
      responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
    },
    valuesPrinted: false,
  };
}

function copyIfMissing(source: string, target: string): "created" | "kept-existing" {
  if (existsSync(target)) return "kept-existing";
  copyFileSync(source, target);
  chmodSync(target, 0o600);
  return "created";
}

function updateCodexConfigToml(current: string, pool: CodexPoolConfig): string {
  return renderCodexLocalConsumerToml(current, {
    providerName: pool.localCodex.providerName,
    baseUrl: codexConsumerBaseUrl(pool),
    wireApi: pool.localCodex.wireApi,
    supportsWebSockets: pool.localCodex.supportsWebSockets,
    responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
  });
}

function codexConsumerBaseUrl(pool: CodexPoolConfig): string {
  return `${pool.publicExposure.publicBaseUrl.replace(/\/+$/u, "")}/`;
}

export function renderCodexLocalConsumerToml(current: string, options: CodexLocalConsumerTomlOptions): string {
  let next = upsertTopLevelTomlString(current, "model_provider", options.providerName);
  next = upsertProviderSection(next, options);
  next = upsertTomlSectionBoolean(next, "features", "responses_websockets_v2", options.responsesWebSocketsV2);
  return next.endsWith("\n") ? next : `${next}\n`;
}

function upsertTopLevelTomlString(text: string, key: string, value: string): string {
  const lines = text.split(/\r?\n/u);
  const firstSection = lines.findIndex((line) => /^\s*\[/.test(line));
  const end = firstSection === -1 ? lines.length : firstSection;
  for (let index = 0; index < end; index += 1) {
    if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
      lines[index] = `${key} = ${tomlString(value)}`;
      return lines.join("\n");
    }
  }
  lines.splice(0, 0, `${key} = ${tomlString(value)}`);
  return lines.join("\n");
}

function upsertProviderSection(text: string, options: CodexLocalConsumerTomlOptions): string {
  const { providerName, baseUrl, wireApi, supportsWebSockets } = options;
  const header = `[model_providers.${providerName}]`;
  const lines = text.split(/\r?\n/u);
  const start = lines.findIndex((line) => line.trim() === header);
  const canonical = [
    `name = ${tomlString(providerName)}`,
    `base_url = ${tomlString(baseUrl)}`,
    `wire_api = ${tomlString(wireApi)}`,
    "requires_openai_auth = true",
    `supports_websockets = ${tomlBoolean(supportsWebSockets)}`,
  ];
  if (start === -1) {
    const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== "";
    return [...lines, ...(needsBlank ? [""] : []), header, ...canonical].join("\n");
  }
  let end = lines.length;
  for (let index = start + 1; index < lines.length; index += 1) {
    if (/^\s*\[/.test(lines[index])) {
      end = index;
      break;
    }
  }
  const preserved = lines.slice(start + 1, end).filter((line) => {
    return !/^\s*(name|base_url|wire_api|requires_openai_auth|supports_websockets|env_key)\s*=/u.test(line);
  });
  const replacement = [header, ...canonical, ...preserved.filter((line) => line.trim() !== "")];
  return [...lines.slice(0, start), ...replacement, ...lines.slice(end)].join("\n");
}

function upsertTomlSectionBoolean(text: string, sectionName: string, key: string, value: boolean): string {
  const header = `[${sectionName}]`;
  const lines = text.split(/\r?\n/u);
  const start = lines.findIndex((line) => line.trim() === header);
  const assignment = `${key} = ${tomlBoolean(value)}`;
  if (start === -1) {
    const needsBlank = lines.length > 0 && lines[lines.length - 1].trim() !== "";
    return [...lines, ...(needsBlank ? [""] : []), header, assignment].join("\n");
  }
  let end = lines.length;
  for (let index = start + 1; index < lines.length; index += 1) {
    if (/^\s*\[/.test(lines[index])) {
      end = index;
      break;
    }
  }
  for (let index = start + 1; index < end; index += 1) {
    if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
      lines[index] = assignment;
      return lines.join("\n");
    }
  }
  return [...lines.slice(0, start + 1), assignment, ...lines.slice(start + 1)].join("\n");
}

async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: string): Promise<Record<string, unknown>> {
  const probe = await probePublicModels(pool, "with-api-key", apiKey, "public");
  return {
    ...probe,
    ok: probe.ok === true,
    apiKeyFingerprint: fingerprint(apiKey),
    keyPreview: apiKeyPreview(apiKey),
    valuesPrinted: false,
  };
}

async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, base: "master" | "public" = "master"): Promise<Record<string, unknown>> {
  const baseUrl = base === "public" ? pool.publicExposure.publicBaseUrl : pool.publicExposure.masterBaseUrl;
  const url = `${baseUrl.replace(/\/+$/u, "")}/v1/models`;
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), 8000);
  try {
    const response = await fetch(url, {
      method: "GET",
      headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
      signal: controller.signal,
    });
    const body = await response.text();
    let parsed: unknown = null;
    try {
      parsed = body.trim().length > 0 ? JSON.parse(body) : null;
    } catch {
      parsed = null;
    }
    const modelCount = isRecord(parsed) && Array.isArray(parsed.data) ? parsed.data.length : null;
    return {
      ok: response.ok,
      mode,
      method: "GET /v1/models",
      baseUrl,
      httpStatus: response.status,
      modelCount,
      bodyPreview: response.ok ? "" : body.slice(0, 500),
      valuesPrinted: false,
    };
  } catch (error) {
    return {
      ok: false,
      mode,
      method: "GET /v1/models",
      baseUrl,
      httpStatus: 0,
      error: error instanceof Error ? error.message : String(error),
      valuesPrinted: false,
    };
  } finally {
    clearTimeout(timeout);
  }
}

function apiKeyPreview(apiKey: string): string {
  if (apiKey.length <= 14) return "***";
  return `${apiKey.slice(0, 10)}...${apiKey.slice(-4)}`;
}

function tomlString(value: string): string {
  return JSON.stringify(value);
}

function tomlBoolean(value: boolean): string {
  return value ? "true" : "false";
}

function escapeRegExp(value: string): string {
  return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
}

function normalizeBaseUrl(value: string | null): string | null {
  if (value === null) return null;
  const trimmed = value.trim().replace(/\/+$/u, "");
  if (trimmed.length === 0) return null;
  try {
    const parsed = new URL(trimmed);
    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
    return parsed.toString().replace(/\/+$/u, "");
  } catch {
    return null;
  }
}

function stringValue(value: unknown): string | null {
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}

function numberValue(value: unknown): number | null {
  if (typeof value === "number" && Number.isFinite(value)) return value;
  if (typeof value === "string" && value.trim().length > 0) {
    const parsed = Number(value);
    return Number.isFinite(parsed) ? parsed : null;
  }
  return null;
}

function booleanValue(value: unknown): boolean | null {
  if (typeof value === "boolean") return value;
  if (typeof value === "string") {
    if (value.trim() === "true") return true;
    if (value.trim() === "false") return false;
  }
  return null;
}

function validateKubernetesName(value: string, key: string, strictDns: boolean): void {
  const pattern = strictDns ? /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u : /^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u;
  if (!pattern.test(value)) throw new Error(`${codexPoolConfigPath}.${key} has an unsupported format`);
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

function fingerprint(value: string): string {
  return createHash("sha256").update(value).digest("hex").slice(0, 12);
}

function syncScript(payload: unknown, pool: CodexPoolConfig): string {
  const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
  return remotePythonScript("sync", encoded, pool);
}

function validateScript(pool: CodexPoolConfig): string {
  return remotePythonScript("validate", "", pool);
}

function sentinelProbeScript(payload: unknown, pool: CodexPoolConfig): string {
  const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
  return remotePythonScript("sentinel-probe", encoded, pool);
}

function sentinelReportScript(pool: CodexPoolConfig, events: number): string {
  const stateName = pool.sentinel.stateConfigMapName;
  const cronJobName = pool.sentinel.cronJobName;
  return `
set -eu
python3 - <<'PY'
import json
import subprocess
from datetime import datetime, timezone

NAMESPACE = ${JSON.stringify(namespace)}
STATE_NAME = ${JSON.stringify(stateName)}
CRONJOB_NAME = ${JSON.stringify(cronJobName)}
EVENT_LIMIT = ${JSON.stringify(events)}

def run(cmd):
    return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def text(data, limit=2000):
    if isinstance(data, bytes):
        data = data.decode("utf-8", errors="replace")
    return data[-limit:]

def kube_json(args):
    proc = run(["kubectl", *args, "-o", "json"])
    if proc.returncode != 0:
        return None, text(proc.stderr)
    try:
        return json.loads(proc.stdout.decode("utf-8")), None
    except Exception as exc:
        return None, str(exc)

def parse_iso(value):
    if not isinstance(value, str) or not value:
        return None
    try:
        return datetime.fromisoformat(value.replace("Z", "+00:00"))
    except Exception:
        return None

def minutes_between(a, b):
    left = parse_iso(a)
    right = parse_iso(b)
    if left is None or right is None:
        return None
    return round((right - left).total_seconds() / 60, 1)

def day_ledgers(state):
    ledger = {}
    raw = state.get("ledger")
    if isinstance(raw, dict):
        for item in raw.values():
            if not isinstance(item, dict):
                continue
            add_ledger(ledger, item)
    return ledger

def add_ledger(target, source):
    target["inputTokens"] = target.get("inputTokens", 0) + int(source.get("inputTokens") or 0)
    target["outputTokens"] = target.get("outputTokens", 0) + int(source.get("outputTokens") or 0)
    target["totalTokens"] = target.get("totalTokens", 0) + int(source.get("totalTokens") or 0)
    target["estimatedCostUsd"] = target.get("estimatedCostUsd", 0.0) + float(source.get("estimatedCostUsd") or 0)
    target["requestCount"] = target.get("requestCount", 0) + int(source.get("requestCount") or 0)

def account_ledger(account_state):
    total = {}
    raw = account_state.get("daily")
    if isinstance(raw, dict):
        for item in raw.values():
            if isinstance(item, dict):
                add_ledger(total, item)
    return total

def action_type(probe):
    action = probe.get("action") if isinstance(probe, dict) else None
    if isinstance(action, dict):
        value = action.get("type")
        if value:
            return value
        return "taken" if action.get("taken") is True else ""
    return ""

def error_code(probe):
    details = probe.get("errorDetails") if isinstance(probe, dict) else None
    if not isinstance(details, dict):
        return ""
    openai_error = details.get("openaiError")
    if isinstance(openai_error, dict):
        return openai_error.get("code") or openai_error.get("type") or ""
    return details.get("kind") or ""

def report():
    cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME])
    state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME])
    state = {}
    parse_error = None
    if isinstance(state_cm, dict):
        raw_state = (state_cm.get("data") or {}).get("state.json")
        if isinstance(raw_state, str) and raw_state.strip():
            try:
                state = json.loads(raw_state)
            except Exception as exc:
                parse_error = str(exc)
    accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
    history = state.get("history") if isinstance(state.get("history"), list) else []
    account_rows = []
    for name, account_state in sorted(accounts.items()):
        if not isinstance(account_state, dict):
            continue
        probe = account_state.get("lastProbe") if isinstance(account_state.get("lastProbe"), dict) else {}
        quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else {}
        ledger = account_ledger(account_state)
        account_rows.append({
            "account": name,
            "status": account_state.get("lastStatus"),
            "quarantineActive": quarantine.get("active") is True,
            "quarantineApplied": quarantine.get("applied") if isinstance(quarantine, dict) else None,
            "freezeIntervalMin": quarantine.get("intervalMinutes") if isinstance(quarantine, dict) else None,
            "freezeUntil": quarantine.get("until") if isinstance(quarantine, dict) else None,
            "successStreak": account_state.get("successStreak") or 0,
            "successIntervalMin": account_state.get("successIntervalMinutes") or 0,
            "probeCount": ledger.get("requestCount", 0),
            "inputTokens": ledger.get("inputTokens", 0),
            "outputTokens": ledger.get("outputTokens", 0),
            "totalTokens": ledger.get("totalTokens", 0),
            "estimatedCostUsd": round(float(ledger.get("estimatedCostUsd", 0)), 6),
            "lastProbeAt": account_state.get("lastProbeAt"),
            "lastPurpose": probe.get("purpose"),
            "lastHttp": probe.get("httpStatus"),
            "lastMarker": probe.get("markerMatched"),
            "lastFailureKind": probe.get("failureKind"),
            "lastErrorCode": error_code(probe),
            "lastAction": action_type(probe),
            "nextProbeAfter": account_state.get("nextProbeAfter"),
            "observedLastToNextMin": minutes_between(account_state.get("lastProbeAt"), account_state.get("nextProbeAfter")),
            "requestShape": probe.get("requestShape"),
        })
    run_rows = []
    for item in history:
        if not isinstance(item, dict):
            continue
        selection = item.get("selection") if isinstance(item.get("selection"), dict) else {}
        run_rows.append({
            "at": item.get("at"),
            "selected": item.get("selected"),
            "due": selection.get("due"),
            "ok": item.get("okCount"),
            "mismatch": item.get("mismatchCount") if item.get("mismatchCount") is not None else item.get("markerMismatchCount"),
            "transportFailures": item.get("transportFailureCount"),
            "actionsTaken": item.get("actionsTaken"),
            "reasserts": len(item.get("reconcile") or []),
        })
    quarantined = [item for item in account_rows if item.get("quarantineActive") is True]
    cron_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
    cron_status = cronjob.get("status") if isinstance(cronjob, dict) else {}
    global_ledger = day_ledgers(state)
    result = {
        "ok": state_error is None and parse_error is None,
        "metadata": {
            "namespace": NAMESPACE,
            "stateConfigMapName": STATE_NAME,
            "cronJobName": CRONJOB_NAME,
            "generatedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
            "valuesPrinted": False,
        },
        "cronJob": {
            "exists": isinstance(cronjob, dict),
            "schedule": cron_spec.get("schedule") if isinstance(cron_spec, dict) else None,
            "suspend": cron_spec.get("suspend") if isinstance(cron_spec, dict) else None,
            "lastScheduleTime": cron_status.get("lastScheduleTime") if isinstance(cron_status, dict) else None,
            "active": len(cron_status.get("active") or []) if isinstance(cron_status, dict) else None,
            "error": cron_error,
        },
        "summary": {
            "accountCount": len(account_rows),
            "quarantinedCount": len(quarantined),
            "historyCount": len(history),
            "historyFrom": run_rows[0].get("at") if run_rows else None,
            "historyTo": run_rows[-1].get("at") if run_rows else None,
            "lastRun": state.get("lastRun"),
        },
        "globalLedger": global_ledger,
        "accounts": account_rows,
        "runs": run_rows[-EVENT_LIMIT:],
        "errors": {
            "state": state_error,
            "parse": parse_error,
        },
        "valuesPrinted": False,
    }
    return result

payload = report()
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload.get("ok") else 1)
PY
`;
}

function cleanupProbesScript(pool: CodexPoolConfig): string {
  return remotePythonScript("cleanup-probes", "", pool);
}

function sentinelImageStatusScript(pool: CodexPoolConfig): string {
  const target = codexPoolSentinelRuntimeImage(pool.sentinel);
  return remoteSentinelImageScript("status", target, pool.sentinel, null);
}

function sentinelImageBuildScript(pool: CodexPoolConfig): string {
  const target = codexPoolSentinelRuntimeImage(pool.sentinel);
  const dockerfile = readFileSync(sentinelImageDockerfilePath, "utf8");
  return remoteSentinelImageScript("build", target, pool.sentinel, dockerfile);
}

function remoteSentinelImageScript(mode: "status" | "build", target: ReturnType<typeof codexPoolSentinelRuntimeImage>, sentinel: CodexPoolSentinelConfig, dockerfile: string | null): string {
  const dockerfileB64 = dockerfile === null ? "" : Buffer.from(dockerfile, "utf8").toString("base64");
  return `
set -eu
mode=${shQuote(mode)}
image=${shQuote(target.runtimeImage)}
repo=${shQuote("platform-infra/sub2api-account-sentinel")}
tag=${shQuote(target.tag)}
base_image=${shQuote(target.baseImage)}
openai_version=${shQuote(sentinel.sdk.openaiPythonVersion)}
work=/tmp/unidesk-sub2api-sentinel-image
mkdir -p "$work"
dockerfile_path="$work/sentinel.Dockerfile"
registry_has_tag=false
if curl -fsS --max-time 10 "http://127.0.0.1:5000/v2/$repo/tags/list" 2>/dev/null | grep -F '"'"$tag"'"' >/dev/null 2>&1; then
  registry_has_tag=true
fi
local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)"
if [ "$mode" = "status" ]; then
  if [ -n "$local_id" ]; then
    python_version="$(docker run --rm "$image" python3 --version 2>/dev/null || true)"
    openai_runtime_version="$(docker run --rm "$image" python3 -c 'import importlib.metadata; print(importlib.metadata.version("openai"))' 2>/dev/null || true)"
  else
    python_version=
    openai_runtime_version=
  fi
  python3 - <<PY
import json
print(json.dumps({
  "ok": bool("${"${registry_has_tag}"}" == "true" or "${"${local_id}"}"),
  "mode": "status",
  "image": "${target.runtimeImage}",
  "baseImage": "${target.baseImage}",
  "tag": "${target.tag}",
  "localExists": bool("${"${local_id}"}"),
  "localImageId": "${"${local_id}"}" or None,
  "registryHasTag": "${"${registry_has_tag}"}" == "true",
  "pythonVersion": "${"${python_version}"}" or None,
  "openaiPythonVersion": "${"${openai_runtime_version}"}" or None,
}, ensure_ascii=False, indent=2))
PY
  exit 0
fi
if [ "$registry_has_tag" = "true" ]; then
  if [ -z "$local_id" ]; then
    docker pull "$image" >/dev/null 2>&1 || true
    local_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)"
  fi
  python3 - <<PY
import json
print(json.dumps({
  "ok": True,
  "mode": "reused-existing",
  "image": "${target.runtimeImage}",
  "baseImage": "${target.baseImage}",
  "tag": "${target.tag}",
  "registryHasTag": True,
  "localImageId": "${"${local_id}"}" or None,
}, ensure_ascii=False, indent=2))
PY
  exit 0
fi
base64 -d > "$dockerfile_path" <<'UNIDESK_SENTINEL_DOCKERFILE_B64'
${dockerfileB64}
UNIDESK_SENTINEL_DOCKERFILE_B64
export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000
export no_proxy=$NO_PROXY
docker build --pull \\
  --build-arg BASE_IMAGE="$base_image" \\
  --build-arg OPENAI_PYTHON_VERSION="$openai_version" \\
  --build-arg HTTP_PROXY= --build-arg HTTPS_PROXY= --build-arg http_proxy= --build-arg https_proxy= \\
  --build-arg NO_PROXY --build-arg no_proxy \\
  -f "$dockerfile_path" \\
  -t "$image" \\
  "$work"
docker run --rm "$image" python3 -c 'import importlib.metadata, sys; expected=sys.argv[1]; actual=importlib.metadata.version("openai"); assert actual == expected, (actual, expected); print("openai", actual)' "$openai_version"
docker push "$image"
digest="$(docker image inspect "$image" --format '{{index .RepoDigests 0}}' 2>/dev/null || true)"
python3 - <<PY
import json
print(json.dumps({
  "ok": True,
  "mode": "built-and-published",
  "image": "${target.runtimeImage}",
  "baseImage": "${target.baseImage}",
  "tag": "${target.tag}",
  "digest": "${"${digest}"}" or None,
}, ensure_ascii=False, indent=2))
PY
`;
}

function desiredAccountNames(pool: CodexPoolConfig): string[] {
  return Object.keys(desiredAccountCapacityMap(pool)).sort();
}

function desiredAccountCapacityTotal(pool: CodexPoolConfig): number {
  return Object.values(desiredAccountCapacityMap(pool)).reduce((total, value) => total + value, 0);
}

function desiredAccountCapacityMap(pool: CodexPoolConfig): Record<string, number> {
  const codexDir = join(homedir(), ".codex");
  const seenAccountNames = new Set<string>();
  const configs = pool.profiles.length > 0
    ? pool.profiles
    : discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
  const capacities: Record<string, number> = {};
  for (const entry of configs) {
    const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
    seenAccountNames.add(accountName);
    capacities[accountName] = entry.capacity ?? pool.defaultAccountCapacity;
  }
  return capacities;
}

function desiredAccountLoadFactorMap(pool: CodexPoolConfig): Record<string, number> {
  const codexDir = join(homedir(), ".codex");
  const seenAccountNames = new Set<string>();
  const configs = pool.profiles.length > 0
    ? pool.profiles
    : discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
  const loadFactors: Record<string, number> = {};
  for (const entry of configs) {
    const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
    seenAccountNames.add(accountName);
    loadFactors[accountName] = entry.loadFactor ?? pool.defaultAccountLoadFactor;
  }
  return loadFactors;
}

function desiredAccountWebSocketsV2ModeMap(pool: CodexPoolConfig): Record<string, OpenAIResponsesWebSocketsV2Mode | null> {
  const codexDir = join(homedir(), ".codex");
  const seenAccountNames = new Set<string>();
  const configs = pool.profiles.length > 0
    ? pool.profiles
    : discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
  const modes: Record<string, OpenAIResponsesWebSocketsV2Mode | null> = {};
  for (const entry of configs) {
    const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
    seenAccountNames.add(accountName);
    modes[accountName] = entry.openaiResponsesWebSocketsV2Mode;
  }
  return modes;
}

function desiredAccountTempUnschedulableMap(pool: CodexPoolConfig): Record<string, unknown> {
  const codexDir = join(homedir(), ".codex");
  const seenAccountNames = new Set<string>();
  const configs = pool.profiles.length > 0
    ? pool.profiles
    : discoverCodexProfileConfigs(codexDir, pool.defaultTempUnschedulable, pool.defaultAccountPriority);
  const policies: Record<string, unknown> = {};
  for (const entry of configs) {
    const accountName = entry.accountName ?? uniqueAccountName(entry.profile, seenAccountNames);
    seenAccountNames.add(accountName);
    policies[accountName] = renderSub2ApiTempUnschedulableCredentials(entry.tempUnschedulable);
  }
  return policies;
}

function remotePythonScript(mode: "sync" | "validate" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig): string {
  return `
set -u
python3 - <<'PY'
import base64
import json
import secrets
import string
import subprocess
import sys
import time
from datetime import datetime
from urllib.parse import quote

NAMESPACE = "${namespace}"
SERVICE_NAME = "${serviceName}"
SERVICE_DNS = "${serviceDns}"
FIELD_MANAGER = "${fieldManager}"
APP_SECRET_NAME = "${appSecretName}"
POOL_GROUP_NAME = "${pool.groupName}"
POOL_API_KEY_NAME = "${pool.apiKeyName}"
POOL_API_KEY_SECRET_NAME = "${pool.apiKeySecretName}"
POOL_API_KEY_SECRET_KEY = "${pool.apiKeySecretKey}"
MIN_OWNER_BALANCE_USD = ${JSON.stringify(pool.minOwnerBalanceUsd)}
MIN_OWNER_CONCURRENCY = ${JSON.stringify(pool.minOwnerConcurrency)}
MIN_OWNER_CONCURRENCY_SOURCE = ${JSON.stringify(pool.minOwnerConcurrencySource)}
POOL_DEFAULT_ACCOUNT_PRIORITY = ${JSON.stringify(pool.defaultAccountPriority)}
POOL_DEFAULT_ACCOUNT_CAPACITY = ${JSON.stringify(pool.defaultAccountCapacity)}
POOL_DEFAULT_ACCOUNT_LOAD_FACTOR = ${JSON.stringify(pool.defaultAccountLoadFactor)}
RESPONSES_SMOKE_MODEL = ${JSON.stringify(pool.localCodex.responsesSmokeModel)}
EXPECTED_ACCOUNT_CAPACITIES = ${JSON.stringify(desiredAccountCapacityMap(pool))}
EXPECTED_ACCOUNT_LOAD_FACTORS = ${JSON.stringify(desiredAccountLoadFactorMap(pool))}
EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))})
EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))})
SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))})
MODE = "${mode}"
PAYLOAD_B64 = "${encodedPayload}"

def run(cmd, input_bytes=None):
    return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def text(data, limit=4000):
    if isinstance(data, bytes):
        data = data.decode("utf-8", errors="replace")
    return data[-limit:]

def kubectl(args, input_obj=None):
    if isinstance(input_obj, str):
        input_bytes = input_obj.encode("utf-8")
    else:
        input_bytes = input_obj
    return run(["kubectl", *args], input_bytes)

def require_kubectl(args, input_obj=None, label="kubectl"):
    proc = kubectl(args, input_obj)
    if proc.returncode != 0:
        raise RuntimeError(f"{label} failed: {text(proc.stderr, 1000)}")
    return proc.stdout

def kube_json(args, label):
    raw = require_kubectl([*args, "-o", "json"], label=label)
    return json.loads(raw.decode("utf-8"))

def decode_secret_value(name, key):
    data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {}
    if key not in data:
        return None
    return base64.b64decode(data[key]).decode("utf-8")

def get_config_value(name, key):
    data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {}
    value = data.get(key)
    return value if isinstance(value, str) and value else None

def select_app_pod():
    pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or []
    for pod in pods:
        status = pod.get("status") or {}
        if status.get("phase") != "Running":
            continue
        statuses = status.get("containerStatuses") or []
        if statuses and all(item.get("ready") is True for item in statuses):
            return pod["metadata"]["name"]
    if pods:
        return pods[0]["metadata"]["name"]
    raise RuntimeError("sub2api app pod not found")

APP_POD = select_app_pod()

def parse_curl_output(proc):
    stdout = proc.stdout.decode("utf-8", errors="replace")
    marker = "\\n__HTTP_CODE__:"
    pos = stdout.rfind(marker)
    if pos < 0:
        return {
            "ok": False,
            "httpStatus": 0,
            "json": None,
            "body": stdout,
            "stderr": text(proc.stderr, 1000),
            "transportExitCode": proc.returncode,
        }
    body = stdout[:pos]
    status_text = stdout[pos + len(marker):].strip()
    try:
        http_status = int(status_text[-3:])
    except ValueError:
        http_status = 0
    try:
        parsed = json.loads(body) if body.strip() else None
    except json.JSONDecodeError:
        parsed = None
    return {
        "ok": proc.returncode == 0 and 200 <= http_status < 300,
        "httpStatus": http_status,
        "json": parsed,
        "body": body,
        "stderr": text(proc.stderr, 1000),
        "transportExitCode": proc.returncode,
    }

def curl_api(method, path, bearer=None, payload=None):
    body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
    script = r'''
set -eu
method="$1"
url="$2"
token="\${3:-}"
tmp="$(mktemp)"
trap 'rm -f "$tmp"' EXIT
cat > "$tmp"
if [ -n "$token" ]; then
  if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then
    curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url"
  else
    curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url"
  fi
else
  if [ "$method" = "GET" ] && [ ! -s "$tmp" ]; then
    curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"
  else
    curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
  fi
fi
'''
    proc = run([
        "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
        "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or "",
    ], body)
    return parse_curl_output(proc)

def envelope_data(parsed):
    if isinstance(parsed, dict) and "data" in parsed:
        return parsed.get("data")
    return parsed

def ensure_success(resp, label):
    parsed = resp.get("json")
    code = parsed.get("code") if isinstance(parsed, dict) else None
    if not resp.get("ok") or (code is not None and code != 0):
        message = parsed.get("message") if isinstance(parsed, dict) else text(resp.get("body", ""), 500)
        raise RuntimeError(f"{label} failed: http={resp.get('httpStatus')} message={message}")
    return envelope_data(parsed)

def ensure_admin_compliance(token):
    status_resp = curl_api("GET", "/api/v1/admin/compliance", bearer=token)
    if status_resp.get("httpStatus") == 404:
        return {
            "ok": True,
            "action": "not-supported-by-runtime",
            "valuesPrinted": False,
        }
    status = ensure_success(status_resp, "get admin compliance")
    if not isinstance(status, dict):
        return {
            "ok": True,
            "action": "status-unstructured",
            "valuesPrinted": False,
        }
    version = status.get("version")
    required = status.get("required") is True
    if not required:
        return {
            "ok": True,
            "action": "already-acknowledged",
            "version": version,
            "requiredBefore": False,
            "requiredAfter": False,
            "phrasePrinted": False,
            "valuesPrinted": False,
        }
    phrase = status.get("ack_phrase_zh") if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else status.get("ack_phrase_en")
    language = "zh" if isinstance(status.get("ack_phrase_zh"), str) and status.get("ack_phrase_zh") else "en"
    if not isinstance(phrase, str) or not phrase:
        raise RuntimeError("admin compliance acknowledgement phrase missing")
    accepted = ensure_success(
        curl_api("POST", "/api/v1/admin/compliance/accept", bearer=token, payload={"phrase": phrase, "language": language}),
        "accept admin compliance",
    )
    required_after = accepted.get("required") if isinstance(accepted, dict) else None
    return {
        "ok": required_after is False,
        "action": "accepted",
        "version": version,
        "requiredBefore": True,
        "requiredAfter": required_after,
        "phrasePrinted": False,
        "valuesPrinted": False,
    }

def extract_items(data):
    if isinstance(data, list):
        return data
    if isinstance(data, dict):
        if isinstance(data.get("items"), list):
            return data["items"]
        for key in ("groups", "accounts", "api_keys", "keys"):
            if isinstance(data.get(key), list):
                return data[key]
    return []

def find_access_token(data):
    if isinstance(data, dict):
        for key in ("access_token", "token"):
            if isinstance(data.get(key), str) and data[key]:
                return data[key]
        for value in data.values():
            token = find_access_token(value)
            if token:
                return token
    return None

def login():
    admin_email = get_config_value("sub2api-config", "ADMIN_EMAIL") or "admin@sub2api.platform-infra.local"
    admin_password = decode_secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD")
    if not admin_password:
        raise RuntimeError("ADMIN_PASSWORD missing from sub2api-secrets")
    data = ensure_success(curl_api("POST", "/api/v1/auth/login", payload={"email": admin_email, "password": admin_password}), "admin login")
    token = find_access_token(data)
    if not token:
        raise RuntimeError("admin login response did not contain access_token")
    compliance = ensure_admin_compliance(token)
    if compliance.get("ok") is not True:
        raise RuntimeError("admin compliance acknowledgement failed")
    return admin_email, token, compliance

def group_payload():
    return {
        "name": POOL_GROUP_NAME,
        "description": "UniDesk-managed Codex API-key pool for G14 k3s internal Sub2API clients.",
        "platform": "openai",
        "rate_multiplier": 1,
        "is_exclusive": False,
        "subscription_type": "standard",
        "allow_messages_dispatch": True,
        "require_oauth_only": False,
        "require_privacy_set": False,
        "rpm_limit": 0,
    }

def ensure_group(token):
    existing_data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")
    existing = next((item for item in extract_items(existing_data) if item.get("name") == POOL_GROUP_NAME), None)
    payload = group_payload()
    if existing is None:
        created = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=payload), "create group")
        return created, "created"
    group_id = existing.get("id")
    if group_id is None:
        raise RuntimeError("existing group has no id")
    payload["status"] = "active"
    updated = ensure_success(curl_api("PUT", f"/api/v1/admin/groups/{group_id}", bearer=token, payload=payload), "update group")
    return updated if isinstance(updated, dict) else existing, "updated"

def list_accounts(token):
    path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-codex-")
    data = ensure_success(curl_api("GET", path, bearer=token), "list accounts")
    return extract_items(data)

def list_probe_accounts(token):
    path = "/api/v1/admin/accounts?page=1&page_size=200&platform=openai&type=apikey&search=" + quote("unidesk-probe-")
    data = ensure_success(curl_api("GET", path, bearer=token), "list probe accounts")
    return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")]

def list_probe_groups(token):
    data = ensure_success(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")
    return [item for item in extract_items(data) if isinstance(item.get("name"), str) and item.get("name").startswith("unidesk-probe-")]

def cleanup_probe_resources(token):
    api_key_results = []
    account_results = []
    group_results = []
    for item in list_user_keys(token):
        name = item.get("name")
        key_id = item.get("id")
        if not isinstance(name, str) or not name.startswith("unidesk-probe-") or key_id is None:
            continue
        api_key_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{key_id}", "api-key"))
    for item in list_probe_accounts(token):
        account_id = item.get("id")
        if account_id is None:
            continue
        account_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account"))
    for item in list_probe_groups(token):
        group_id = item.get("id")
        if group_id is None:
            continue
        group_results.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group"))
    return {
        "ok": all(item.get("ok") is True for item in api_key_results + account_results + group_results),
        "apiKeysDeleted": len(api_key_results),
        "accountsDeleted": len(account_results),
        "groupsDeleted": len(group_results),
        "items": {
            "apiKeys": api_key_results,
            "accounts": account_results,
            "groups": group_results,
        },
        "valuesPrinted": False,
    }

def temp_unschedulable_credentials(profile):
    credentials = profile.get("tempUnschedulableCredentials")
    if not isinstance(credentials, dict):
        credentials = {}
    return normalize_temp_unschedulable_credentials(credentials)

def account_payload(profile, group_id):
    extra = {
        "openai_responses_mode": "force_responses",
        "unidesk_codex_profile": profile["profile"],
        "unidesk_managed": True,
    }
    ws_mode = profile.get("openaiResponsesWebSocketsV2Mode")
    if ws_mode:
        extra["openai_apikey_responses_websockets_v2_mode"] = ws_mode
        extra["openai_apikey_responses_websockets_v2_enabled"] = ws_mode != "off"
    credentials = {
        "api_key": profile["apiKey"],
        "base_url": profile["baseUrl"],
    }
    upstream_user_agent = profile.get("upstreamUserAgent")
    if upstream_user_agent:
        credentials["user_agent"] = upstream_user_agent
    temp_unschedulable = temp_unschedulable_credentials(profile)
    credentials["temp_unschedulable_enabled"] = temp_unschedulable["enabled"]
    credentials["temp_unschedulable_rules"] = temp_unschedulable["rules"]
    return {
        "name": profile["accountName"],
        "notes": f"UniDesk-managed Codex profile {profile['profile']} from {profile['configFile']} and {profile['authFile']}; secret source={profile['apiKeySource']}; fingerprint={profile['apiKeyFingerprint']}.",
        "platform": "openai",
        "type": "apikey",
        "credentials": credentials,
        "extra": extra,
        "concurrency": int(profile.get("capacity", 5) or 5),
        "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
        "rate_multiplier": 1,
        "load_factor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR),
        "group_ids": [group_id],
        "confirm_mixed_channel_risk": True,
    }

def ensure_account_schedulable(token, account_id, account_name):
    data = ensure_success(
        curl_api("POST", f"/api/v1/admin/accounts/{account_id}/schedulable", bearer=token, payload={"schedulable": True}),
        f"set account {account_name} schedulable",
    )
    return data if isinstance(data, dict) else {}

def ensure_accounts(token, profiles, group_id, prune_removed=False):
    existing_accounts = list_accounts(token)
    existing = {item.get("name"): item for item in existing_accounts}
    desired_names = {profile["accountName"] for profile in profiles}
    results = []
    for profile in profiles:
        payload = account_payload(profile, group_id)
        current = existing.get(profile["accountName"])
        if current and current.get("id") is not None:
            account_id = current["id"]
            update_payload = dict(payload)
            update_payload.pop("platform", None)
            update_payload["status"] = "active"
            data = ensure_success(curl_api("PUT", f"/api/v1/admin/accounts/{account_id}", bearer=token, payload=update_payload), f"update account {profile['accountName']}")
            action = "updated"
        else:
            data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}")
            action = "created"
        if isinstance(data, dict) and data.get("id") is not None:
            schedulable_data = ensure_account_schedulable(token, data["id"], profile["accountName"])
            if schedulable_data:
                data = schedulable_data
        results.append({
            "profile": profile["profile"],
            "accountName": profile["accountName"],
            "accountId": data.get("id") if isinstance(data, dict) else None,
            "action": action,
            "baseUrl": profile["baseUrl"],
            "apiKeySource": profile["apiKeySource"],
            "apiKeyFingerprint": profile["apiKeyFingerprint"],
            "openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"),
            "priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
            "capacity": int(profile.get("capacity", 5) or 5),
            "loadFactor": int(profile.get("loadFactor", POOL_DEFAULT_ACCOUNT_LOAD_FACTOR) or POOL_DEFAULT_ACCOUNT_LOAD_FACTOR),
            "runtimeConcurrency": data.get("concurrency") if isinstance(data, dict) else None,
            "runtimeLoadFactor": data.get("load_factor") if isinstance(data, dict) else None,
            "runtimeSchedulable": data.get("schedulable") if isinstance(data, dict) else None,
            "tempUnschedulableConfigured": bool(payload["credentials"].get("temp_unschedulable_enabled")),
            "tempUnschedulableRuleCount": len(payload["credentials"].get("temp_unschedulable_rules") or []),
            "upstreamUserAgentConfigured": bool(profile.get("upstreamUserAgent")),
            "valuesPrinted": False,
        })
    prune_results = prune_removed_accounts(token, existing_accounts, desired_names) if prune_removed else []
    return results, prune_results

def prune_removed_accounts(token, existing_accounts, desired_names):
    results = []
    for account in existing_accounts:
        name = account.get("name")
        account_id = account.get("id")
        extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
        if not isinstance(name, str) or not name.startswith("unidesk-codex-"):
            continue
        if extra.get("unidesk_managed") is not True:
            continue
        if name in desired_names:
            continue
        if account_id is None:
            raise RuntimeError(f"removed account {name} has no id")
        ensure_success(curl_api("DELETE", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"delete removed account {name}")
        results.append({
            "accountName": name,
            "accountId": account_id,
            "profile": extra.get("unidesk_codex_profile"),
            "action": "deleted",
            "reason": "removed-from-yaml",
            "valuesPrinted": False,
        })
    return results

def generate_api_key():
    alphabet = string.ascii_letters + string.digits
    return "sk-unidesk-codex-" + "".join(secrets.choice(alphabet) for _ in range(48))

def ensure_api_key_secret(group_id):
    existing = None
    try:
        existing = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY)
    except Exception:
        existing = None
    api_key = existing if existing else generate_api_key()
    secret_action = "kept-existing" if existing else "created"
    manifest = {
        "apiVersion": "v1",
        "kind": "Secret",
        "metadata": {
            "name": POOL_API_KEY_SECRET_NAME,
            "namespace": NAMESPACE,
            "labels": {
                "app.kubernetes.io/name": "sub2api",
                "app.kubernetes.io/part-of": "platform-infra",
                "app.kubernetes.io/managed-by": "unidesk",
                "unidesk.ai/secret-purpose": "sub2api-codex-pool-api-key",
            },
        },
        "type": "Opaque",
        "stringData": {
            POOL_API_KEY_SECRET_KEY: api_key,
            "GROUP_ID": str(group_id),
            "GROUP_NAME": POOL_GROUP_NAME,
            "SERVICE_DNS": SERVICE_DNS,
        },
    }
    proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
    if proc.returncode != 0:
        raise RuntimeError(f"apply API key secret failed: {text(proc.stderr, 1000)}")
    return api_key, secret_action, text(proc.stdout, 1000)

def apply_sentinel_manifest(manifest):
    if not isinstance(manifest, str) or not manifest.strip():
        return {
            "ok": False,
            "action": "missing-manifest",
            "valuesPrinted": False,
        }
    proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], manifest)
    if proc.returncode != 0:
        return {
            "ok": False,
            "action": "apply-failed",
            "stdoutTail": text(proc.stdout, 2000),
            "stderrTail": text(proc.stderr, 4000),
            "valuesPrinted": False,
        }
    status = sentinel_runtime_status()
    return {
        "ok": status.get("ok") is True,
        "action": "applied",
        "stdoutTail": text(proc.stdout, 2000),
        "runtime": status,
        "valuesPrinted": False,
    }

def safe_kube_json(args, label):
    proc = kubectl([*args, "-o", "json"])
    if proc.returncode != 0:
        return None, {"label": label, "exitCode": proc.returncode, "stderrTail": text(proc.stderr, 1000)}
    try:
        return json.loads(proc.stdout.decode("utf-8")), None
    except Exception as exc:
        return None, {"label": label, "exitCode": proc.returncode, "error": str(exc), "stdoutTail": text(proc.stdout, 1000)}

def sentinel_runtime_status():
    cfg = SENTINEL_CONFIG
    cronjob_name = cfg.get("cronJobName")
    secret_name = cfg.get("credentialsSecretName")
    configmap_name = cfg.get("configMapName")
    state_name = cfg.get("stateConfigMapName")
    cronjob, cronjob_error = safe_kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
    secret, secret_error = safe_kube_json(["-n", NAMESPACE, "get", "secret", secret_name], f"secret/{secret_name}")
    configmap, configmap_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", configmap_name], f"configmap/{configmap_name}")
    state_cm, state_error = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
    state = None
    if isinstance(state_cm, dict):
        raw_state = (state_cm.get("data") or {}).get("state.json")
        if isinstance(raw_state, str) and raw_state:
            try:
                state = json.loads(raw_state)
            except Exception as exc:
                state = {"parseError": str(exc)}
    accounts = (state.get("accounts") or {}) if isinstance(state, dict) else {}
    quarantined = []
    recent_accounts = []
    for name, account_state in accounts.items():
        if not isinstance(account_state, dict):
            continue
        quarantine = account_state.get("quarantine")
        if isinstance(quarantine, dict) and quarantine.get("active") is True:
            quarantined.append({
                "accountName": name,
                "until": quarantine.get("until"),
                "applied": quarantine.get("applied"),
                "reason": quarantine.get("reason"),
                "failureKind": quarantine.get("failureKind"),
                "errorDetails": quarantine.get("errorDetails"),
                "intervalMinutes": quarantine.get("intervalMinutes"),
            })
        last_probe = account_state.get("lastProbe")
        if isinstance(last_probe, dict):
            recent_accounts.append({
                "accountName": name,
                "lastProbeAt": account_state.get("lastProbeAt"),
                "lastStatus": account_state.get("lastStatus"),
                "nextProbeAfter": account_state.get("nextProbeAfter"),
                "ok": last_probe.get("ok"),
                "purpose": last_probe.get("purpose"),
                "httpStatus": last_probe.get("httpStatus"),
                "durationMs": last_probe.get("durationMs"),
                "markerMatched": last_probe.get("markerMatched"),
                "outputHash": last_probe.get("outputHash"),
                "outputPreview": last_probe.get("outputPreview"),
                "responseBodyHash": last_probe.get("responseBodyHash"),
                "responseBodyPreview": last_probe.get("responseBodyPreview"),
                "error": last_probe.get("error"),
                "errorDetails": last_probe.get("errorDetails"),
                "usage": last_probe.get("usage"),
                "failureKind": last_probe.get("failureKind"),
                "requestShape": last_probe.get("requestShape"),
                "action": last_probe.get("action"),
            })
    recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
    last_run = state.get("lastRun") if isinstance(state, dict) else None
    cronjob_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
    secret_data = secret.get("data") if isinstance(secret, dict) else {}
    configmap_data = configmap.get("data") if isinstance(configmap, dict) else {}
    ok = cronjob is not None and secret is not None and configmap is not None
    return {
        "ok": ok,
        "desired": {
            "monitorEnabled": cfg.get("monitor", {}).get("enabled"),
            "actionsEnabled": cfg.get("actions", {}).get("enabled"),
            "schedule": cfg.get("schedule"),
            "cronJobName": cronjob_name,
            "configMapName": configmap_name,
            "credentialsSecretName": secret_name,
            "stateConfigMapName": state_name,
        },
        "cronJob": {
            "exists": cronjob is not None,
            "schedule": cronjob_spec.get("schedule") if isinstance(cronjob_spec, dict) else None,
            "suspend": cronjob_spec.get("suspend") if isinstance(cronjob_spec, dict) else None,
            "lastScheduleTime": (cronjob.get("status") or {}).get("lastScheduleTime") if isinstance(cronjob, dict) else None,
            "active": len((cronjob.get("status") or {}).get("active") or []) if isinstance(cronjob, dict) else None,
            "error": cronjob_error,
        },
        "secret": {
            "exists": secret is not None,
            "profileSecretPresent": isinstance(secret_data, dict) and "profiles.json" in secret_data,
            "valuesPrinted": False,
            "error": secret_error,
        },
        "configMap": {
            "exists": configmap is not None,
            "configPresent": isinstance(configmap_data, dict) and "config.json" in configmap_data,
            "runnerPresent": isinstance(configmap_data, dict) and "sentinel.py" in configmap_data,
            "error": configmap_error,
        },
        "state": {
            "exists": state_cm is not None,
            "accountCount": len(accounts),
            "quarantinedCount": len(quarantined),
            "quarantined": quarantined[-10:],
            "recentAccounts": recent_accounts[-12:],
            "lastRun": last_run,
            "error": state_error,
        },
        "valuesPrinted": False,
    }

def parse_epoch_z(value):
    if not isinstance(value, str) or not value:
        return None
    try:
        return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
    except Exception:
        return None

def sentinel_state_object():
    state_name = SENTINEL_CONFIG.get("stateConfigMapName")
    if not state_name:
        return None
    obj, err = safe_kube_json(["-n", NAMESPACE, "get", "configmap", state_name], f"configmap/{state_name}")
    if not isinstance(obj, dict):
        return None
    raw_state = (obj.get("data") or {}).get("state.json")
    if not isinstance(raw_state, str) or not raw_state:
        return None
    try:
        return json.loads(raw_state)
    except Exception:
        return None

def sentinel_state_summary():
    state = sentinel_state_object()
    if not isinstance(state, dict):
        return {"exists": False, "valuesPrinted": False}
    accounts = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
    quarantined = []
    recent_accounts = []
    for name, account_state in accounts.items():
        if not isinstance(account_state, dict):
            continue
        quarantine = account_state.get("quarantine")
        if isinstance(quarantine, dict) and quarantine.get("active") is True:
            quarantined.append({
                "accountName": name,
                "until": quarantine.get("until"),
                "applied": quarantine.get("applied"),
                "reason": quarantine.get("reason"),
                "failureKind": quarantine.get("failureKind"),
                "errorDetails": quarantine.get("errorDetails"),
                "intervalMinutes": quarantine.get("intervalMinutes"),
            })
        last_probe = account_state.get("lastProbe")
        if isinstance(last_probe, dict):
            recent_accounts.append({
                "accountName": name,
                "lastProbeAt": account_state.get("lastProbeAt"),
                "lastStatus": account_state.get("lastStatus"),
                "nextProbeAfter": account_state.get("nextProbeAfter"),
                "ok": last_probe.get("ok"),
                "purpose": last_probe.get("purpose"),
                "httpStatus": last_probe.get("httpStatus"),
                "durationMs": last_probe.get("durationMs"),
                "markerMatched": last_probe.get("markerMatched"),
                "usage": last_probe.get("usage"),
                "responseBodyHash": last_probe.get("responseBodyHash"),
                "responseBodyPreview": last_probe.get("responseBodyPreview"),
                "error": last_probe.get("error"),
                "errorDetails": last_probe.get("errorDetails"),
                "failureKind": last_probe.get("failureKind"),
                "requestShape": last_probe.get("requestShape"),
                "action": last_probe.get("action"),
            })
    recent_accounts.sort(key=lambda item: item.get("lastProbeAt") or "")
    return {
        "exists": True,
        "accountCount": len(accounts),
        "quarantinedCount": len(quarantined),
        "quarantined": quarantined[-10:],
        "recentAccounts": recent_accounts[-12:],
        "lastRun": state.get("lastRun"),
        "valuesPrinted": False,
    }

def reassert_sentinel_freezes_after_sync(token):
    if (SENTINEL_CONFIG.get("actions") or {}).get("enabled") is not True:
        return {"ok": True, "skipped": True, "reason": "actions-disabled", "items": [], "valuesPrinted": False}
    state = sentinel_state_object()
    if not isinstance(state, dict):
        return {"ok": True, "skipped": True, "reason": "state-missing", "items": [], "valuesPrinted": False}
    accounts_state = state.get("accounts") if isinstance(state.get("accounts"), dict) else {}
    accounts = list_accounts(token)
    by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
    now_epoch = time.time()
    items = []
    for name, account_state in accounts_state.items():
        if not isinstance(account_state, dict):
            continue
        quarantine = account_state.get("quarantine")
        if not isinstance(quarantine, dict) or quarantine.get("active") is not True or quarantine.get("applied") is not True:
            continue
        until_epoch = parse_epoch_z(quarantine.get("until"))
        if until_epoch is not None and until_epoch <= now_epoch:
            continue
        account = by_name.get(name)
        if not account or account.get("id") is None:
            items.append({"accountName": name, "ok": False, "reason": "account-not-found"})
            continue
        try:
            ensure_success(
                curl_api("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", bearer=token, payload={"schedulable": False}),
                f"reassert sentinel freeze for {name}",
            )
            items.append({"accountName": name, "accountId": account.get("id"), "ok": True, "until": quarantine.get("until")})
        except Exception as exc:
            items.append({"accountName": name, "accountId": account.get("id"), "ok": False, "error": str(exc)})
    return {
        "ok": all(item.get("ok") is True for item in items),
        "skipped": False,
        "items": items,
        "valuesPrinted": False,
    }

def list_user_keys(token):
    data = ensure_success(curl_api("GET", "/api/v1/keys?page=1&page_size=200", bearer=token), "list user keys")
    return extract_items(data)

def ensure_sub2api_api_key(token, api_key, group_id):
    keys = list_user_keys(token)
    existing = next((item for item in keys if item.get("key") == api_key), None)
    action = "kept-existing"
    if existing is None:
        existing = next((item for item in keys if item.get("name") == POOL_API_KEY_NAME), None)
    if existing is None or existing.get("key") != api_key:
        payload = {
            "name": POOL_API_KEY_NAME,
            "group_id": group_id,
            "custom_key": api_key,
            "quota": 0,
            "rate_limit_5h": 0,
            "rate_limit_1d": 0,
            "rate_limit_7d": 0,
        }
        created = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload=payload), "create pool API key")
        existing = created if isinstance(created, dict) else existing
        action = "created"
    elif existing.get("id") is not None and existing.get("group_id") != group_id:
        updated = ensure_success(curl_api("PUT", f"/api/v1/keys/{existing['id']}", bearer=token, payload={"name": POOL_API_KEY_NAME, "group_id": group_id}), "update pool API key group")
        existing = updated if isinstance(updated, dict) else existing
        action = "updated-group"
    return {
        "action": action,
        "id": existing.get("id") if isinstance(existing, dict) else None,
        "name": existing.get("name") if isinstance(existing, dict) else POOL_API_KEY_NAME,
        "groupId": existing.get("group_id") if isinstance(existing, dict) else group_id,
        "userId": existing.get("user_id") if isinstance(existing, dict) else None,
    }

def get_admin_user(token, user_id):
    data = ensure_success(curl_api("GET", f"/api/v1/admin/users/{user_id}", bearer=token), "get API key owner")
    if not isinstance(data, dict):
        raise RuntimeError("API key owner response is not an object")
    return data

def ensure_pool_owner_balance(token, user_id):
    user = get_admin_user(token, user_id)
    current = float(user.get("balance") or 0)
    if current >= MIN_OWNER_BALANCE_USD:
        return {
            "action": "kept-existing",
            "userId": user_id,
            "balanceBefore": current,
            "balanceAfter": current,
            "minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
        }
    updated = ensure_success(curl_api("POST", f"/api/v1/admin/users/{user_id}/balance", bearer=token, payload={
        "balance": MIN_OWNER_BALANCE_USD,
        "operation": "set",
        "notes": "UniDesk Sub2API Codex pool internal API key bootstrap balance.",
    }), "set API key owner balance")
    after = float(updated.get("balance") or MIN_OWNER_BALANCE_USD) if isinstance(updated, dict) else MIN_OWNER_BALANCE_USD
    return {
        "action": "set",
        "userId": user_id,
        "balanceBefore": current,
        "balanceAfter": after,
        "minimumBalanceUsd": MIN_OWNER_BALANCE_USD,
    }

def ensure_pool_owner_concurrency(token, user_id):
    user = get_admin_user(token, user_id)
    try:
        current = int(user.get("concurrency") or 0)
    except Exception:
        current = 0
    if current >= MIN_OWNER_CONCURRENCY:
        return {
            "ok": True,
            "action": "kept-existing",
            "userId": user_id,
            "concurrencyBefore": current,
            "concurrencyAfter": current,
            "minimumConcurrency": MIN_OWNER_CONCURRENCY,
            "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
        }
    updated = ensure_success(curl_api("PUT", f"/api/v1/admin/users/{user_id}", bearer=token, payload={
        "concurrency": MIN_OWNER_CONCURRENCY,
    }), "set API key owner concurrency")
    try:
        after = int(updated.get("concurrency") or MIN_OWNER_CONCURRENCY) if isinstance(updated, dict) else MIN_OWNER_CONCURRENCY
    except Exception:
        after = MIN_OWNER_CONCURRENCY
    return {
        "ok": after >= MIN_OWNER_CONCURRENCY,
        "action": "set",
        "userId": user_id,
        "concurrencyBefore": current,
        "concurrencyAfter": after,
        "minimumConcurrency": MIN_OWNER_CONCURRENCY,
        "minimumConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
    }

def validate_gateway(api_key):
    resp = curl_api("GET", "/v1/models", bearer=api_key)
    parsed = resp.get("json")
    model_count = None
    if isinstance(parsed, dict) and isinstance(parsed.get("data"), list):
        model_count = len(parsed["data"])
    return {
        "ok": resp.get("ok"),
        "httpStatus": resp.get("httpStatus"),
        "transportExitCode": resp.get("transportExitCode"),
        "modelCount": model_count,
        "bodyPreview": text(resp.get("body", ""), 500) if not resp.get("ok") else "",
        "stderr": resp.get("stderr", ""),
        "method": "GET /v1/models",
        "serviceDns": SERVICE_DNS,
        "valuesPrinted": False,
    }

def response_output_preview(parsed):
    if not isinstance(parsed, dict):
        return ""
    if isinstance(parsed.get("output_text"), str):
        return parsed["output_text"][:240]
    output = parsed.get("output")
    if not isinstance(output, list):
        return ""
    parts = []
    for item in output:
        if not isinstance(item, dict):
            continue
        content = item.get("content")
        if not isinstance(content, list):
            continue
        for block in content:
            if not isinstance(block, dict):
                continue
            text_value = block.get("text")
            if isinstance(text_value, str) and text_value:
                parts.append(text_value)
    return "\\n".join(parts)[:240]

def request_log_evidence(request_id):
    proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=5m", "--tail=800"])
    stdout = proc.stdout.decode("utf-8", errors="replace")
    lines = [line for line in stdout.splitlines() if request_id in line]
    failovers = []
    final = None
    for line in lines:
        json_start = line.find("{")
        if json_start < 0:
            continue
        try:
            item = json.loads(line[json_start:])
        except Exception:
            continue
        if "upstream_failover_switching" in line:
            failovers.append({
                "accountId": item.get("account_id"),
                "upstreamStatus": item.get("upstream_status"),
                "switchCount": item.get("switch_count"),
                "maxSwitches": item.get("max_switches"),
            })
        if "http request completed" in line:
            final = {
                "accountId": item.get("account_id"),
                "statusCode": item.get("status_code"),
                "latencyMs": item.get("latency_ms"),
                "path": item.get("path"),
            }
    return {
        "requestId": request_id,
        "matchedLogLineCount": len(lines),
        "failovers": failovers,
        "final": final,
        "logsExitCode": proc.returncode,
        "logsStderr": text(proc.stderr, 1000),
    }

def recent_compact_gateway_evidence():
    proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
    stdout = proc.stdout.decode("utf-8", errors="replace")
    failures = []
    successes = []
    failovers = []
    final_errors = []
    context_canceled = []
    for line in stdout.splitlines():
        if "/responses/compact" not in line and "remote_compact" not in line:
            continue
        json_start = line.find("{")
        if json_start < 0:
            continue
        try:
            item = json.loads(line[json_start:])
        except Exception:
            continue
        path = item.get("path")
        entry = {
            "requestId": item.get("request_id"),
            "clientRequestId": item.get("client_request_id"),
            "accountId": item.get("account_id"),
            "statusCode": item.get("status_code"),
            "upstreamStatus": item.get("upstream_status"),
            "latencyMs": item.get("latency_ms"),
            "path": path,
        }
        if "codex.remote_compact.failed" in line:
            failures.append(entry)
        elif "codex.remote_compact.succeeded" in line:
            successes.append(entry)
        elif "upstream_failover_switching" in line and path == "/responses/compact":
            failovers.append({
                **entry,
                "switchCount": item.get("switch_count"),
                "maxSwitches": item.get("max_switches"),
            })
        elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
            final_errors.append(entry)
        if "context canceled" in line and path == "/responses/compact":
            context_canceled.append(entry)
    return {
        "ok": True,
        "degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
        "window": "6h",
        "tailLines": 2500,
        "failureCount": len(failures),
        "successCount": len(successes),
        "failoverCount": len(failovers),
        "finalErrorCount": len(final_errors),
        "contextCanceledCount": len(context_canceled),
        "recentFailures": failures[-5:],
        "recentSuccesses": successes[-5:],
        "recentFailovers": failovers[-8:],
        "recentFinalErrors": final_errors[-5:],
        "recentContextCanceled": context_canceled[-5:],
        "logsExitCode": proc.returncode,
        "logsStderr": text(proc.stderr, 1000),
        "valuesPrinted": False,
    }

def is_ignored_probe_noise(entry):
    for value in (entry.get("requestId"), entry.get("clientRequestId")):
        if isinstance(value, str) and value.startswith("unidesk-400-model-probe-"):
            return True
    return False

def filter_ignored_probe_noise(items):
    return [item for item in items if not is_ignored_probe_noise(item)]

def ignored_probe_noise(items, section):
    result = []
    for item in items:
        if is_ignored_probe_noise(item):
            probe = dict(item)
            probe["section"] = section
            result.append(probe)
    return result

def recent_responses_gateway_evidence():
    proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
    stdout = proc.stdout.decode("utf-8", errors="replace")
    failovers = []
    forward_failures = []
    final_errors = []
    context_canceled = []
    slow_final_errors = []
    for line in stdout.splitlines():
        if '"/responses"' not in line and '"/v1/responses"' not in line:
            continue
        json_start = line.find("{")
        if json_start < 0:
            continue
        try:
            item = json.loads(line[json_start:])
        except Exception:
            continue
        path = item.get("path")
        if path not in ("/responses", "/v1/responses"):
            continue
        entry = {
            "requestId": item.get("request_id"),
            "clientRequestId": item.get("client_request_id"),
            "accountId": item.get("account_id"),
            "statusCode": item.get("status_code"),
            "upstreamStatus": item.get("upstream_status"),
            "latencyMs": item.get("latency_ms"),
            "path": path,
        }
        if "upstream_failover_switching" in line:
            failovers.append({
                **entry,
                "switchCount": item.get("switch_count"),
                "maxSwitches": item.get("max_switches"),
            })
        elif "openai.forward_failed" in line:
            forward_failures.append({
                **entry,
                "errorPreview": text(str(item.get("error") or ""), 500),
                "fallbackErrorResponseWritten": item.get("fallback_error_response_written"),
                "upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"),
            })
        elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
            final_errors.append(entry)
            latency_ms = item.get("latency_ms")
            if isinstance(latency_ms, int) and latency_ms >= 30000:
                slow_final_errors.append(entry)
        if "context canceled" in line:
            context_canceled.append(entry)
    visible_failovers = filter_ignored_probe_noise(failovers)
    visible_forward_failures = filter_ignored_probe_noise(forward_failures)
    visible_final_errors = filter_ignored_probe_noise(final_errors)
    visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors)
    visible_context_canceled = filter_ignored_probe_noise(context_canceled)
    probe_noise = (
        ignored_probe_noise(failovers, "failovers")
        + ignored_probe_noise(forward_failures, "forwardFailures")
        + ignored_probe_noise(final_errors, "finalErrors")
        + ignored_probe_noise(slow_final_errors, "slowFinalErrors")
        + ignored_probe_noise(context_canceled, "contextCanceled")
    )
    return {
        "ok": True,
        "degraded": len(visible_forward_failures) > 0 or len(visible_final_errors) > 0 or len(visible_context_canceled) > 0,
        "window": "6h",
        "tailLines": 2500,
        "failoverCount": len(visible_failovers),
        "forwardFailureCount": len(visible_forward_failures),
        "finalErrorCount": len(visible_final_errors),
        "slowFinalErrorCount": len(visible_slow_final_errors),
        "contextCanceledCount": len(visible_context_canceled),
        "ignoredProbeNoiseCount": len(probe_noise),
        "rawCounts": {
            "failoverCount": len(failovers),
            "forwardFailureCount": len(forward_failures),
            "finalErrorCount": len(final_errors),
            "slowFinalErrorCount": len(slow_final_errors),
            "contextCanceledCount": len(context_canceled),
        },
        "recentFailovers": visible_failovers[-8:],
        "recentForwardFailures": visible_forward_failures[-8:],
        "recentFinalErrors": visible_final_errors[-8:],
        "recentSlowFinalErrors": visible_slow_final_errors[-5:],
        "recentContextCanceled": visible_context_canceled[-5:],
        "recentProbeNoise": probe_noise[-5:],
        "logsExitCode": proc.returncode,
        "logsStderr": text(proc.stderr, 1000),
        "valuesPrinted": False,
    }

def validate_gateway_responses(api_key):
    request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
    payload = {
        "model": RESPONSES_SMOKE_MODEL,
        "input": "Reply exactly: unidesk-sub2api-validate-ok",
        "stream": False,
        "store": False,
        "max_output_tokens": 32,
    }
    body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
    script = r'''
set -eu
token="$1"
request_id="$2"
tmp="$(mktemp)"
trap 'rm -f "$tmp"' EXIT
cat > "$tmp"
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \
  -H "Authorization: Bearer $token" \
  -H 'Content-Type: application/json' \
  -H "X-Request-ID: $request_id" \
  -H "OpenAI-Client-Request-ID: $request_id" \
  --data-binary @"$tmp" \
  http://127.0.0.1:8080/v1/responses
'''
    started = time.time()
    proc = run([
        "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
        "--", "sh", "-c", script, "sh", api_key, request_id,
    ], body)
    resp = parse_curl_output(proc)
    evidence = request_log_evidence(request_id)
    parsed = resp.get("json")
    failover_count = len(evidence.get("failovers") or [])
    return {
        "ok": resp.get("ok"),
        "degraded": failover_count > 0,
        "outcome": "succeeded-with-failover" if resp.get("ok") and failover_count > 0 else ("succeeded" if resp.get("ok") else "failed"),
        "httpStatus": resp.get("httpStatus"),
        "transportExitCode": resp.get("transportExitCode"),
        "method": "POST /v1/responses",
        "model": RESPONSES_SMOKE_MODEL,
        "requestId": request_id,
        "durationMs": int((time.time() - started) * 1000),
        "outputTextPreview": response_output_preview(parsed),
        "bodyPreview": "" if resp.get("ok") else text(resp.get("body", ""), 800),
        "stderr": resp.get("stderr", ""),
        "evidence": evidence,
        "valuesPrinted": False,
    }

def bool_value(value):
    if isinstance(value, bool):
        return value
    if isinstance(value, str):
        if value.lower() == "true":
            return True
        if value.lower() == "false":
            return False
    return False

def normalize_temp_unschedulable_credentials(credentials):
    if not isinstance(credentials, dict):
        credentials = {}
    enabled = bool_value(credentials.get("temp_unschedulable_enabled"))
    raw_rules = credentials.get("temp_unschedulable_rules")
    if isinstance(raw_rules, str):
        try:
            raw_rules = json.loads(raw_rules)
        except json.JSONDecodeError:
            raw_rules = []
    rules = []
    if isinstance(raw_rules, list):
        for rule in raw_rules:
            if not isinstance(rule, dict):
                continue
            error_code = rule.get("error_code", rule.get("statusCode"))
            duration_minutes = rule.get("duration_minutes", rule.get("durationMinutes"))
            keywords = rule.get("keywords")
            if not isinstance(error_code, int) or not isinstance(duration_minutes, int) or not isinstance(keywords, list):
                continue
            clean_keywords = [item for item in keywords if isinstance(item, str) and item.strip()]
            if not clean_keywords:
                continue
            description = rule.get("description") if isinstance(rule.get("description"), str) else ""
            rules.append({
                "error_code": error_code,
                "keywords": clean_keywords,
                "duration_minutes": duration_minutes,
                "description": description,
            })
    return {
        "enabled": enabled and len(rules) > 0,
        "rules": rules if enabled else [],
    }

def summarize_temp_unschedulable_rules(rules):
    return [{
        "errorCode": rule.get("error_code"),
        "durationMinutes": rule.get("duration_minutes"),
        "keywordCount": len(rule.get("keywords") or []),
        "keywords": rule.get("keywords") or [],
        "hasDescription": bool(rule.get("description")),
    } for rule in rules]

def success_body_reclassification_requirement():
    for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
        expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
        for rule in expected["rules"]:
            error_code = rule.get("error_code")
            keywords = rule.get("keywords") or []
            if isinstance(error_code, int) and 200 <= error_code < 300 and keywords:
                return {
                    "required": True,
                    "sourceAccountName": name,
                    "statusCode": error_code,
                    "keywords": keywords,
                    "probeKeyword": keywords[0],
                    "durationMinutes": rule.get("duration_minutes"),
                }
    return {
        "required": False,
        "sourceAccountName": None,
        "statusCode": None,
        "keywords": [],
        "probeKeyword": None,
        "durationMinutes": None,
    }

def model_routing_400_failover_requirement():
    preferred = ["暂不支持", "可用模型", "unsupported model", "model not supported", "does not support", "not supported", "model_not_found", "no available channel for model"]
    for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
        expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
        for rule in expected["rules"]:
            error_code = rule.get("error_code")
            keywords = rule.get("keywords") or []
            if error_code != 400 or not keywords:
                continue
            probe_keyword = next((item for item in preferred if item in keywords), keywords[0])
            return {
                "required": True,
                "sourceAccountName": name,
                "statusCode": error_code,
                "keywords": keywords,
                "probeKeyword": probe_keyword,
                "durationMinutes": rule.get("duration_minutes"),
            }
    return {
        "required": False,
        "sourceAccountName": None,
        "statusCode": None,
        "keywords": [],
        "probeKeyword": None,
        "durationMinutes": None,
    }

def delete_probe_resource(token, method, path, label):
    if not path:
        return {"label": label, "ok": True, "skipped": True}
    resp = curl_api(method, path, bearer=token)
    ok = resp.get("ok") is True or resp.get("httpStatus") in (404, 410)
    return {
        "label": label,
        "ok": ok,
        "method": method,
        "path": path,
        "httpStatus": resp.get("httpStatus"),
        "transportExitCode": resp.get("transportExitCode"),
        "bodyPreview": "" if ok else text(resp.get("body", ""), 500),
        "valuesPrinted": False,
    }

def launch_success_body_mock_upstream(status_code, body_text):
    port = 28000 + secrets.randbelow(2000)
    body_b64 = base64.b64encode(body_text.encode("utf-8")).decode("ascii")
    script = r'''
set -eu
port="$1"
status="$2"
body_b64="$3"
body="$(printf "%s" "$body_b64" | base64 -d)"
length="$(printf "%s" "$body" | wc -c | tr -d " ")"
{ printf "HTTP/1.1 %s OK\r\nContent-Type: application/json\r\nContent-Length: %s\r\nConnection: close\r\n\r\n" "$status" "$length"; printf "%s" "$body"; } | nc -l -p "$port" -w 5
'''
    proc = subprocess.Popen([
        "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
        "--", "sh", "-c", script, "sh", str(port), str(status_code), body_b64,
    ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    time.sleep(0.35)
    if proc.poll() is None:
        return port, proc, None
    stdout, stderr = proc.communicate(timeout=2)
    return None, None, {
        "ok": False,
        "error": "mock-upstream-exited-before-probe",
        "exitCode": proc.returncode,
        "stdoutTail": text(stdout, 1000),
        "stderrTail": text(stderr, 1000),
    }

def finish_mock_upstream(proc):
    if proc is None:
        return None
    timed_out = False
    try:
        stdout, stderr = proc.communicate(timeout=2)
    except subprocess.TimeoutExpired:
        timed_out = True
        proc.kill()
        stdout, stderr = proc.communicate(timeout=2)
    return {
        "exitCode": proc.returncode,
        "timedOut": timed_out,
        "stdoutTail": text(stdout, 1000),
        "stderrTail": text(stderr, 1000),
    }

def create_success_body_probe_resources(token, base_url, requirement):
    stamp = str(int(time.time() * 1000))
    suffix = stamp + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(6))
    group_payload_obj = group_payload()
    group_payload_obj["name"] = "unidesk-probe-2xx-body-" + suffix
    group_payload_obj["description"] = "UniDesk validate probe for OpenAI 2xx success-body reclassification."
    group_id = None
    account_id = None
    api_key_id = None
    try:
        group = ensure_success(curl_api("POST", "/api/v1/admin/groups", bearer=token, payload=group_payload_obj), "create 2xx success-body probe group")
        group_id = group.get("id") if isinstance(group, dict) else None
        if group_id is None:
            raise RuntimeError("2xx success-body probe group id missing")

        account_payload_obj = {
            "name": "unidesk-probe-2xx-body-" + suffix,
            "notes": "Temporary UniDesk validate probe account; safe to delete.",
            "platform": "openai",
            "type": "apikey",
            "credentials": {
                "api_key": "sk-unidesk-probe-upstream",
                "base_url": base_url,
                "temp_unschedulable_enabled": True,
                "temp_unschedulable_rules": [{
                    "error_code": requirement["statusCode"],
                    "keywords": [requirement["probeKeyword"]],
                    "duration_minutes": 1,
                    "description": "UniDesk runtime capability probe for OpenAI 2xx success-body reclassification.",
                }],
            },
            "extra": {
                "openai_responses_mode": "force_responses",
                "unidesk_probe": "success_body_reclassification",
            },
            "concurrency": 1,
            "priority": 0,
            "rate_multiplier": 1,
            "load_factor": 1,
            "group_ids": [group_id],
            "confirm_mixed_channel_risk": True,
        }
        account = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=account_payload_obj), "create 2xx success-body probe account")
        account_id = account.get("id") if isinstance(account, dict) else None
        if account_id is None:
            raise RuntimeError("2xx success-body probe account id missing")

        api_key = "sk-unidesk-probe-" + "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(36))
        api_key_obj = ensure_success(curl_api("POST", "/api/v1/keys", bearer=token, payload={
            "name": "unidesk-probe-2xx-body-" + suffix,
            "group_id": group_id,
            "custom_key": api_key,
            "quota": 0,
            "rate_limit_5h": 0,
            "rate_limit_1d": 0,
            "rate_limit_7d": 0,
        }), "create 2xx success-body probe API key")
        api_key_id = api_key_obj.get("id") if isinstance(api_key_obj, dict) else None
        return {
            "groupId": group_id,
            "groupName": group_payload_obj["name"],
            "accountId": account_id,
            "accountName": account_payload_obj["name"],
            "apiKeyId": api_key_id,
            "apiKey": api_key,
            "keyPreview": api_key_preview(api_key),
            "valuesPrinted": False,
        }
    except Exception:
        if api_key_id is not None:
            delete_probe_resource(token, "DELETE", f"/api/v1/keys/{api_key_id}", "api-key")
        if account_id is not None:
            delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{account_id}", "account")
        if group_id is not None:
            delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{group_id}", "group")
        raise

def gateway_success_body_probe_request(api_key, request_id):
    payload = {
        "model": RESPONSES_SMOKE_MODEL,
        "input": "Reply exactly: success-body probe",
        "stream": False,
        "store": False,
        "max_output_tokens": 8,
    }
    body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
    script = r'''
set -eu
token="$1"
request_id="$2"
tmp="$(mktemp)"
trap 'rm -f "$tmp"' EXIT
cat > "$tmp"
curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X POST \
  -H "Authorization: Bearer $token" \
  -H 'Content-Type: application/json' \
  -H "X-Request-ID: $request_id" \
  -H "OpenAI-Client-Request-ID: $request_id" \
  --data-binary @"$tmp" \
  http://127.0.0.1:8080/v1/responses
'''
    proc = run([
        "kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD,
        "--", "sh", "-c", script, "sh", api_key, request_id,
    ], body)
    return parse_curl_output(proc)

def account_temp_unschedulable_probe_state(token, account_id):
    detail = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), "get temp-unschedulable probe account")
    if not isinstance(detail, dict):
        return {
            "accountId": account_id,
            "status": None,
            "schedulable": None,
            "tempUnschedulableUntil": None,
            "tempUnschedulableReasonPreview": "",
            "tempUnschedulableSet": False,
        }
    until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil")
    reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or ""
    return {
        "accountId": account_id,
        "status": detail.get("status"),
        "schedulable": detail.get("schedulable"),
        "tempUnschedulableUntil": until,
        "tempUnschedulableReasonPreview": text(str(reason), 500) if reason else "",
        "tempUnschedulableSet": until is not None or bool(reason),
    }

def validate_success_body_reclassification(token):
    requirement = success_body_reclassification_requirement()
    if not requirement["required"]:
        return {
            "ok": True,
            "required": False,
            "capability": "openai-2xx-success-body-temp-unschedulable-failover",
            "outcome": "not-required-by-yaml",
            "valuesPrinted": False,
        }

    resources = None
    cleanup = []
    mock_proc = None
    try:
        keyword = requirement["probeKeyword"]
        upstream_body = json.dumps({
            "id": "resp_unidesk_success_body_probe",
            "object": "response",
            "created_at": int(time.time()),
            "status": "completed",
            "model": RESPONSES_SMOKE_MODEL,
            "output": [{
                "type": "message",
                "role": "assistant",
                "content": [{"type": "output_text", "text": keyword}],
            }],
            "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
        }, separators=(",", ":"))
        port, mock_proc, mock_error = launch_success_body_mock_upstream(requirement["statusCode"], upstream_body)
        if mock_error is not None:
            return {
                "ok": False,
                "required": True,
                "capability": "openai-2xx-success-body-temp-unschedulable-failover",
                "outcome": "probe-infrastructure-failed",
                "requirement": requirement,
                "mock": mock_error,
                "valuesPrinted": False,
            }
        resources = create_success_body_probe_resources(token, f"http://127.0.0.1:{port}", requirement)
        request_id = "unidesk-2xx-body-probe-" + str(int(time.time() * 1000))
        started = time.time()
        response = gateway_success_body_probe_request(resources["apiKey"], request_id)
        mock_result = finish_mock_upstream(mock_proc)
        mock_proc = None
        state = account_temp_unschedulable_probe_state(token, resources["accountId"])
        evidence = request_log_evidence(request_id)
        supported = response.get("ok") is not True and state.get("tempUnschedulableSet") is True
        return {
            "ok": supported,
            "required": True,
            "capability": "openai-2xx-success-body-temp-unschedulable-failover",
            "outcome": "supported" if supported else "unsupported-runtime-image",
            "requirement": requirement,
            "probe": {
                "requestId": request_id,
                "durationMs": int((time.time() - started) * 1000),
                "httpStatus": response.get("httpStatus"),
                "transportExitCode": response.get("transportExitCode"),
                "responseOk": response.get("ok"),
                "bodyPreview": text(response.get("body", ""), 800),
                "stderr": response.get("stderr", ""),
                "accountState": state,
                "logEvidence": evidence,
            },
            "mock": mock_result,
            "resources": {
                "groupId": resources["groupId"],
                "accountId": resources["accountId"],
                "apiKeyId": resources["apiKeyId"],
                "keyPreview": resources["keyPreview"],
                "valuesPrinted": False,
            },
            "cleanup": cleanup,
            "message": "Sub2API image must reclassify matching 2xx OpenAI bodies into failover/temp-unschedulable before statusCode=200 YAML rules are effective." if not supported else "Matching 2xx OpenAI bodies are reclassified into account failover/temp-unschedulable.",
            "valuesPrinted": False,
        }
    except Exception as exc:
        return {
            "ok": False,
            "required": True,
            "capability": "openai-2xx-success-body-temp-unschedulable-failover",
            "outcome": "probe-failed",
            "requirement": requirement,
            "error": str(exc),
            "cleanup": cleanup,
            "valuesPrinted": False,
        }
    finally:
        if mock_proc is not None:
            _ = finish_mock_upstream(mock_proc)
        if resources is not None:
            cleanup.append(delete_probe_resource(token, "DELETE", f"/api/v1/keys/{resources['apiKeyId']}" if resources.get("apiKeyId") is not None else "", "api-key"))
            cleanup.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/accounts/{resources['accountId']}", "account"))
            cleanup.append(delete_probe_resource(token, "DELETE", f"/api/v1/admin/groups/{resources['groupId']}", "group"))

def validate_runtime_capabilities(token):
    success_body = validate_success_body_reclassification(token)
    model_routing_400 = model_routing_400_failover_requirement()
    return {
        "ok": success_body.get("ok") is True,
        "runtimeImage": app_pod_runtime_image(),
        "successBodyReclassification": success_body,
        "modelRouting400Failover": {
            "ok": True,
            "required": model_routing_400.get("required") is True,
            "capability": "openai-400-model-routing-temp-unschedulable-failover",
            "outcome": "declared-by-yaml-and-checked-by-runtime-rules",
            "requirement": model_routing_400,
            "message": "Default validate stays short; runtime proof comes from synced rules plus real request logs/failover evidence.",
            "valuesPrinted": False,
        },
        "valuesPrinted": False,
    }

def app_pod_runtime_image():
    try:
        pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], f"pod/{APP_POD}")
        spec_containers = ((pod.get("spec") or {}).get("containers") or []) if isinstance(pod, dict) else []
        status_containers = ((pod.get("status") or {}).get("containerStatuses") or []) if isinstance(pod, dict) else []
        spec = next((item for item in spec_containers if item.get("name") == "sub2api"), spec_containers[0] if spec_containers else {})
        status = next((item for item in status_containers if item.get("name") == "sub2api"), status_containers[0] if status_containers else {})
        return {
            "pod": APP_POD,
            "image": spec.get("image"),
            "imageID": status.get("imageID"),
            "ready": status.get("ready"),
            "restartCount": status.get("restartCount"),
        }
    except Exception as exc:
        return {"pod": APP_POD, "error": str(exc)}

def get_account_detail(token, account):
    account_id = account.get("id") if isinstance(account, dict) else None
    if account_id is None:
        return account
    data = ensure_success(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), f"get account {account.get('name')}")
    return data if isinstance(data, dict) else account

def account_temp_unschedulable_status(token):
    accounts = list_accounts(token)
    by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
    items = []
    missing = []
    mismatched = []
    enabled_names = []
    for name in sorted(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE):
        expected = normalize_temp_unschedulable_credentials(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE[name])
        account = by_name.get(name)
        if account is None:
            missing.append(name)
            items.append({
                "accountName": name,
                "accountId": None,
                "expectedEnabled": expected["enabled"],
                "runtimeEnabled": None,
                "expectedRuleCount": len(expected["rules"]),
                "runtimeRuleCount": None,
                "ok": False,
            })
            continue
        detail = get_account_detail(token, account)
        credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {}
        runtime = normalize_temp_unschedulable_credentials(credentials)
        temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil")
        temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or ""
        ok = runtime == expected
        if expected["enabled"]:
            enabled_names.append(name)
        if not ok:
            mismatched.append(name)
        items.append({
            "accountName": name,
            "accountId": account.get("id"),
            "expectedEnabled": expected["enabled"],
            "runtimeEnabled": runtime["enabled"],
            "expectedRuleCount": len(expected["rules"]),
            "runtimeRuleCount": len(runtime["rules"]),
            "expectedRules": summarize_temp_unschedulable_rules(expected["rules"]),
            "runtimeRules": summarize_temp_unschedulable_rules(runtime["rules"]),
            "status": account.get("status"),
            "schedulable": account.get("schedulable"),
            "tempUnschedulableUntil": temp_until,
            "tempUnschedulableReasonPreview": text(str(temp_reason), 500) if temp_reason else "",
            "tempUnschedulableSet": temp_until is not None or bool(temp_reason),
            "ok": ok,
        })
    return {
        "ok": len(missing) == 0 and len(mismatched) == 0,
        "desired": len(EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE),
        "enabled": enabled_names,
        "missing": missing,
        "mismatched": mismatched,
        "items": items,
        "valuesPrinted": False,
    }

def account_capacity_status(token):
    accounts = list_accounts(token)
    by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
    items = []
    missing = []
    mismatched = []
    expected_capacity_total = 0
    runtime_concurrency_total = 0
    schedulable_runtime_concurrency_total = 0
    available_runtime_concurrency_total = 0
    temp_unschedulable_runtime_concurrency_total = 0
    for name in sorted(EXPECTED_ACCOUNT_CAPACITIES):
        expected = int(EXPECTED_ACCOUNT_CAPACITIES[name])
        expected_capacity_total += expected
        account = by_name.get(name)
        if account is None:
            missing.append(name)
            items.append({
                "accountName": name,
                "accountId": None,
                "expectedCapacity": expected,
                "runtimeConcurrency": None,
                "ok": False,
            })
            continue
        runtime = account.get("concurrency")
        runtime_int = runtime if isinstance(runtime, int) else 0
        runtime_concurrency_total += runtime_int
        if account.get("schedulable") is True:
            runtime_status = account.get("status")
            if runtime_status is None or runtime_status == "active":
                runtime_status_ok = True
            else:
                runtime_status_ok = False
            if runtime_status_ok:
                schedulable_runtime_concurrency_total += runtime_int
                temp_until = account.get("temp_unschedulable_until") or account.get("tempUnschedulableUntil")
                temp_reason = account.get("temp_unschedulable_reason") or account.get("tempUnschedulableReason")
                if temp_until is None and not bool(temp_reason):
                    available_runtime_concurrency_total += runtime_int
                else:
                    temp_unschedulable_runtime_concurrency_total += runtime_int
        ok = runtime == expected
        if not ok:
            mismatched.append(name)
        items.append({
            "accountName": name,
            "accountId": account.get("id"),
            "expectedCapacity": expected,
            "runtimeConcurrency": runtime,
            "priority": account.get("priority"),
            "status": account.get("status"),
            "schedulable": account.get("schedulable"),
            "ok": ok,
        })
    return {
        "ok": len(missing) == 0 and len(mismatched) == 0,
        "defaultAccountCapacity": POOL_DEFAULT_ACCOUNT_CAPACITY,
        "desired": len(EXPECTED_ACCOUNT_CAPACITIES),
        "totals": {
            "expectedCapacityTotal": expected_capacity_total,
            "runtimeConcurrencyTotal": runtime_concurrency_total,
            "schedulableRuntimeConcurrencyTotal": schedulable_runtime_concurrency_total,
            "availableRuntimeConcurrencyTotal": available_runtime_concurrency_total,
            "tempUnschedulableRuntimeConcurrencyTotal": temp_unschedulable_runtime_concurrency_total,
            "minOwnerConcurrency": MIN_OWNER_CONCURRENCY,
            "minOwnerConcurrencySource": MIN_OWNER_CONCURRENCY_SOURCE,
        },
        "missing": missing,
        "mismatched": mismatched,
        "items": items,
        "valuesPrinted": False,
    }

def account_load_factor_status(token):
    accounts = list_accounts(token)
    by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
    items = []
    missing = []
    mismatched = []
    for name in sorted(EXPECTED_ACCOUNT_LOAD_FACTORS):
        expected = int(EXPECTED_ACCOUNT_LOAD_FACTORS[name])
        account = by_name.get(name)
        if account is None:
            missing.append(name)
            items.append({
                "accountName": name,
                "accountId": None,
                "expectedLoadFactor": expected,
                "runtimeLoadFactor": None,
                "ok": False,
            })
            continue
        runtime_raw = account.get("load_factor")
        if runtime_raw is None:
            detail = get_account_detail(token, account)
            runtime_raw = detail.get("load_factor") if isinstance(detail, dict) else None
        try:
            runtime = int(runtime_raw)
        except Exception:
            runtime = runtime_raw
        ok = runtime == expected
        if not ok:
            mismatched.append(name)
        items.append({
            "accountName": name,
            "accountId": account.get("id"),
            "expectedLoadFactor": expected,
            "runtimeLoadFactor": runtime,
            "priority": account.get("priority"),
            "status": account.get("status"),
            "schedulable": account.get("schedulable"),
            "ok": ok,
        })
    return {
        "ok": len(missing) == 0 and len(mismatched) == 0,
        "defaultAccountLoadFactor": POOL_DEFAULT_ACCOUNT_LOAD_FACTOR,
        "desired": len(EXPECTED_ACCOUNT_LOAD_FACTORS),
        "missing": missing,
        "mismatched": mismatched,
        "items": items,
        "valuesPrinted": False,
    }

def account_ws_v2_status(token):
    accounts = list_accounts(token)
    by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)}
    items = []
    missing = []
    mismatched = []
    enabled_names = []
    unschedulable_enabled = []
    schedulable_enabled = []
    for name in sorted(EXPECTED_ACCOUNT_WS_MODES):
        expected_mode = EXPECTED_ACCOUNT_WS_MODES[name]
        expected_enabled = expected_mode not in (None, "", "off")
        account = by_name.get(name)
        if account is None:
            missing.append(name)
            items.append({
                "accountName": name,
                "accountId": None,
                "expectedMode": expected_mode,
                "expectedEnabled": expected_enabled,
                "runtimeMode": None,
                "runtimeEnabled": None,
                "ok": False,
            })
            continue
        extra = account.get("extra") if isinstance(account.get("extra"), dict) else {}
        runtime_mode = extra.get("openai_apikey_responses_websockets_v2_mode")
        runtime_enabled = extra.get("openai_apikey_responses_websockets_v2_enabled")
        schedulable = account.get("schedulable")
        if expected_mode == "off":
            ok = runtime_mode == "off" and runtime_enabled is False
        elif expected_mode is None:
            ok = runtime_mode in (None, "", "off") and runtime_enabled in (None, False)
        else:
            ok = runtime_mode == expected_mode and runtime_enabled is True
        if not ok:
            mismatched.append(name)
        if expected_enabled:
            enabled_names.append(name)
            if schedulable is True:
                schedulable_enabled.append(name)
            else:
                unschedulable_enabled.append(name)
        items.append({
            "accountName": name,
            "accountId": account.get("id"),
            "expectedMode": expected_mode,
            "expectedEnabled": expected_enabled,
            "runtimeMode": runtime_mode,
            "runtimeEnabled": runtime_enabled,
            "status": account.get("status"),
            "schedulable": schedulable,
            "ok": ok and ((not expected_enabled) or schedulable is True),
        })
    availability_ok = len(enabled_names) == 0 or len(schedulable_enabled) > 0
    return {
        "ok": len(missing) == 0 and len(mismatched) == 0 and availability_ok,
        "desired": len(EXPECTED_ACCOUNT_WS_MODES),
        "enabled": enabled_names,
        "schedulableEnabled": schedulable_enabled,
        "unschedulableEnabled": unschedulable_enabled,
        "missing": missing,
        "mismatched": mismatched,
        "items": items,
        "valuesPrinted": False,
    }

def api_key_preview(api_key):
    if len(api_key) <= 14:
        return "***"
    return api_key[:10] + "..." + api_key[-4:]

def run_sync():
    payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8"))
    profiles = payload.get("profiles") or []
    prune_removed = bool(payload.get("pruneRemoved"))
    sentinel_payload = payload.get("sentinel") if isinstance(payload.get("sentinel"), dict) else {}
    if not profiles:
        raise RuntimeError("sync payload has no profiles")
    admin_email, token, admin_compliance = login()
    group, group_action = ensure_group(token)
    group_id = group.get("id") if isinstance(group, dict) else None
    if group_id is None:
        raise RuntimeError("pool group id missing after ensure")
    account_results, pruned_account_results = ensure_accounts(token, profiles, group_id, prune_removed)
    capacity_status = account_capacity_status(token)
    load_factor_status = account_load_factor_status(token)
    ws_v2_status = account_ws_v2_status(token)
    temp_unschedulable_status = account_temp_unschedulable_status(token)
    api_key, secret_action, secret_apply_stdout = ensure_api_key_secret(group_id)
    api_key_result = ensure_sub2api_api_key(token, api_key, group_id)
    owner_balance = ensure_pool_owner_balance(token, api_key_result["userId"])
    owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"])
    gateway = validate_gateway(api_key)
    responses_smoke = validate_gateway_responses(api_key)
    compact_evidence = recent_compact_gateway_evidence()
    responses_evidence = recent_responses_gateway_evidence()
    runtime_capabilities = validate_runtime_capabilities(token)
    sentinel = apply_sentinel_manifest(sentinel_payload.get("manifest"))
    sentinel_reassert = reassert_sentinel_freezes_after_sync(token)
    return {
        "ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and sentinel.get("ok") is True and sentinel_reassert.get("ok") is True,
        "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
        "mode": "sync",
        "namespace": NAMESPACE,
        "serviceDns": SERVICE_DNS,
        "appPod": APP_POD,
        "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
        "pool": {"name": POOL_GROUP_NAME, "id": group_id, "action": group_action, "platform": group.get("platform") if isinstance(group, dict) else "openai"},
        "accounts": {
            "desired": len(profiles),
            "created": sum(1 for item in account_results if item["action"] == "created"),
            "updated": sum(1 for item in account_results if item["action"] == "updated"),
            "pruned": len(pruned_account_results),
            "pruneMode": "explicit" if prune_removed else "disabled-by-default",
            "items": account_results,
            "prunedItems": pruned_account_results,
            "processControl": {"schedulableRestore": "POST /api/v1/admin/accounts/:id/schedulable", "durableConfig": False},
            "valuesPrinted": False,
        },
        "capacity": capacity_status,
        "loadFactor": load_factor_status,
        "webSocketsV2": ws_v2_status,
        "tempUnschedulable": temp_unschedulable_status,
        "apiKey": {
            "name": POOL_API_KEY_NAME,
            "secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}",
            "secretAction": secret_action,
            "secretApply": secret_apply_stdout,
            "sub2apiAction": api_key_result["action"],
            "sub2apiId": api_key_result["id"],
            "groupId": api_key_result["groupId"],
            "userId": api_key_result["userId"],
            "keyPreview": api_key_preview(api_key),
            "valuesPrinted": False,
        },
        "ownerBalance": owner_balance,
        "ownerConcurrency": owner_concurrency,
        "sentinel": {**sentinel, "freezeReassert": sentinel_reassert},
        "runtimeCapabilities": runtime_capabilities,
        "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
    }

def run_validate():
    api_key = decode_secret_value(POOL_API_KEY_SECRET_NAME, POOL_API_KEY_SECRET_KEY)
    if not api_key:
        raise RuntimeError(f"{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY} missing")
    admin_email, token, admin_compliance = login()
    key_item = next((item for item in list_user_keys(token) if item.get("key") == api_key), None)
    owner_balance = None
    owner_concurrency = None
    if key_item is not None and key_item.get("user_id") is not None:
        owner_balance = ensure_pool_owner_balance(token, key_item["user_id"])
        owner_concurrency = ensure_pool_owner_concurrency(token, key_item["user_id"])
    capacity_status = account_capacity_status(token)
    load_factor_status = account_load_factor_status(token)
    ws_v2_status = account_ws_v2_status(token)
    temp_unschedulable_status = account_temp_unschedulable_status(token)
    gateway = validate_gateway(api_key)
    responses_smoke = validate_gateway_responses(api_key)
    compact_evidence = recent_compact_gateway_evidence()
    responses_evidence = recent_responses_gateway_evidence()
    runtime_capabilities = validate_runtime_capabilities(token)
    sentinel = sentinel_runtime_status()
    return {
        "ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True and sentinel.get("ok") is True,
        "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
        "mode": "validate",
        "namespace": NAMESPACE,
        "serviceDns": SERVICE_DNS,
        "appPod": APP_POD,
        "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
        "apiKey": {
            "secret": f"{NAMESPACE}/{POOL_API_KEY_SECRET_NAME}.{POOL_API_KEY_SECRET_KEY}",
            "sub2apiId": key_item.get("id") if isinstance(key_item, dict) else None,
            "userId": key_item.get("user_id") if isinstance(key_item, dict) else None,
            "keyPreview": api_key_preview(api_key),
            "valuesPrinted": False,
        },
        "ownerBalance": owner_balance,
        "ownerConcurrency": owner_concurrency,
        "capacity": capacity_status,
        "loadFactor": load_factor_status,
        "webSocketsV2": ws_v2_status,
        "tempUnschedulable": temp_unschedulable_status,
        "sentinel": sentinel,
        "runtimeCapabilities": runtime_capabilities,
        "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
    }

def parse_embedded_json(stdout):
    if not isinstance(stdout, str) or not stdout.strip():
        return None
    start = stdout.find("{")
    end = stdout.rfind("}")
    if start < 0 or end <= start:
        return None
    try:
        return json.loads(stdout[start:end + 1])
    except Exception:
        return None

def inject_env(template, name, value):
    spec = template.setdefault("spec", {})
    containers = spec.setdefault("containers", [])
    if not containers:
        raise RuntimeError("sentinel job template has no containers")
    container = containers[0]
    env = container.setdefault("env", [])
    env[:] = [item for item in env if not (isinstance(item, dict) and item.get("name") == name)]
    env.append({"name": name, "value": value})

def sentinel_probe_job_manifest(accounts):
    cronjob_name = SENTINEL_CONFIG.get("cronJobName")
    if not isinstance(cronjob_name, str) or not cronjob_name:
        raise RuntimeError("sentinel cronJobName missing from config")
    cronjob = kube_json(["-n", NAMESPACE, "get", "cronjob", cronjob_name], f"cronjob/{cronjob_name}")
    job_template = ((cronjob.get("spec") or {}).get("jobTemplate") or {}) if isinstance(cronjob, dict) else {}
    job_spec = copy_json(job_template.get("spec") or {})
    if not isinstance(job_spec, dict) or not job_spec:
        raise RuntimeError("sentinel CronJob jobTemplate.spec missing")
    template = job_spec.get("template")
    if not isinstance(template, dict):
        raise RuntimeError("sentinel CronJob jobTemplate.spec.template missing")
    inject_env(template, "SENTINEL_ACCOUNT_NAMES", ",".join(accounts))
    job_spec["ttlSecondsAfterFinished"] = int(job_spec.get("ttlSecondsAfterFinished") or 3600)
    suffix = str(int(time.time()))[-8:] + "-" + "".join(secrets.choice(string.ascii_lowercase + string.digits) for _ in range(5))
    job_name = ("sub2api-sentinel-probe-" + suffix)[:63]
    return job_name, {
        "apiVersion": "batch/v1",
        "kind": "Job",
        "metadata": {
            "name": job_name,
            "namespace": NAMESPACE,
            "labels": {
                "app.kubernetes.io/name": cronjob_name,
                "app.kubernetes.io/part-of": "platform-infra",
                "app.kubernetes.io/managed-by": "unidesk",
                "unidesk.ai/job-purpose": "sub2api-account-sentinel-manual-probe",
            },
        },
        "spec": job_spec,
    }

def copy_json(value):
    return json.loads(json.dumps(value))

def job_condition(job, cond_type):
    for item in ((job.get("status") or {}).get("conditions") or []):
        if item.get("type") == cond_type and item.get("status") == "True":
            return item
    return None

def wait_sentinel_probe_job(job_name, timeout_seconds):
    deadline = time.time() + timeout_seconds
    latest = None
    while time.time() < deadline:
        latest, err = safe_kube_json(["-n", NAMESPACE, "get", "job", job_name], f"job/{job_name}")
        if isinstance(latest, dict):
            complete = job_condition(latest, "Complete")
            failed = job_condition(latest, "Failed")
            if complete is not None:
                return "succeeded", latest, complete
            if failed is not None:
                return "failed", latest, failed
        time.sleep(2)
    return "timeout", latest, None

def job_logs(job_name):
    proc = kubectl(["-n", NAMESPACE, "logs", f"job/{job_name}", "--tail=4000"])
    return {
        "exitCode": proc.returncode,
        "stdout": proc.stdout.decode("utf-8", errors="replace"),
        "stderr": text(proc.stderr, 4000),
    }

def run_sentinel_probe():
    payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
    accounts = payload.get("accounts") if isinstance(payload, dict) else None
    if not isinstance(accounts, list) or not accounts or not all(isinstance(item, str) and item for item in accounts):
        raise RuntimeError("sentinel-probe payload requires non-empty accounts")
    job_name, manifest = sentinel_probe_job_manifest(accounts)
    proc = kubectl(["apply", "--server-side", "--force-conflicts", f"--field-manager={FIELD_MANAGER}", "-f", "-"], json.dumps(manifest))
    if proc.returncode != 0:
        raise RuntimeError(f"apply sentinel probe job failed: {text(proc.stderr, 2000)}")
    timeout_seconds = max(300, int(SENTINEL_CONFIG.get("probe", {}).get("timeoutSeconds") or 30) + 240)
    status, job, condition = wait_sentinel_probe_job(job_name, timeout_seconds)
    logs = job_logs(job_name)
    parsed = parse_embedded_json(logs.get("stdout") or "")
    results = parsed.get("results") if isinstance(parsed, dict) and isinstance(parsed.get("results"), list) else []
    requested = set(accounts)
    measured = {item.get("accountName") for item in results if isinstance(item, dict)}
    missing = sorted(name for name in requested if name not in measured)
    marker_ok = len(missing) == 0 and all(isinstance(item, dict) and item.get("accountName") in requested and item.get("markerMatched") is True for item in results if isinstance(item, dict) and item.get("accountName") in requested)
    job_ok = status == "succeeded" and isinstance(parsed, dict) and parsed.get("ok") is True
    return {
        "ok": job_ok and marker_ok,
        "jobExecutionOk": job_ok,
        "markerOk": marker_ok,
        "mode": "sentinel-probe",
        "namespace": NAMESPACE,
        "requestedAccounts": accounts,
        "missingAccounts": missing,
        "job": {
            "name": job_name,
            "status": status,
            "condition": condition,
            "timeoutSeconds": timeout_seconds,
            "applyStdoutTail": text(proc.stdout, 1200),
            "logsExitCode": logs.get("exitCode"),
            "logsStderrTail": logs.get("stderr"),
        },
        "probe": parsed,
        "sentinelState": sentinel_state_summary(),
        "valuesPrinted": False,
    }

def run_cleanup_probes():
    admin_email, token, admin_compliance = login()
    cleanup = cleanup_probe_resources(token)
    return {
        "ok": cleanup.get("ok") is True,
        "mode": "cleanup-probes",
        "namespace": NAMESPACE,
        "serviceDns": SERVICE_DNS,
        "appPod": APP_POD,
        "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance},
        "cleanup": cleanup,
        "valuesPrinted": False,
    }

try:
    if MODE == "sync":
        result = run_sync()
    elif MODE == "cleanup-probes":
        result = run_cleanup_probes()
    elif MODE == "sentinel-probe":
        result = run_sentinel_probe()
    else:
        result = run_validate()
except Exception as exc:
    result = {
        "ok": False,
        "mode": MODE,
        "namespace": NAMESPACE,
        "serviceDns": SERVICE_DNS,
        "appPod": globals().get("APP_POD"),
        "error": str(exc),
        "valuesPrinted": False,
    }

print(json.dumps(result, ensure_ascii=False, indent=2))
sys.exit(0 if result.get("ok") else 1)
PY
`;
}

async function capture(config: UniDeskConfig, target: string, args: string[], input?: string): Promise<SshCaptureResult> {
  return await runSshCommandCapture(config, target, args, input);
}

type RemoteCodexPoolMode = "sync" | "validate" | "sentinel-probe" | "sentinel-image-status" | "sentinel-image-build";

async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: RemoteCodexPoolMode, script: string): Promise<SshCaptureResult> {
  const jobName = `codex-pool-${mode}-${Date.now().toString(36)}`.slice(0, 63);
  const startedAtMs = Date.now();
  const start = await capture(config, g14K3sRoute, ["script"], remoteJobStartScript(jobName, script));
  const started = parseJsonOutput(start.stdout);
  if (start.exitCode !== 0 || boolField(started, "ok", false) !== true) return start;

  let latest: RemoteCodexPoolJobStatus | null = null;
  while (Date.now() - startedAtMs <= remoteJobTimeoutMs) {
    await sleep(remoteJobPollMs);
    const probe = await capture(config, g14K3sRoute, ["script"], remoteJobStatusScript(jobName));
    const parsed = parseJsonOutput(probe.stdout);
    latest = normalizeRemoteJobStatus(parsed);
    process.stderr.write(`${JSON.stringify({
      event: "platform-infra.sub2api.codex-pool.progress",
      at: new Date().toISOString(),
      mode,
      jobName,
      status: latest?.status ?? "unknown",
      exitCode: latest?.exitCode ?? null,
      elapsedMs: Date.now() - startedAtMs,
      stdoutBytes: latest?.stdoutBytes ?? null,
      stderrBytes: latest?.stderrBytes ?? null,
    })}\n`);
    if (probe.exitCode !== 0) {
      return {
        exitCode: probe.exitCode,
        stdout: probe.stdout,
        stderr: `${start.stderr}${probe.stderr}`,
      };
    }
    if (latest?.status === "succeeded" || latest?.status === "failed") {
      return {
        exitCode: latest.exitCode ?? (latest.status === "succeeded" ? 0 : 1),
        stdout: latest.stdout ?? "",
        stderr: latest.stderr ?? "",
      };
    }
  }

  return {
    exitCode: 124,
    stdout: latest?.stdout ?? "",
    stderr: [
      latest?.stderr ?? "",
      `remote codex-pool ${mode} job ${jobName} did not finish within ${remoteJobTimeoutMs}ms`,
      `status command: ${codexPoolModeCommand(mode)}`,
    ].filter(Boolean).join("\n"),
  };
}

function codexPoolModeCommand(mode: RemoteCodexPoolMode): string {
  if (mode === "sync") return "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm";
  if (mode === "sentinel-probe") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account <accountName> --confirm";
  return "bun scripts/cli.ts platform-infra sub2api codex-pool validate";
}

interface RemoteCodexPoolJobStatus {
  status: "running" | "succeeded" | "failed" | "unknown";
  exitCode: number | null;
  stdout: string | null;
  stderr: string | null;
  stdoutBytes: number | null;
  stderrBytes: number | null;
}

function normalizeRemoteJobStatus(parsed: Record<string, unknown> | null): RemoteCodexPoolJobStatus | null {
  if (parsed === null) return null;
  const status = parsed.status === "running" || parsed.status === "succeeded" || parsed.status === "failed" || parsed.status === "unknown"
    ? parsed.status
    : "unknown";
  return {
    status,
    exitCode: typeof parsed.exitCode === "number" ? parsed.exitCode : null,
    stdout: typeof parsed.stdout === "string" ? parsed.stdout : null,
    stderr: typeof parsed.stderr === "string" ? parsed.stderr : null,
    stdoutBytes: typeof parsed.stdoutBytes === "number" ? parsed.stdoutBytes : null,
    stderrBytes: typeof parsed.stderrBytes === "number" ? parsed.stderrBytes : null,
  };
}

function remoteJobStartScript(jobName: string, script: string): string {
  const scriptB64 = Buffer.from(script, "utf8").toString("base64");
  return `
set -eu
job=${shQuote(jobName)}
dir=${shQuote(remoteJobDir)}
mkdir -p "$dir"
chmod 700 "$dir"
script_path="$dir/$job.sh"
stdout_path="$dir/$job.stdout"
stderr_path="$dir/$job.stderr"
exit_path="$dir/$job.exit"
done_path="$dir/$job.done"
pid_path="$dir/$job.pid"
rm -f "$script_path" "$stdout_path" "$stderr_path" "$exit_path" "$done_path" "$pid_path"
base64 -d > "$script_path" <<'UNIDESK_REMOTE_SCRIPT_B64'
${scriptB64}
UNIDESK_REMOTE_SCRIPT_B64
chmod 700 "$script_path"
(
  set +e
  trap '' HUP
  sh "$script_path" > "$stdout_path" 2> "$stderr_path"
  code=$?
  printf '%s' "$code" > "$exit_path"
  date -u +%Y-%m-%dT%H:%M:%SZ > "$done_path"
  rm -f "$script_path"
) >/dev/null 2>&1 &
pid=$!
printf '%s' "$pid" > "$pid_path"
python3 - <<PY
import json
print(json.dumps({"ok": True, "jobName": ${JSON.stringify(jobName)}, "pid": int("${"${pid}"}"), "dir": ${JSON.stringify(remoteJobDir)}}))
PY
`;
}

function remoteJobStatusScript(jobName: string): string {
  return `
set -eu
python3 - <<'PY'
import json
import os
import signal

job = ${JSON.stringify(jobName)}
directory = ${JSON.stringify(remoteJobDir)}
stdout_path = os.path.join(directory, job + ".stdout")
stderr_path = os.path.join(directory, job + ".stderr")
exit_path = os.path.join(directory, job + ".exit")
done_path = os.path.join(directory, job + ".done")
pid_path = os.path.join(directory, job + ".pid")

def read_text(path, limit=None):
    try:
        with open(path, "rb") as handle:
            data = handle.read()
    except FileNotFoundError:
        return None
    if limit is not None and len(data) > limit:
        data = data[-limit:]
    return data.decode("utf-8", errors="replace")

def file_size(path):
    try:
        return os.path.getsize(path)
    except FileNotFoundError:
        return 0

pid_text = read_text(pid_path)
done = os.path.exists(done_path)
exit_text = read_text(exit_path)
exit_code = None
if exit_text is not None:
    try:
        exit_code = int(exit_text.strip())
    except ValueError:
        exit_code = None
running = False
if pid_text is not None and not done:
    try:
        os.kill(int(pid_text.strip()), 0)
        running = True
    except Exception:
        running = False
status = "succeeded" if done and exit_code == 0 else "failed" if done else "running" if running else "unknown"
print(json.dumps({
    "ok": True,
    "jobName": job,
    "status": status,
    "pid": int(pid_text.strip()) if pid_text and pid_text.strip().isdigit() else None,
    "exitCode": exit_code,
    "stdoutBytes": file_size(stdout_path),
    "stderrBytes": file_size(stderr_path),
    "stdout": read_text(stdout_path, 500000) if done else read_text(stdout_path, 12000),
    "stderr": read_text(stderr_path, 120000) if done else read_text(stderr_path, 12000),
    "doneAt": read_text(done_path),
}, ensure_ascii=False))
PY
`;
}

function shQuote(value: string): string {
  return `'${value.replace(/'/gu, `'\\''`)}'`;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function parseJsonOutput(stdout: string): Record<string, unknown> | null {
  const trimmed = stdout.trim();
  if (trimmed.length === 0) return null;
  const start = trimmed.indexOf("{");
  const end = trimmed.lastIndexOf("}");
  if (start === -1 || end === -1 || end <= start) return null;
  try {
    const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown;
    return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
  } catch {
    return null;
  }
}

function boolField(value: Record<string, unknown> | null, key: string, defaultValue: boolean): boolean {
  if (value === null) return defaultValue;
  const field = value[key];
  return typeof field === "boolean" ? field : defaultValue;
}

function compactCapture(result: SshCaptureResult, options: { full?: boolean } = {}): Record<string, unknown> {
  const full = options.full ?? false;
  return {
    exitCode: result.exitCode,
    stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
    stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
    stdoutTail: full || result.exitCode !== 0 ? result.stdout.slice(-8000) : "",
    stderrTail: full || result.exitCode !== 0 ? result.stderr.slice(-4000) : "",
  };
}