From c7a1b877e3e0d47ec1fae2c6909acbbe28ef2c34 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 14 Jun 2026 16:47:03 +0000 Subject: [PATCH] refactor: route sub2api codex pool through target adapter --- config/platform-infra/sub2api.yaml | 44 ++ scripts/src/platform-infra-sub2api-codex.ts | 549 ++++++++++---------- 2 files changed, 323 insertions(+), 270 deletions(-) diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index edf6ef1c..a19d72a6 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -47,6 +47,28 @@ targets: redisMode: local-ephemeral appReplicas: 0 redisReplicas: 0 + codexPool: + sentinelImageBuild: + baseImageCachePolicy: pull + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - host.docker.internal + - 74.48.78.17 + - 192.168.0.0/16 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 10.42.0.0/16 + - 10.43.0.0/16 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 127.0.0.1:5000 + - localhost:5000 - id: D601 route: D601:k3s namespace: platform-infra @@ -63,6 +85,28 @@ targets: dependencyImages: postgres: docker.m.daocloud.io/library/postgres:18-alpine redis: docker.m.daocloud.io/library/redis:8-alpine + codexPool: + sentinelImageBuild: + baseImageCachePolicy: local-if-present + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - host.docker.internal + - 74.48.78.17 + - 192.168.0.0/16 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 10.42.0.0/16 + - 10.43.0.0/16 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 127.0.0.1:5000 + - localhost:5000 publicExposure: enabled: true publicBaseUrl: https://api.pikapython.com diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index fc94571e..0fbec7e9 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import type { RenderedCliResult } from "./output"; -import { applyLocalCaddyManagedSite } from "./pk01-caddy"; +import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest, type PublicServiceExposure, type PublicServiceTarget } from "./platform-infra-public-service"; import { codexPoolSentinelSummary, codexPoolSentinelRuntimeImage, @@ -14,9 +14,9 @@ import { type CodexPoolSentinelConfig, type CodexPoolSentinelProfileSecret, } from "./platform-infra-sub2api-codex-sentinel"; +import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "./secrets"; import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; -const g14K3sRoute = "G14:k3s"; const serviceName = "sub2api"; const fieldManager = "unidesk-platform-infra"; const sub2apiConfigPath = rootPath("config", "platform-infra", "sub2api.yaml"); @@ -70,7 +70,13 @@ interface CodexPoolRuntimeTarget { serviceName: string; serviceDns: string; publicBaseUrl: string | null; + publicExposure: PublicServiceExposure | null; appSecretName: string; + secretsRoot: string; + sentinelImageBuild: { + baseImageCachePolicy: "pull" | "local-if-present"; + noProxy: string; + }; egressProxy: { enabled: boolean; applyToSentinel: boolean; @@ -285,8 +291,8 @@ export function codexPoolHelp(): unknown { poolGroupName: pool.groupName, poolApiKeySecretName: pool.apiKeySecretName, poolApiKeySecretKey: pool.apiKeySecretKey, - publicBaseUrl: pool.publicExposure.enabled ? pool.publicExposure.publicBaseUrl : runtimeTarget.publicBaseUrl, - masterBaseUrl: pool.publicExposure.enabled ? pool.publicExposure.masterBaseUrl : null, + publicBaseUrl: runtimeTarget.publicBaseUrl, + publicExposureSource: `${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure`, sentinelMonitorEnabled: pool.sentinel.monitor.enabled, sentinelActionsEnabled: pool.sentinel.actions.enabled, secretValuesPrinted: false, @@ -591,6 +597,7 @@ function stripBooleanOptions(args: string[], stripped: Set): string[] { interface Sub2ApiRuntimeConfig { defaultTargetId: string; appSecretName: string; + secretsRoot: string; targets: Record[]; } @@ -605,10 +612,14 @@ function readSub2ApiRuntimeConfig(): Sub2ApiRuntimeConfig { const appSecretName = database === null ? null : stringValue(database.secretName); if (appSecretName === null) throw new Error(`${sub2apiConfigPath}.runtime.database.secretName is required`); validateKubernetesName(appSecretName, `${sub2apiConfigPath}.runtime.database.secretName`, true); + const secrets = runtime !== null && isRecord(runtime.secrets) ? runtime.secrets : null; + const secretsRoot = secrets === null ? null : stringValue(secrets.root); + if (secretsRoot === null || !secretsRoot.startsWith("/")) throw new Error(`${sub2apiConfigPath}.runtime.secrets.root must be an absolute path`); if (!Array.isArray(parsed.targets) || !parsed.targets.every(isRecord)) throw new Error(`${sub2apiConfigPath}.targets must be a list`); return { defaultTargetId, appSecretName, + secretsRoot, targets: parsed.targets, }; } @@ -628,6 +639,7 @@ function codexPoolRuntimeTarget(targetId?: string): CodexPoolRuntimeTarget { if (route.length === 0) throw new Error(`${sub2apiConfigPath}.targets[${id}].route is required`); if (targetNamespace === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].namespace is required`); validateKubernetesName(targetNamespace, `${sub2apiConfigPath}.targets[${id}].namespace`, true); + const sentinelImageBuild = readTargetSentinelImageBuild(raw, id); let egressProxy: CodexPoolRuntimeTarget["egressProxy"] = null; if (isRecord(raw.egressProxy) && raw.egressProxy.enabled === true) { @@ -649,10 +661,8 @@ function codexPoolRuntimeTarget(targetId?: string): CodexPoolRuntimeTarget { } let publicBaseUrl: string | null = null; - if (isRecord(raw.publicExposure) && raw.publicExposure.enabled === true) { - publicBaseUrl = normalizeBaseUrl(stringValue(raw.publicExposure.publicBaseUrl)); - if (publicBaseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].publicExposure.publicBaseUrl must be a valid http(s) URL when target public exposure is enabled`); - } + const publicExposure = readTargetPublicExposure(raw, id); + if (publicExposure !== null && publicExposure.enabled) publicBaseUrl = publicExposure.publicBaseUrl; return { id, @@ -661,11 +671,131 @@ function codexPoolRuntimeTarget(targetId?: string): CodexPoolRuntimeTarget { serviceName, serviceDns: `${serviceName}.${targetNamespace}.svc.cluster.local:8080`, publicBaseUrl, + publicExposure, appSecretName: runtimeConfig.appSecretName, + secretsRoot: runtimeConfig.secretsRoot, + sentinelImageBuild, egressProxy, }; } +function readTargetSentinelImageBuild(raw: Record, targetId: string): CodexPoolRuntimeTarget["sentinelImageBuild"] { + const codexPool = isRecord(raw.codexPool) ? raw.codexPool : null; + const imageBuild = codexPool !== null && isRecord(codexPool.sentinelImageBuild) ? codexPool.sentinelImageBuild : null; + if (imageBuild === null) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild must be a YAML object`); + const policy = stringValue(imageBuild.baseImageCachePolicy); + if (policy !== "pull" && policy !== "local-if-present") { + throw new Error(`${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild.baseImageCachePolicy must be pull or local-if-present`); + } + return { + baseImageCachePolicy: policy, + noProxy: readTargetNoProxy(imageBuild.noProxy, `${sub2apiConfigPath}.targets[${targetId}].codexPool.sentinelImageBuild.noProxy`), + }; +} + +function readTargetPublicExposure(raw: Record, targetId: string): PublicServiceExposure | null { + if (raw.publicExposure === undefined || raw.publicExposure === null) return null; + if (!isRecord(raw.publicExposure)) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].publicExposure must be a YAML object`); + const exposure = raw.publicExposure; + const enabled = readRequiredTargetBoolean(exposure.enabled, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.enabled`); + const dns = readRequiredTargetRecord(exposure.dns, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns`); + const frpc = readRequiredTargetRecord(exposure.frpc, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc`); + const pk01 = readRequiredTargetRecord(exposure.pk01, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01`); + const publicBaseUrl = readTargetBaseUrl(exposure.publicBaseUrl, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.publicBaseUrl`, "https:"); + const hostname = readRequiredTargetString(dns.hostname, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.hostname`); + if (new URL(publicBaseUrl).hostname !== hostname) throw new Error(`${sub2apiConfigPath}.targets[${targetId}].publicExposure publicBaseUrl hostname must match dns.hostname`); + const config: PublicServiceExposure = { + enabled, + publicBaseUrl, + dns: { + hostname, + expectedA: readRequiredTargetString(dns.expectedA, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.expectedA`), + resolvers: readTargetStringArray(dns.resolvers, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.dns.resolvers`), + }, + frpc: { + deploymentName: readRequiredTargetString(frpc.deploymentName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.deploymentName`), + secretName: readRequiredTargetString(frpc.secretName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretName`), + secretKey: readRequiredTargetString(frpc.secretKey, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretKey`), + image: readRequiredTargetString(frpc.image, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.image`), + serverAddr: readRequiredTargetString(frpc.serverAddr, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverAddr`), + serverPort: readTargetPort(frpc.serverPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverPort`), + proxyName: readRequiredTargetString(frpc.proxyName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.proxyName`), + remotePort: readTargetPort(frpc.remotePort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.remotePort`), + localIP: readRequiredTargetString(frpc.localIP, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localIP`), + localPort: readTargetPort(frpc.localPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localPort`), + tokenSourceRef: readRequiredTargetString(frpc.tokenSourceRef, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.tokenSourceRef`), + tokenSourceKey: readRequiredTargetString(frpc.tokenSourceKey, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.tokenSourceKey`), + }, + pk01: { + route: readRequiredTargetString(pk01.route, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.route`), + caddyConfigPath: readAbsoluteTargetPath(pk01.caddyConfigPath, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.caddyConfigPath`), + caddyServiceName: readRequiredTargetString(pk01.caddyServiceName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.caddyServiceName`), + responseHeaderTimeoutSeconds: readPositiveTargetInteger(pk01.responseHeaderTimeoutSeconds, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.pk01.responseHeaderTimeoutSeconds`), + }, + }; + validateKubernetesName(config.frpc.deploymentName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.deploymentName`, true); + validateKubernetesName(config.frpc.secretName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.secretName`, true); + validateProxyName(config.frpc.proxyName, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.proxyName`); + validatePort(config.frpc.serverPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.serverPort`); + validatePort(config.frpc.remotePort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.remotePort`); + validatePort(config.frpc.localPort, `${sub2apiConfigPath}.targets[${targetId}].publicExposure.frpc.localPort`); + return config; +} + +function readRequiredTargetRecord(value: unknown, path: string): Record { + if (!isRecord(value)) throw new Error(`${path} must be a YAML object`); + return value; +} + +function readRequiredTargetString(value: unknown, path: string): string { + const text = stringValue(value); + if (text === null) throw new Error(`${path} must be a non-empty string`); + if (/[\r\n]/u.test(text)) throw new Error(`${path} must not contain newlines`); + return text; +} + +function readRequiredTargetBoolean(value: unknown, path: string): boolean { + const parsed = booleanValue(value); + if (parsed === null) throw new Error(`${path} must be a boolean`); + return parsed; +} + +function readTargetBaseUrl(value: unknown, path: string, protocol: "http:" | "https:" | null = null): string { + const baseUrl = normalizeBaseUrl(stringValue(value)); + if (baseUrl === null) throw new Error(`${path} must be a valid http(s) URL`); + if (protocol !== null && new URL(baseUrl).protocol !== protocol) throw new Error(`${path} must use ${protocol}//`); + return baseUrl; +} + +function readTargetPort(value: unknown, path: string): number { + const port = numberValue(value); + if (port === null || !Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path} must be an integer TCP port`); + return port; +} + +function readPositiveTargetInteger(value: unknown, path: string): number { + const number = numberValue(value); + if (number === null || !Number.isInteger(number) || number < 1) throw new Error(`${path} must be a positive integer`); + return number; +} + +function readAbsoluteTargetPath(value: unknown, path: string): string { + const text = readRequiredTargetString(value, path); + if (!text.startsWith("/")) throw new Error(`${path} must be an absolute path`); + return text; +} + +function readTargetStringArray(value: unknown, path: string): string[] { + if (!Array.isArray(value)) throw new Error(`${path} must be a YAML array`); + const result = value.map((item, index) => readRequiredTargetString(item, `${path}[${index}]`)); + if (result.length === 0) throw new Error(`${path} must not be empty`); + return result; +} + +function readTargetNoProxy(value: unknown, path: string): string { + return readTargetStringArray(value, path).join(","); +} + function targetFlag(target: CodexPoolRuntimeTarget): string { return target.id === defaultCodexPoolRuntimeTargetId() ? "" : ` --target ${target.id}`; } @@ -676,7 +806,7 @@ function codexPoolPlan(options?: DisclosureOptions): Record { const runtimeTarget = codexPoolRuntimeTarget(resolvedOptions.targetId); const profiles = collectCodexProfiles(); const ok = profiles.length > 0 && profiles.every((profile) => profile.ok); - const consumerBaseUrl = codexConsumerBaseUrl(pool, runtimeTarget); + const consumerBaseUrl = runtimeTarget.publicBaseUrl === null ? null : codexConsumerBaseUrl(pool, runtimeTarget); return { ok, action: "platform-infra-sub2api-codex-pool-plan", @@ -689,7 +819,7 @@ function codexPoolPlan(options?: DisclosureOptions): Record { target: poolTarget(pool, runtimeTarget), config: { path: codexPoolConfigPath, - pool: options.full ? pool : codexPoolConfigSummary(pool), + pool: options.full ? pool : codexPoolConfigSummary(pool, runtimeTarget), }, profiles: options.full ? profiles.map(redactProfile) : profiles.map(compactProfile), decision: { @@ -699,11 +829,9 @@ function codexPoolPlan(options?: DisclosureOptions): Record { sentinel: pool.sentinel.monitor.enabled ? `Account sentinel is enabled as k8s CronJob ${runtimeTarget.namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.` : "Account sentinel monitoring is disabled by YAML.", - publicExposure: pool.publicExposure.enabled - ? `Default Codex consumers use ${consumerBaseUrl}; bounded master-local probes may use ${pool.publicExposure.masterBaseUrl}. Legacy Codex-pool FRP proxy ${pool.publicExposure.proxyName} maps public ${pool.publicExposure.publicBaseUrl} to ${pool.publicExposure.localIP}:${pool.publicExposure.localPort}.` - : runtimeTarget.publicBaseUrl === null - ? "Public FRP exposure is disabled by YAML." - : `Legacy Codex-pool FRP exposure is disabled by YAML; Codex consumers for target ${runtimeTarget.id} use target-level public exposure ${consumerBaseUrl}.`, + publicExposure: runtimeTarget.publicBaseUrl === null + ? `Target-level public exposure is disabled or absent in ${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.` + : `Codex consumers for target ${runtimeTarget.id} use target-level public exposure ${consumerBaseUrl}.`, idempotency: "sync reuses the group, account names, and k3s Secret when they already exist; credentials are updated from the current local Codex files for YAML-managed profiles only; managed accounts missing from YAML are preserved unless --prune-removed is explicitly provided.", configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.", manualAccountProtection: pool.manualAccounts.protected.length === 0 @@ -1049,12 +1177,13 @@ async function codexPoolCleanupProbes(config: UniDeskConfig, options: ConfirmOpt async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); - if (!pool.publicExposure.enabled) { + if (runtimeTarget.publicExposure === null || !runtimeTarget.publicExposure.enabled) { return { ok: true, action: "platform-infra-sub2api-codex-pool-expose", mode: "disabled-by-yaml", target: poolTarget(pool, runtimeTarget), + source: `${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure`, }; } if (!options.confirm) { @@ -1063,24 +1192,24 @@ async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): action: "platform-infra-sub2api-codex-pool-expose", mode: "dry-run", target: poolTarget(pool, runtimeTarget), - publicExposure: publicExposureSummary(pool, runtimeTarget), + publicExposure: targetPublicExposureSummary(runtimeTarget), next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool expose${targetFlag(runtimeTarget)} --confirm`, }, }; } - const masterResult = await applyMasterFrpsAllowPort(pool); - const caddyResult = await applyMasterCaddySite(pool); - const remoteResult = await capture(config, runtimeTarget.route, ["script"], exposeScript(pool, runtimeTarget)); + const secretMaterial = prepareTargetPublicExposureSecret(runtimeTarget); + const caddyResult = await applyPk01CaddyBlock(config, serviceName, runtimeTarget.publicExposure); + const remoteResult = await capture(config, runtimeTarget.route, ["script"], targetPublicExposureApplyScript(runtimeTarget, secretMaterial)); const parsed = parseJsonOutput(remoteResult.stdout); - const publicProbe = await probePublicModels(pool, "without-api-key", undefined, "public"); - const ok = masterResult.ok && caddyResult.ok && remoteResult.exitCode === 0 && boolField(parsed, "ok", false) && publicProbe.httpStatus === 401; + const publicProbe = await probePublicModels(pool, "without-api-key", undefined, runtimeTarget); + const ok = caddyResult.ok === true && remoteResult.exitCode === 0 && boolField(parsed, "ok", false) && publicProbe.httpStatus === 401; if (options.raw) { return { ok, action: "platform-infra-sub2api-codex-pool-expose", - masterFrps: masterResult, - masterCaddy: caddyResult, + frpcSecret: secretMaterialSummary(secretMaterial), + pk01Caddy: caddyResult, remote: compactCapture(remoteResult, { full: true }), parsed, publicProbe, @@ -1090,14 +1219,14 @@ async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): ok, action: "platform-infra-sub2api-codex-pool-expose", mode: "confirmed", - publicExposure: publicExposureSummary(pool, runtimeTarget), - masterFrps: masterResult, - masterCaddy: caddyResult, + publicExposure: targetPublicExposureSummary(runtimeTarget), + frpcSecret: secretMaterialSummary(secretMaterial), + pk01Caddy: caddyResult, remote: parsed ?? compactCapture(remoteResult, { full: options.full || remoteResult.exitCode !== 0 }), publicProbe, next: { - configureLocal: "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm", - validate: "bun scripts/cli.ts platform-infra sub2api codex-pool validate", + configureLocal: `bun scripts/cli.ts platform-infra sub2api codex-pool configure-local${targetFlag(runtimeTarget)} --confirm`, + validate: `bun scripts/cli.ts platform-infra sub2api codex-pool validate${targetFlag(runtimeTarget)}`, }, }; } @@ -1105,7 +1234,7 @@ async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOptions): Promise> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); - const consumerExposureAvailable = pool.publicExposure.enabled || runtimeTarget.publicBaseUrl !== null; + const consumerExposureAvailable = runtimeTarget.publicBaseUrl !== null; const codexDir = join(homedir(), ".codex"); const configPath = join(codexDir, "config.toml"); const authPath = join(codexDir, "auth.json"); @@ -1137,7 +1266,7 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp }, }; } - if (!consumerExposureAvailable) throw new Error(`${codexPoolConfigPath}.publicExposure.enabled is false and ${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.enabled is not true; configure-local needs one consumer URL`); + if (!consumerExposureAvailable) throw new Error(`${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.enabled must be true; configure-local needs one consumer URL`); const keyResult = await fetchPoolApiKey(config, pool, runtimeTarget); if (keyResult.apiKey === null) { return { @@ -1927,7 +2056,7 @@ function tempUnschedulableSummary(policy: CodexTempUnschedulablePolicy): Record< }; } -function codexPoolConfigSummary(pool: CodexPoolConfig): Record { +function codexPoolConfigSummary(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): Record { const accountCapacityTotal = desiredAccountCapacityTotal(pool); return { version: pool.version, @@ -1954,7 +2083,7 @@ function codexPoolConfigSummary(pool: CodexPoolConfig): Record protected: pool.manualAccounts.protected, controlPolicy: "manual accounts are not created, credential-updated, pruned, probed, or frozen by UniDesk codex-pool sync/sentinel; optional proxy_id and pool group membership bindings are narrow YAML-controlled exceptions", }, - publicExposure: publicExposureSummary(pool), + publicExposure: targetPublicExposureSummary(target), localCodex: pool.localCodex, sentinel: codexPoolSentinelSummary(pool.sentinel), disclosure: { @@ -2899,11 +3028,17 @@ function poolTarget(pool = readCodexPoolConfig(), target = codexPoolRuntimeTarge service: serviceName, serviceDns: target.serviceDns, publicBaseUrl: target.publicBaseUrl, - consumerBaseUrl: pool.publicExposure.enabled || target.publicBaseUrl !== null ? codexConsumerBaseUrl(pool, target) : null, + consumerBaseUrl: target.publicBaseUrl === null ? null : codexConsumerBaseUrl(pool, target), configPath: codexPoolConfigPath, groupName: pool.groupName, apiKeyName: pool.apiKeyName, apiKeySecret: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, + publicExposure: targetPublicExposureSummary(target), + sentinelImageBuild: { + source: `${sub2apiConfigPath}.targets[${target.id}].codexPool.sentinelImageBuild`, + baseImageCachePolicy: target.sentinelImageBuild.baseImageCachePolicy, + noProxy: target.sentinelImageBuild.noProxy, + }, minOwnerConcurrency: pool.minOwnerConcurrency, minOwnerConcurrencySource: pool.minOwnerConcurrencySource, accountCapacityTotal: desiredAccountCapacityTotal(pool), @@ -2943,206 +3078,139 @@ function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProf })); } -function publicExposureSummary(pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): Record { +function targetPublicExposureSummary(target: CodexPoolRuntimeTarget): Record | null { + const exposure = target.publicExposure; + if (exposure === null) return null; return { - enabled: pool.publicExposure.enabled, - proxyName: pool.publicExposure.proxyName, - namespace: target.namespace, - configMapName: pool.publicExposure.configMapName, - deploymentName: pool.publicExposure.deploymentName, - frpcImage: pool.publicExposure.frpcImage, - frps: { - serverAddr: pool.publicExposure.serverAddr, - serverPort: pool.publicExposure.serverPort, - remotePort: pool.publicExposure.remotePort, - masterConfigPath: pool.publicExposure.masterFrps.configPath, - masterContainerName: pool.publicExposure.masterFrps.containerName, - }, - caddy: { - enabled: pool.publicExposure.masterCaddy.enabled, - domain: pool.publicExposure.masterCaddy.domain, - configPath: pool.publicExposure.masterCaddy.configPath, - serviceName: pool.publicExposure.masterCaddy.serviceName, - upstreamBaseUrl: pool.publicExposure.masterCaddy.upstreamBaseUrl, - responseHeaderTimeoutSeconds: pool.publicExposure.masterCaddy.responseHeaderTimeoutSeconds, - edgeRetry: pool.publicExposure.masterCaddy.edgeRetry, - }, - upstream: { - localIP: pool.publicExposure.localIP, - localPort: pool.publicExposure.localPort, + source: `${sub2apiConfigPath}.targets[${target.id}].publicExposure`, + enabled: exposure.enabled, + target: { + id: target.id, + route: target.route, + namespace: target.namespace, serviceDns: target.serviceDns, }, urls: { - publicBaseUrl: pool.publicExposure.publicBaseUrl, - masterBaseUrl: pool.publicExposure.masterBaseUrl, + publicBaseUrl: exposure.publicBaseUrl, + consumerBaseUrl: target.publicBaseUrl === null ? null : `${target.publicBaseUrl.replace(/\/+$/u, "")}/`, }, - }; -} - -async function applyMasterFrpsAllowPort(pool: CodexPoolConfig): Promise> { - const path = pool.publicExposure.masterFrps.configPath; - const port = pool.publicExposure.remotePort; - const container = pool.publicExposure.masterFrps.containerName; - if (!existsSync(path)) { - return { ok: false, error: "master-frps-config-missing", path, valuesPrinted: false }; - } - const before = readFileSync(path, "utf8"); - const alreadyAllowed = frpsAllowPortExists(before, port); - let backupPath: string | null = null; - let action = "kept-existing"; - if (!alreadyAllowed) { - const stamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/\..*$/u, "Z"); - backupPath = `${path}.bak-sub2api-${stamp}`; - copyFileSync(path, backupPath); - const next = before.replace(/\s*$/u, "") + `\n\n[[allowPorts]]\nstart = ${port}\nend = ${port}\n`; - writeFileSync(path, next, "utf8"); - chmodSync(path, statSync(backupPath).mode & 0o777); - action = "added-allow-port"; - } - const restart = alreadyAllowed ? null : Bun.spawnSync(["docker", "restart", container]); - const inspect = Bun.spawnSync(["docker", "inspect", "-f", "{{.State.Running}}", container]); - const ok = alreadyAllowed || (restart?.exitCode === 0 && inspect.exitCode === 0 && String(inspect.stdout).trim() === "true"); - return { - ok, - action, - path, - backupPath, - remotePort: port, - containerName: container, - restart: restart === null ? null : { - exitCode: restart.exitCode, - stdoutTail: Buffer.from(restart.stdout).toString("utf8").slice(-1000), - stderrTail: Buffer.from(restart.stderr).toString("utf8").slice(-1000), - }, - inspect: { - exitCode: inspect.exitCode, - running: String(inspect.stdout).trim() === "true", - stderrTail: Buffer.from(inspect.stderr).toString("utf8").slice(-1000), + dns: exposure.dns, + frpc: { + deploymentName: exposure.frpc.deploymentName, + secretName: exposure.frpc.secretName, + secretKey: exposure.frpc.secretKey, + image: exposure.frpc.image, + serverAddr: exposure.frpc.serverAddr, + serverPort: exposure.frpc.serverPort, + proxyName: exposure.frpc.proxyName, + remotePort: exposure.frpc.remotePort, + localIP: exposure.frpc.localIP, + localPort: exposure.frpc.localPort, + tokenSourceRef: exposure.frpc.tokenSourceRef, + tokenSourceKey: exposure.frpc.tokenSourceKey, }, + pk01: exposure.pk01, valuesPrinted: false, }; } - -async function applyMasterCaddySite(pool: CodexPoolConfig): Promise> { - const caddy = pool.publicExposure.masterCaddy; - if (!caddy.enabled) return { ok: true, action: "disabled-by-yaml", valuesPrinted: false }; - const path = caddy.configPath; - if (!existsSync(path)) return { ok: false, error: "master-caddy-config-missing", path, valuesPrinted: false }; - const desiredBlock = renderCaddySiteBlock(caddy.domain, caddy.upstreamBaseUrl, caddy.responseHeaderTimeoutSeconds, caddy.edgeRetry); - const applied = applyLocalCaddyManagedSite({ - serviceId: "sub2api-codex-pool", - markerName: "sub2api-codex-pool", - configPath: path, - serviceName: caddy.serviceName, - siteBlock: desiredBlock, - legacySiteDomain: caddy.domain, +function prepareTargetPublicExposureSecret(target: CodexPoolRuntimeTarget): ReturnType { + if (target.publicExposure === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure is required`); + return prepareFrpcSecret({ + secretRoot: target.secretsRoot, + exposure: target.publicExposure, + sourcePathRedactor: redactRepoPath, + parseEnvFile, + requiredEnvValue, + readTextFile: (sourcePath) => { + if (!existsSync(sourcePath)) throw new Error(`publicExposure requires ${redactRepoPath(sourcePath)} with ${target.publicExposure?.frpc.tokenSourceKey}; materialize the PK01 frps token source first`); + return readTextFile(sourcePath); + }, }); +} + +function secretMaterialSummary(secret: ReturnType): Record { return { - ...applied, - domain: caddy.domain, - upstreamBaseUrl: caddy.upstreamBaseUrl, - responseHeaderTimeoutSeconds: caddy.responseHeaderTimeoutSeconds, - edgeRetry: caddy.edgeRetry, + sourceRef: secret.sourceRef, + sourcePath: secret.sourcePath, + secretName: secret.secretName, + secretKey: secret.secretKey, + fingerprint: secret.fingerprint, valuesPrinted: false, }; } -export function renderCaddySiteBlock( - domain: string, - upstreamBaseUrl: string, - responseHeaderTimeoutSeconds = 180, - edgeRetry: CodexPoolCaddyEdgeRetryConfig | null = null, -): string { - const upstream = new URL(upstreamBaseUrl); - const upstreamHost = `${upstream.hostname}${upstream.port ? `:${upstream.port}` : ""}`; - const retryBlock = renderCaddyEdgeRetryBlock(edgeRetry); - return `${domain} { - encode zstd gzip - reverse_proxy ${upstreamHost} { -${retryBlock} header_up Host {host} - header_up X-Real-IP {remote_host} - transport http { - dial_timeout 5s - response_header_timeout ${responseHeaderTimeoutSeconds}s - } - } -}`; -} - -function renderCaddyEdgeRetryBlock(edgeRetry: CodexPoolCaddyEdgeRetryConfig | null): string { - if (edgeRetry === null || !edgeRetry.enabled) return ""; - const matchLines = [ - edgeRetry.retryMatch.methods.length > 0 ? ` method ${edgeRetry.retryMatch.methods.join(" ")}` : null, - edgeRetry.retryMatch.paths.length > 0 ? ` path ${edgeRetry.retryMatch.paths.join(" ")}` : null, - ].filter((line): line is string => line !== null); - return ` lb_try_duration ${edgeRetry.tryDurationSeconds}s - lb_try_interval ${edgeRetry.tryIntervalMilliseconds}ms - lb_retry_match { -${matchLines.join("\n")} - } -`; -} - -function frpsAllowPortExists(toml: string, port: number): boolean { - const sections = toml.split(/(?=\[\[allowPorts\]\])/u); - return sections.some((section) => { - if (!section.includes("[[allowPorts]]")) return false; - const start = section.match(/^\s*start\s*=\s*(\d+)\s*$/mu); - const end = section.match(/^\s*end\s*=\s*(\d+)\s*$/mu); - return Number(start?.[1]) === port && Number(end?.[1]) === port; - }); -} - -function exposeScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { - const manifest = frpcManifest(pool, target); - const encoded = Buffer.from(manifest, "utf8").toString("base64"); +function targetPublicExposureApplyScript(target: CodexPoolRuntimeTarget, secret: ReturnType): string { + if (target.publicExposure === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure is required`); + const manifest = renderFrpcManifest({ + id: target.id, + route: target.route, + namespace: target.namespace, + replicas: 1, + publicExposure: target.publicExposure, + } satisfies PublicServiceTarget); + const manifestB64 = Buffer.from(manifest, "utf8").toString("base64"); + const frpcTomlB64 = Buffer.from(secret.frpcToml, "utf8").toString("base64"); return ` set -u tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT -manifest="$tmp/sub2api-frpc.yaml" -printf '%s' '${encoded}' | base64 -d > "$manifest" +manifest="$tmp/sub2api-public-exposure.yaml" +frpc_toml="$tmp/frpc.toml" +printf '%s' '${manifestB64}' | base64 -d > "$manifest" +printf '%s' '${frpcTomlB64}' | base64 -d > "$frpc_toml" ns_out="$tmp/ns.out" ns_err="$tmp/ns.err" +secret_out="$tmp/secret.out" +secret_err="$tmp/secret.err" apply_out="$tmp/apply.out" apply_err="$tmp/apply.err" rollout_out="$tmp/rollout.out" rollout_err="$tmp/rollout.err" -kubectl get namespace ${target.namespace} >"$ns_out" 2>"$ns_err" +pods_json="$tmp/pods.json" +pods_err="$tmp/pods.err" +logs_out="$tmp/logs.out" +logs_err="$tmp/logs.err" +kubectl create namespace ${target.namespace} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$ns_out" 2>"$ns_err" ns_rc=$? +secret_rc=1 apply_rc=1 rollout_rc=1 if [ "$ns_rc" -eq 0 ]; then + kubectl -n ${target.namespace} create secret generic ${secret.secretName} \\ + --from-file=${secret.secretKey}="$frpc_toml" \\ + --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$secret_out" 2>"$secret_err" + secret_rc=$? +else + : >"$secret_out" + printf '%s\\n' 'skipped because namespace apply failed' >"$secret_err" +fi +if [ "$ns_rc" -eq 0 ] && [ "$secret_rc" -eq 0 ]; then kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$apply_out" 2>"$apply_err" apply_rc=$? if [ "$apply_rc" -eq 0 ]; then - kubectl -n ${target.namespace} rollout status deployment/${pool.publicExposure.deploymentName} --timeout=30s >"$rollout_out" 2>"$rollout_err" + kubectl -n ${target.namespace} rollout status deployment/${target.publicExposure.frpc.deploymentName} --timeout=60s >"$rollout_out" 2>"$rollout_err" rollout_rc=$? else printf '%s\\n' 'skipped because apply failed' >"$rollout_err" fi else : >"$apply_out" - printf '%s\\n' 'skipped because namespace is missing' >"$apply_err" + printf '%s\\n' 'skipped because namespace or secret step failed' >"$apply_err" : >"$rollout_out" - printf '%s\\n' 'skipped because namespace is missing' >"$rollout_err" + printf '%s\\n' 'skipped because namespace or secret step failed' >"$rollout_err" fi -pods_json="$tmp/pods.json" -pods_err="$tmp/pods.err" -kubectl -n ${target.namespace} get pods -l app.kubernetes.io/name=${pool.publicExposure.deploymentName} -o json >"$pods_json" 2>"$pods_err" +kubectl -n ${target.namespace} get pods -l app.kubernetes.io/name=${target.publicExposure.frpc.deploymentName} -o json >"$pods_json" 2>"$pods_err" pods_rc=$? -logs_out="$tmp/logs.out" -logs_err="$tmp/logs.err" -kubectl -n ${target.namespace} logs deployment/${pool.publicExposure.deploymentName} --tail=80 >"$logs_out" 2>"$logs_err" +kubectl -n ${target.namespace} logs deployment/${target.publicExposure.frpc.deploymentName} --tail=80 >"$logs_out" 2>"$logs_err" logs_rc=$? -python3 - "$tmp" "$ns_rc" "$apply_rc" "$rollout_rc" "$pods_rc" "$logs_rc" <<'PY' +python3 - "$tmp" "$ns_rc" "$secret_rc" "$apply_rc" "$rollout_rc" "$pods_rc" "$logs_rc" <<'PY' import json import os import sys tmp = sys.argv[1] -ns_rc, apply_rc, rollout_rc, pods_rc, logs_rc = [int(value) for value in sys.argv[2:]] +ns_rc, secret_rc, apply_rc, rollout_rc, pods_rc, logs_rc = [int(value) for value in sys.argv[2:]] def text(name, limit=4000): path = os.path.join(tmp, name) @@ -3173,23 +3241,26 @@ if isinstance(pods_obj, dict): logs = text("logs.out", 4000) payload = { - "ok": ns_rc == 0 and apply_rc == 0 and rollout_rc == 0 and pods_rc == 0 and len(pods) > 0 and all(p["ready"] for p in pods), + "ok": ns_rc == 0 and secret_rc == 0 and apply_rc == 0 and rollout_rc == 0 and pods_rc == 0 and len(pods) > 0 and all(p["ready"] for p in pods), + "target": "${target.id}", "namespace": "${target.namespace}", + "source": "${sub2apiConfigPath}.targets[${target.id}].publicExposure", "proxy": { - "name": "${pool.publicExposure.proxyName}", - "remotePort": ${JSON.stringify(pool.publicExposure.remotePort)}, - "publicBaseUrl": "${pool.publicExposure.publicBaseUrl}", - "masterBaseUrl": "${pool.publicExposure.masterBaseUrl}", - "localIP": "${pool.publicExposure.localIP}", - "localPort": ${JSON.stringify(pool.publicExposure.localPort)}, + "name": "${target.publicExposure.frpc.proxyName}", + "remotePort": ${JSON.stringify(target.publicExposure.frpc.remotePort)}, + "publicBaseUrl": "${target.publicExposure.publicBaseUrl}", + "localIP": "${target.publicExposure.frpc.localIP}", + "localPort": ${JSON.stringify(target.publicExposure.frpc.localPort)}, }, "resources": { - "configMap": "${pool.publicExposure.configMapName}", - "deployment": "${pool.publicExposure.deploymentName}", - "image": "${pool.publicExposure.frpcImage}", + "secret": "${secret.secretName}", + "secretKey": "${secret.secretKey}", + "deployment": "${target.publicExposure.frpc.deploymentName}", + "image": "${target.publicExposure.frpc.image}", }, "steps": { "namespace": {"exitCode": ns_rc, "stdout": text("ns.out"), "stderr": text("ns.err")}, + "secret": {"exitCode": secret_rc, "stdout": text("secret.out"), "stderr": text("secret.err"), "valuesPrinted": False}, "apply": {"exitCode": apply_rc, "stdout": text("apply.out"), "stderr": text("apply.err")}, "rollout": {"exitCode": rollout_rc, "stdout": text("rollout.out"), "stderr": text("rollout.err")}, "pods": {"exitCode": pods_rc, "items": pods, "stderr": text("pods.err")}, @@ -3200,6 +3271,7 @@ payload = { "stderrTail": text("logs.err"), }, }, + "valuesPrinted": False, } print(json.dumps(payload, ensure_ascii=False, indent=2)) sys.exit(0 if payload["ok"] else 1) @@ -3207,68 +3279,6 @@ PY `; } -function frpcManifest(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { - return `apiVersion: v1 -kind: ConfigMap -metadata: - name: ${pool.publicExposure.configMapName} - namespace: ${target.namespace} - labels: - app.kubernetes.io/name: ${pool.publicExposure.deploymentName} - app.kubernetes.io/part-of: platform-infra - app.kubernetes.io/managed-by: unidesk -data: - frpc.toml: | - serverAddr = "${pool.publicExposure.serverAddr}" - serverPort = ${pool.publicExposure.serverPort} - loginFailExit = true - - [[proxies]] - name = "${pool.publicExposure.proxyName}" - type = "tcp" - localIP = "${pool.publicExposure.localIP}" - localPort = ${pool.publicExposure.localPort} - remotePort = ${pool.publicExposure.remotePort} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${pool.publicExposure.deploymentName} - namespace: ${target.namespace} - labels: - app.kubernetes.io/name: ${pool.publicExposure.deploymentName} - app.kubernetes.io/part-of: platform-infra - app.kubernetes.io/managed-by: unidesk -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ${pool.publicExposure.deploymentName} - template: - metadata: - labels: - app.kubernetes.io/name: ${pool.publicExposure.deploymentName} - app.kubernetes.io/part-of: platform-infra - app.kubernetes.io/managed-by: unidesk - spec: - containers: - - name: frpc - image: ${pool.publicExposure.frpcImage} - imagePullPolicy: IfNotPresent - args: - - -c - - /etc/frp/frpc.toml - volumeMounts: - - name: config - mountPath: /etc/frp - readOnly: true - volumes: - - name: config - configMap: - name: ${pool.publicExposure.configMapName} -`; -} - async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): Promise<{ apiKey: string | null; error: string | null }> { const result = await capture(config, target.route, ["script"], ` set -u @@ -3360,7 +3370,9 @@ function updateCodexConfigToml(current: string, pool: CodexPoolConfig, target = } function codexConsumerBaseUrl(pool: CodexPoolConfig, target = codexPoolRuntimeTarget()): string { - const baseUrl = target.publicBaseUrl ?? pool.publicExposure.publicBaseUrl; + void pool; + const baseUrl = target.publicBaseUrl; + if (baseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure.enabled must be true before resolving the Codex consumer base URL`); return `${baseUrl.replace(/\/+$/u, "")}/`; } @@ -3458,8 +3470,7 @@ function upsertTomlSectionBoolean(text: string, sectionName: string, key: string } async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget()): Promise> { - const probeBase = target.publicBaseUrl === null ? "public" : "target-public"; - const probe = await probePublicModels(pool, "with-api-key", apiKey, probeBase, target); + const probe = await probePublicModels(pool, "with-api-key", apiKey, target); return { ...probe, ok: probe.ok === true, @@ -3469,12 +3480,10 @@ async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: strin }; } -async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, base: "master" | "public" | "target-public" = "master", target = codexPoolRuntimeTarget()): Promise> { - const baseUrl = base === "target-public" - ? (target.publicBaseUrl ?? pool.publicExposure.publicBaseUrl) - : base === "public" - ? pool.publicExposure.publicBaseUrl - : pool.publicExposure.masterBaseUrl; +async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, target = codexPoolRuntimeTarget()): Promise> { + void pool; + if (target.publicBaseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${target.id}].publicExposure.enabled must be true before probing the public gateway`); + const baseUrl = target.publicBaseUrl; const url = `${baseUrl.replace(/\/+$/u, "")}/v1/models`; const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 8000); @@ -3967,7 +3976,7 @@ repo=${shQuote("platform-infra/sub2api-account-sentinel")} tag=${shQuote(target.tag)} base_image=${shQuote(target.baseImage)} openai_version=${shQuote(sentinel.sdk.openaiPythonVersion)} -runtime_target=${shQuote(targetRuntime.id)} +base_image_cache_policy=${shQuote(targetRuntime.sentinelImageBuild.baseImageCachePolicy)} work=/tmp/unidesk-sub2api-sentinel-image mkdir -p "$work" dockerfile_path="$work/sentinel.Dockerfile" @@ -4023,11 +4032,11 @@ fi base64 -d > "$dockerfile_path" <<'UNIDESK_SENTINEL_DOCKERFILE_B64' ${dockerfileB64} UNIDESK_SENTINEL_DOCKERFILE_B64 -export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000 +export NO_PROXY=${shQuote(targetRuntime.sentinelImageBuild.noProxy)} export no_proxy=$NO_PROXY set -- --pull base_image_source="registry" -if [ "$runtime_target" != "G14" ] && docker image inspect "$base_image" >/dev/null 2>&1; then +if [ "$base_image_cache_policy" = "local-if-present" ] && docker image inspect "$base_image" >/dev/null 2>&1; then set -- base_image_source="local-cache" fi