From 24efae9f52e238eafbde0100c3b90482c7969538 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 9 Jun 2026 09:45:48 +0000 Subject: [PATCH] fix: allow sub2api http upstream URLs --- config/platform-infra/sub2api.yaml | 6 ++ ...fra-sub2api-http-upstream-contract-test.ts | 48 ++++++++++ scripts/src/check.ts | 2 + scripts/src/platform-infra.ts | 90 +++++++++++++++++-- .../platform-infra/sub2api/sub2api.k8s.yaml | 8 +- 5 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 scripts/platform-infra-sub2api-http-upstream-contract-test.ts diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index 5cab6974..89c0ed9d 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -2,3 +2,9 @@ image: repository: weishaw/sub2api tag: 0.1.135 pullPolicy: IfNotPresent +security: + urlAllowlist: + enabled: false + allowInsecureHttp: true + allowPrivateHosts: false + upstreamHosts: [] diff --git a/scripts/platform-infra-sub2api-http-upstream-contract-test.ts b/scripts/platform-infra-sub2api-http-upstream-contract-test.ts new file mode 100644 index 00000000..09c4d044 --- /dev/null +++ b/scripts/platform-infra-sub2api-http-upstream-contract-test.ts @@ -0,0 +1,48 @@ +import { readFileSync } from "node:fs"; +import { rootPath } from "./src/config"; + +function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { + if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); +} + +const sub2apiConfigPath = rootPath("config", "platform-infra", "sub2api.yaml"); +const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml"); +const manifestPath = rootPath("src", "components", "platform-infra", "sub2api", "sub2api.k8s.yaml"); + +const sub2api = Bun.YAML.parse(readFileSync(sub2apiConfigPath, "utf8")) as { + security?: { + urlAllowlist?: { + enabled?: boolean; + allowInsecureHttp?: boolean; + allowPrivateHosts?: boolean; + upstreamHosts?: string[]; + }; + }; +}; +const codexPool = Bun.YAML.parse(readFileSync(codexPoolConfigPath, "utf8")) as { + profiles?: { + entries?: Array<{ + accountName?: string; + }>; + }; +}; +const manifest = readFileSync(manifestPath, "utf8"); + +assertCondition((codexPool.profiles?.entries ?? []).length > 0, "Codex pool must have YAML-selected upstream accounts", codexPool.profiles); +assertCondition(sub2api.security?.urlAllowlist?.enabled === false, "Sub2API URL allowlist must be disabled for current HTTP upstream pool policy", sub2api.security); +assertCondition(sub2api.security?.urlAllowlist?.allowInsecureHttp === true, "Sub2API must allow http:// upstream base URLs for account tests and normal scheduling", { + security: sub2api.security, + accounts: (codexPool.profiles?.entries ?? []).map((entry) => entry.accountName), +}); +assertCondition(sub2api.security?.urlAllowlist?.allowPrivateHosts === false, "Sub2API must not allow private hosts for this public HTTP upstream exception", sub2api.security); +assertCondition(Array.isArray(sub2api.security?.urlAllowlist?.upstreamHosts), "Sub2API upstreamHosts must be YAML-controlled even when empty", sub2api.security); +assertCondition(manifest.includes('SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP: "__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP__"'), "Sub2API manifest must render allowInsecureHttp from YAML", manifest); + +console.log(JSON.stringify({ + ok: true, + checks: [ + "Sub2API runtime URL policy explicitly allows http:// upstream base URLs", + "Sub2API manifest renders URL policy from YAML instead of hardcoding the old value", + ], + accounts: (codexPool.profiles?.entries ?? []).map((entry) => entry.accountName), +})); diff --git a/scripts/src/check.ts b/scripts/src/check.ts index 87ec50b7..6dce37b6 100644 --- a/scripts/src/check.ts +++ b/scripts/src/check.ts @@ -18,6 +18,7 @@ const syntaxFiles = [ "scripts/platform-infra-sub2api-codex-local-config-contract-test.ts", "scripts/platform-infra-sub2api-codex-routing-contract-test.ts", "scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts", + "scripts/platform-infra-sub2api-http-upstream-contract-test.ts", "scripts/src/playwright-cli.ts", "scripts/src/check.ts", "scripts/src/artifact-registry.ts", @@ -455,6 +456,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default items.push(commandItem("platform-infra:sub2api-codex-local-config-contract", ["bun", "scripts/platform-infra-sub2api-codex-local-config-contract-test.ts"], 30_000)); items.push(commandItem("platform-infra:sub2api-codex-routing-contract", ["bun", "scripts/platform-infra-sub2api-codex-routing-contract-test.ts"], 30_000)); items.push(commandItem("platform-infra:sub2api-codex-temp-unsched-contract", ["bun", "scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts"], 30_000)); + items.push(commandItem("platform-infra:sub2api-http-upstream-contract", ["bun", "scripts/platform-infra-sub2api-http-upstream-contract-test.ts"], 30_000)); items.push(commandItem("auth-broker:p0-contract", ["bun", "scripts/auth-broker-contract-test.ts"], 30_000)); items.push(commandItem("d601:recovery-guardrails-contract", ["bun", "scripts/d601-recovery-guardrails-contract-test.ts"], 30_000)); items.push(commandItem("hwlab:cd-wrapper-contract", ["bun", "scripts/hwlab-cd-wrapper-contract-test.ts"], 30_000)); diff --git a/scripts/src/platform-infra.ts b/scripts/src/platform-infra.ts index aef2ea37..8192c03d 100644 --- a/scripts/src/platform-infra.ts +++ b/scripts/src/platform-infra.ts @@ -19,6 +19,14 @@ interface Sub2ApiConfig { tag: string; pullPolicy: "Always" | "IfNotPresent" | "Never"; }; + security: { + urlAllowlist: { + enabled: boolean; + allowInsecureHttp: boolean; + allowPrivateHosts: boolean; + upstreamHosts: string[]; + }; + }; } export function platformInfraHelp(): unknown { @@ -131,7 +139,23 @@ function readSub2ApiConfig(): Sub2ApiConfig { if (pullPolicy !== "Always" && pullPolicy !== "IfNotPresent" && pullPolicy !== "Never") throw new Error(`${configPath}.image.pullPolicy must be Always, IfNotPresent, or Never`); if (!/^[a-z0-9._/-]+(?::[0-9]+)?$/u.test(repository)) throw new Error(`${configPath}.image.repository has an unsupported format`); if (!/^[A-Za-z0-9._-]+$/u.test(tag)) throw new Error(`${configPath}.image.tag has an unsupported format`); - return { image: { repository, tag, pullPolicy } }; + const security = objectField(parsed as Record, "security", ""); + const urlAllowlist = objectField(security, "urlAllowlist", "security"); + const enabled = booleanField(urlAllowlist, "enabled", "security.urlAllowlist"); + const allowInsecureHttp = booleanField(urlAllowlist, "allowInsecureHttp", "security.urlAllowlist"); + const allowPrivateHosts = booleanField(urlAllowlist, "allowPrivateHosts", "security.urlAllowlist"); + const upstreamHosts = stringArrayField(urlAllowlist, "upstreamHosts", "security.urlAllowlist"); + return { + image: { repository, tag, pullPolicy }, + security: { + urlAllowlist: { + enabled, + allowInsecureHttp, + allowPrivateHosts, + upstreamHosts, + }, + }, + }; } function stringField(obj: Record, key: string, path: string): string { @@ -140,6 +164,27 @@ function stringField(obj: Record, key: string, path: string): s return value; } +function objectField(obj: Record, key: string, path: string): Record { + const value = obj[key]; + const prefix = path.length > 0 ? `${path}.` : ""; + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${configPath}.${prefix}${key} must be an object`); + return value as Record; +} + +function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${configPath}.${path}.${key} must be a boolean`); + return value; +} + +function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || !value.every((item) => typeof item === "string" && item.trim().length > 0)) { + throw new Error(`${configPath}.${path}.${key} must be an array of non-empty strings`); + } + return value.map((item) => item.trim()); +} + function imageRef(sub2api: Sub2ApiConfig): string { return `${sub2api.image.repository}:${sub2api.image.tag}`; } @@ -147,9 +192,14 @@ function imageRef(sub2api: Sub2ApiConfig): string { function manifest(): string { const sub2api = readSub2ApiConfig(); const template = readFileSync(manifestPath, "utf8"); + const urlAllowlist = sub2api.security.urlAllowlist; return template .replaceAll("__SUB2API_IMAGE__", imageRef(sub2api)) - .replaceAll("__SUB2API_IMAGE_PULL_POLICY__", sub2api.image.pullPolicy); + .replaceAll("__SUB2API_IMAGE_PULL_POLICY__", sub2api.image.pullPolicy) + .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ENABLED__", String(urlAllowlist.enabled)) + .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP__", String(urlAllowlist.allowInsecureHttp)) + .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS__", String(urlAllowlist.allowPrivateHosts)) + .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS__", urlAllowlist.upstreamHosts.join(",")); } function plan(): Record { @@ -170,6 +220,7 @@ function plan(): Record { config: { image: imageRef(sub2api), pullPolicy: sub2api.image.pullPolicy, + security: sub2api.security, }, decision: { owner: "UniDesk", @@ -178,6 +229,7 @@ function plan(): Record { exposure: "ClusterIP only; no public ingress or node-level exposure.", resourcePolicy: "No Kubernetes CPU/memory requests or limits, matching issue #220.", imageVersionControl: "Sub2API image repository/tag/pullPolicy are controlled by config/platform-infra/sub2api.yaml in the UniDesk repository.", + urlAllowlistControl: "Sub2API upstream URL validation options are controlled by config/platform-infra/sub2api.yaml and rendered to SECURITY_URL_ALLOWLIST_* env vars.", dataStores: ["PostgreSQL 18", "Redis 8"], appPoolCaps: { databaseMaxOpenConns: 10, @@ -253,7 +305,8 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise> { - const result = await capture(config, g14K3sRoute, ["script"], statusScript(imageRef(readSub2ApiConfig()))); + const sub2api = readSub2ApiConfig(); + const result = await capture(config, g14K3sRoute, ["script"], statusScript(sub2api)); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { @@ -472,7 +525,9 @@ PY `; } -function statusScript(expectedImage: string): string { +function statusScript(sub2api: Sub2ApiConfig): string { + const expectedImage = imageRef(sub2api); + const expectedUrlAllowlist = sub2api.security.urlAllowlist; return ` set -u tmp="$(mktemp -d)" @@ -491,6 +546,7 @@ capture_json pods kubectl -n ${namespace} get pods -l app.kubernetes.io/part-of= capture_json services kubectl -n ${namespace} get services -l app.kubernetes.io/part-of=platform-infra capture_json pvc kubectl -n ${namespace} get pvc -l app.kubernetes.io/part-of=platform-infra capture_json secrets kubectl -n ${namespace} get secret ${secretName} +capture_json configmap kubectl -n ${namespace} get configmap sub2api-config capture_json ingresses kubectl -n ${namespace} get ingress capture_json quotas kubectl -n ${namespace} get resourcequota capture_json limitranges kubectl -n ${namespace} get limitrange @@ -650,6 +706,8 @@ services = items("services") pods = items("pods") pvcs = items("pvc") secret = load("secrets") +configmap = load("configmap") +configmap_data = (configmap or {}).get("data") or {} secret_keys = sorted(((secret or {}).get("data") or {}).keys()) missing_secret_keys = [key for key in ${JSON.stringify(requiredSecretKeys)} if key not in secret_keys] service_violations = [] @@ -662,8 +720,21 @@ for svc in services: service_violations.append({"name": svc["metadata"]["name"], "nodePort": port.get("nodePort")}) resource_violations = resource_findings("Deployment", deployments) + resource_findings("StatefulSet", statefulsets) expected_image = "${expectedImage}" +expected_url_allowlist = json.loads(${JSON.stringify(JSON.stringify(expectedUrlAllowlist))}) sub2api_deployment = next((deployment_summary(item) for item in deployments if item["metadata"]["name"] == "${serviceName}"), None) image_aligned = sub2api_deployment is not None and expected_image in sub2api_deployment.get("images", []) +url_allowlist_runtime = { + "enabled": configmap_data.get("SECURITY_URL_ALLOWLIST_ENABLED"), + "allowInsecureHttp": configmap_data.get("SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP"), + "allowPrivateHosts": configmap_data.get("SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS"), + "upstreamHosts": configmap_data.get("SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS"), +} +url_allowlist_aligned = ( + url_allowlist_runtime["enabled"] == str(expected_url_allowlist.get("enabled")).lower() + and url_allowlist_runtime["allowInsecureHttp"] == str(expected_url_allowlist.get("allowInsecureHttp")).lower() + and url_allowlist_runtime["allowPrivateHosts"] == str(expected_url_allowlist.get("allowPrivateHosts")).lower() + and url_allowlist_runtime["upstreamHosts"] == ",".join(expected_url_allowlist.get("upstreamHosts") or []) +) boundary = { "internalOnly": len(service_violations) == 0 and len(items("ingresses")) == 0, "serviceViolations": service_violations, @@ -674,7 +745,7 @@ boundary = { } workload_ready = all(d["ready"] for d in map(deployment_summary, deployments)) and all(s["ready"] for s in map(statefulset_summary, statefulsets)) payload = { - "ok": rc("ns") == 0 and workload_ready and image_aligned and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and len(missing_secret_keys) == 0, + "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and len(missing_secret_keys) == 0, "namespace": "${namespace}", "namespaceExists": rc("ns") == 0, "deployments": [deployment_summary(item) for item in deployments], @@ -695,6 +766,15 @@ payload = { "aligned": image_aligned, "runningImages": sub2api_deployment.get("images", []) if sub2api_deployment else [], }, + "security": { + "urlAllowlist": { + "configPath": "config/platform-infra/sub2api.yaml", + "expected": expected_url_allowlist, + "runtime": url_allowlist_runtime, + "aligned": url_allowlist_aligned, + "configMapExists": rc("configmap") == 0, + } + }, "boundary": boundary, "serviceDns": "${serviceName}.${namespace}.svc.cluster.local:8080", "next": { diff --git a/src/components/platform-infra/sub2api/sub2api.k8s.yaml b/src/components/platform-infra/sub2api/sub2api.k8s.yaml index debda34c..7073a341 100644 --- a/src/components/platform-infra/sub2api/sub2api.k8s.yaml +++ b/src/components/platform-infra/sub2api/sub2api.k8s.yaml @@ -41,10 +41,10 @@ data: ADMIN_EMAIL: "admin@sub2api.platform-infra.local" JWT_EXPIRE_HOUR: "24" TZ: "Asia/Shanghai" - SECURITY_URL_ALLOWLIST_ENABLED: "false" - SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP: "false" - SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS: "false" - SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS: "" + SECURITY_URL_ALLOWLIST_ENABLED: "__SUB2API_SECURITY_URL_ALLOWLIST_ENABLED__" + SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP: "__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP__" + SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS: "__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS__" + SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS: "__SUB2API_SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS__" UPDATE_PROXY_URL: "" GATEWAY_OPENAI_RESPONSE_HEADER_TIMEOUT: "0" GATEWAY_OPENAI_HTTP2_ENABLED: "true"