From 20dda46e039c42d147bbd8fd21bde49bc11998d1 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 10 Jun 2026 16:12:34 +0000 Subject: [PATCH] fix: manage platform-infra network policy --- .agents/skills/unidesk-sub2api/SKILL.md | 7 +- docs/reference/platform-infra.md | 2 + ...ra-sub2api-network-policy-contract-test.ts | 37 ++++++ scripts/src/platform-infra.ts | 124 +++++++++++++++++- .../platform-infra/sub2api/sub2api.k8s.yaml | 19 +++ 5 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 scripts/platform-infra-sub2api-network-policy-contract-test.ts diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 38ff98e2..74a1c6bc 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -29,7 +29,7 @@ bun scripts/cli.ts platform-infra sub2api status bun scripts/cli.ts platform-infra sub2api validate ``` -- `plan` 读取 `config/platform-infra/sub2api.yaml`，渲染 `src/components/platform-infra/sub2api/sub2api.k8s.yaml`，检查 no Ingress/NodePort/LoadBalancer/hostPort/hostNetwork/resource limits。 +- `plan` 读取 `config/platform-infra/sub2api.yaml`，渲染 `src/components/platform-infra/sub2api/sub2api.k8s.yaml`，检查 no Ingress/NodePort/LoadBalancer/hostPort/hostNetwork/resource limits，并要求 `NetworkPolicy/allow-all` 随 manifest 受控创建。 - `apply --confirm` 默认创建异步 job；按返回的 `job status` 命令轮询，再跑 `status` 和 `validate`。 - `status --full|--raw` 只在需要展开远端 stdout/stderr 或原始 JSON 时使用。 - `validate` 是按需验收，不是连续可用性探针。 @@ -138,8 +138,8 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm 部署 closeout 至少包含： -- `sub2api status`：Deployment/StatefulSet/Service/Secret 可见，运行镜像与 YAML 一致。 -- `sub2api validate`：app、PostgreSQL、Redis 和 service proxy 基础检查通过。 +- `sub2api status`：Deployment/StatefulSet/Service/Secret/NetworkPolicy 可见，运行镜像与 YAML 一致，`NetworkPolicy/allow-all` 符合 `podSelector: {}`、Ingress/Egress 全放行。 +- `sub2api validate`：app、PostgreSQL、Redis、service proxy、`NetworkPolicy/allow-all` 和临时跨 Pod PostgreSQL/Redis 连通性检查通过。 - `codex-pool validate`：统一 key 的 `GET /v1/models` 成功，并用 `localCodex.responsesSmokeModel` 跑一次小的 `POST /v1/responses` smoke；owner balance / owner concurrency 已满足 YAML 最小值，capacity、WebSocket v2 和 temporary-unschedulable 运行时状态与 YAML 对齐；`validation.gatewayResponsesRecent` 汇总最近 6 小时普通 `/responses` 和 `/v1/responses` 的 failover、forward failure、最终 4xx/5xx、慢 final error 与 `context canceled` 证据，`validation.gatewayCompactRecent` 单独汇总 `/responses/compact` 证据。若当前 Responses smoke `ok=true` 但 recent 字段 `degraded=true`，先区分是历史窗口残留还是新的 request id 正在失败；长期判定见 `docs/reference/platform-infra.md`。 - 若 `publicExposure.enabled=true`，确认 FRP path 可用；`expose --confirm` 会用未带 key 的 public `/v1/models` 401 作为网关可达性探针。 @@ -148,6 +148,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm ## 排障 - profile invalid：先修 `~/.codex/config.toml.` 的 `base_url`、`wire_api`、`model` 或 `auth.json.` 的 API key；不要在 YAML 中写密钥。 +- Sub2API 卡在 `wait-postgres` / `wait-redis` 或服务内大量 `context deadline exceeded`：先跑 `sub2api status` 看 `networkPolicy.ok`，再跑 `sub2api validate` 看 `postgresCrossPodPgIsReady` / `redisCrossPodPing`；缺失或异常时用 `sub2api apply --confirm` 恢复受控 `NetworkPolicy/allow-all`，不要保留手工 iptables bypass 作为长期修复。 - pool key 401：跑 `codex-pool sync --confirm` 重建 Sub2API key 与 k3s Secret 绑定，再跑 `codex-pool validate`。 - 运行中过去的验证探针残留：只用 `codex-pool cleanup-probes --confirm` 清理 `unidesk-probe-*` 临时资源；不要把真实 managed account 删除当作探针清理或可用性恢复。 - FRP 不通：先看 `codex-pool expose --confirm` 输出的 `masterFrps`、`masterCaddy`、`sub2api-frpc` 和 public 401 probe；需要低层证据时只用 `trans G14:k3s` 做 bounded 查询。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 4214dd81..a1b0365d 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -129,3 +129,5 @@ spec: ``` This policy must be included in the `sub2api plan` / `apply` manifest rendering so that it is created as part of the normal deployment flow, not maintained as a manual one-off. + +`platform-infra sub2api status` must report whether `NetworkPolicy/allow-all` exists and still has `podSelector: {}`, `policyTypes: [Ingress, Egress]`, `ingress: [{}]`, and `egress: [{}]`. `platform-infra sub2api validate` must also run temporary in-namespace probe pods that connect to `sub2api-postgres:5432` and `sub2api-redis:6379`; local `pg_isready` inside the PostgreSQL pod alone is insufficient because it does not exercise kube-router cross-pod policy evaluation. diff --git a/scripts/platform-infra-sub2api-network-policy-contract-test.ts b/scripts/platform-infra-sub2api-network-policy-contract-test.ts new file mode 100644 index 00000000..463ed775 --- /dev/null +++ b/scripts/platform-infra-sub2api-network-policy-contract-test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from "node:fs"; +import { rootPath } from "./src/config"; + +function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { + if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); +} + +const manifestPath = rootPath("src", "components", "platform-infra", "sub2api", "sub2api.k8s.yaml"); +const platformInfraSourcePath = rootPath("scripts", "src", "platform-infra.ts"); + +const manifest = readFileSync(manifestPath, "utf8"); +const platformInfraSource = readFileSync(platformInfraSourcePath, "utf8"); +const allowAllNetworkPolicy = manifest.split(/^---\s*$/mu).find((document) => /^\s*kind:\s*NetworkPolicy\s*$/mu.test(document) && /^\s*name:\s*allow-all\s*$/mu.test(document)); + +assertCondition(allowAllNetworkPolicy !== undefined, "Sub2API manifest must include NetworkPolicy/allow-all", manifest); +assertCondition(/^\s*namespace:\s*platform-infra\s*$/mu.test(allowAllNetworkPolicy ?? ""), "allow-all NetworkPolicy must live in platform-infra", allowAllNetworkPolicy); +assertCondition(/^\s*podSelector:\s*\{\}\s*$/mu.test(allowAllNetworkPolicy ?? ""), "allow-all NetworkPolicy must select all pods", allowAllNetworkPolicy); +assertCondition(/^\s*-\s*Ingress\s*$/mu.test(allowAllNetworkPolicy ?? "") && /^\s*-\s*Egress\s*$/mu.test(allowAllNetworkPolicy ?? ""), "allow-all NetworkPolicy must cover ingress and egress", allowAllNetworkPolicy); +assertCondition(/^\s*ingress:\s*\n\s*-\s*\{\}\s*$/mu.test(allowAllNetworkPolicy ?? ""), "allow-all NetworkPolicy must allow all ingress", allowAllNetworkPolicy); +assertCondition(/^\s*egress:\s*\n\s*-\s*\{\}\s*$/mu.test(allowAllNetworkPolicy ?? ""), "allow-all NetworkPolicy must allow all egress", allowAllNetworkPolicy); +assertCondition(platformInfraSource.includes("allow-all-network-policy"), "plan policy checks must require the allow-all NetworkPolicy", platformInfraSource); +assertCondition(platformInfraSource.includes("capture_json networkpolicies"), "status must report NetworkPolicy resources", platformInfraSource); +assertCondition(platformInfraSource.includes("network_policy[\"ok\"]"), "status ok must fail when NetworkPolicy/allow-all is missing or malformed", platformInfraSource); +assertCondition(platformInfraSource.includes("postgresCrossPodPgIsReady"), "validate must include a cross-pod PostgreSQL connectivity probe", platformInfraSource); +assertCondition(platformInfraSource.includes("redisCrossPodPing"), "validate must include a cross-pod Redis connectivity probe", platformInfraSource); +assertCondition(platformInfraSource.includes("kubectl -n ${namespace} run \"$pg_probe\""), "validate must run PostgreSQL probe from a temporary pod, not from the PostgreSQL pod itself", platformInfraSource); +assertCondition(platformInfraSource.includes("kubectl -n ${namespace} run \"$redis_probe\""), "validate must run Redis probe from a temporary pod, not from the Redis pod itself", platformInfraSource); + +console.log(JSON.stringify({ + ok: true, + checks: [ + "Sub2API manifest includes controlled NetworkPolicy/allow-all", + "plan blocks manifests that drop the required NetworkPolicy", + "status reports NetworkPolicy/allow-all shape", + "validate exercises cross-pod PostgreSQL and Redis traffic through temporary probe pods", + ], +})); diff --git a/scripts/src/platform-infra.ts b/scripts/src/platform-infra.ts index b6b51873..738cca6e 100644 --- a/scripts/src/platform-infra.ts +++ b/scripts/src/platform-infra.ts @@ -239,6 +239,7 @@ function plan(): Record { resourcePolicy: "No Kubernetes CPU/memory requests or limits, matching issue #220.", imageVersionControl: "Sub2API image repository/tag/pullPolicy are controlled by config/platform-infra/sub2api.yaml in the UniDesk repository.", urlAllowlistControl: "Sub2API upstream URL validation options are controlled by config/platform-infra/sub2api.yaml and rendered to SECURITY_URL_ALLOWLIST_* env vars.", + networkPolicy: "NetworkPolicy/allow-all is rendered with the deployment so kube-router cannot silently default-deny Sub2API cross-pod traffic.", dataStores: ["PostgreSQL 18", "Redis 8"], appPoolCaps: { databaseMaxOpenConns: 10, @@ -389,9 +390,27 @@ function policyChecks(yaml: string): PolicyCheck[] { ok: new RegExp(`^\\s*name:\\s*${namespace}\\s*$`, "mu").test(yaml), detail: `Manifest declares namespace ${namespace}.`, }, + { + name: "allow-all-network-policy", + ok: hasAllowAllNetworkPolicy(yaml), + detail: `Manifest must include NetworkPolicy/allow-all in ${namespace} to keep kube-router from blocking Sub2API cross-pod traffic.`, + }, ]; } +function hasAllowAllNetworkPolicy(yaml: string): boolean { + return yaml.split(/^---\s*$/mu).some((document) => { + return /^\s*kind:\s*NetworkPolicy\s*$/mu.test(document) + && /^\s*name:\s*allow-all\s*$/mu.test(document) + && new RegExp(`^\\s*namespace:\\s*${namespace}\\s*$`, "mu").test(document) + && /^\s*podSelector:\s*\{\}\s*$/mu.test(document) + && /^\s*-\s*Ingress\s*$/mu.test(document) + && /^\s*-\s*Egress\s*$/mu.test(document) + && /^\s*ingress:\s*\n\s*-\s*\{\}\s*$/mu.test(document) + && /^\s*egress:\s*\n\s*-\s*\{\}\s*$/mu.test(document); + }); +} + function dryRunScript(yaml: string): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); return ` @@ -556,6 +575,7 @@ capture_json services kubectl -n ${namespace} get services -l app.kubernetes.io/ capture_json pvc kubectl -n ${namespace} get pvc -l app.kubernetes.io/part-of=platform-infra capture_json secrets kubectl -n ${namespace} get secret ${secretName} capture_json configmap kubectl -n ${namespace} get configmap sub2api-config +capture_json networkpolicies kubectl -n ${namespace} get networkpolicy capture_json ingresses kubectl -n ${namespace} get ingress capture_json quotas kubectl -n ${namespace} get resourcequota capture_json limitranges kubectl -n ${namespace} get limitrange @@ -707,6 +727,26 @@ def pvc_summary(item): "requestedStorage": req.get("storage"), } +def network_policy_summary(item): + spec = item.get("spec") or {} + return { + "name": item["metadata"]["name"], + "podSelector": spec.get("podSelector"), + "policyTypes": spec.get("policyTypes") or [], + "ingress": spec.get("ingress") or [], + "egress": spec.get("egress") or [], + } + +def is_allow_all_network_policy(item): + spec = item.get("spec") or {} + return ( + item.get("metadata", {}).get("name") == "allow-all" + and spec.get("podSelector") == {} + and set(spec.get("policyTypes") or []) == {"Ingress", "Egress"} + and spec.get("ingress") == [{}] + and spec.get("egress") == [{}] + ) + def resource_findings(kind, collection): findings = [] for item in collection: @@ -731,6 +771,7 @@ statefulsets = items("statefulsets") services = items("services") pods = items("pods") pvcs = items("pvc") +networkpolicies = items("networkpolicies") secret = load("secrets") configmap = load("configmap") configmap_data = (configmap or {}).get("data") or {} @@ -776,6 +817,13 @@ expected_url_allowlist_strings = { url_allowlist_configmap_aligned = url_allowlist_runtime == expected_url_allowlist_strings url_allowlist_pod_env_aligned = rc("pod-env") == 0 and url_allowlist_pod_env == expected_url_allowlist_strings url_allowlist_aligned = url_allowlist_configmap_aligned and url_allowlist_pod_env_aligned +allow_all_network_policy = next((item for item in networkpolicies if item.get("metadata", {}).get("name") == "allow-all"), None) +network_policy = { + "requiredName": "allow-all", + "exists": allow_all_network_policy is not None, + "ok": allow_all_network_policy is not None and is_allow_all_network_policy(allow_all_network_policy), + "policies": [network_policy_summary(item) for item in networkpolicies], +} boundary = { "internalOnly": len(service_violations) == 0 and len(items("ingresses")) == 0, "serviceViolations": service_violations, @@ -786,7 +834,7 @@ boundary = { } workload_ready = all(d["ready"] for d in map(deployment_summary, deployments)) and all(s["ready"] for s in map(statefulset_summary, statefulsets)) payload = { - "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and len(missing_secret_keys) == 0, + "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and network_policy["ok"] and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and len(missing_secret_keys) == 0, "namespace": "${namespace}", "namespaceExists": rc("ns") == 0, "deployments": [deployment_summary(item) for item in deployments], @@ -794,6 +842,7 @@ payload = { "pods": [pod_summary(item) for item in pods], "services": [service_summary(item) for item in services], "pvcs": [pvc_summary(item) for item in pvcs], + "networkPolicy": network_policy, "secret": { "name": "${secretName}", "exists": rc("secrets") == 0, @@ -841,11 +890,20 @@ function validateScript(): string { return ` set -u tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT +probe_suffix="$(date +%s)-$$" +pg_probe="unidesk-sub2api-netcheck-pg-$probe_suffix" +redis_probe="unidesk-sub2api-netcheck-redis-$probe_suffix" +cleanup() { + kubectl -n ${namespace} delete pod "$pg_probe" "$redis_probe" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true + rm -rf "$tmp" +} +trap cleanup EXIT kubectl get --raw /api/v1/namespaces/${namespace}/services/${serviceName}:8080/proxy/health >"$tmp/health.body" 2>"$tmp/health.err" health_rc=$? kubectl get --raw /api/v1/namespaces/${namespace}/services/${serviceName}:8080/proxy/ >"$tmp/root.body" 2>"$tmp/root.err" root_rc=$? +kubectl -n ${namespace} get networkpolicy allow-all -o json >"$tmp/network-policy.json" 2>"$tmp/network-policy.err" +network_policy_rc=$? pg_pod="$(kubectl -n ${namespace} get pod -l app.kubernetes.io/name=sub2api-postgres -o jsonpath='{.items[0].metadata.name}' 2>"$tmp/pg-pod.err")" redis_pod="$(kubectl -n ${namespace} get pod -l app.kubernetes.io/name=sub2api-redis -o jsonpath='{.items[0].metadata.name}' 2>"$tmp/redis-pod.err")" if [ -n "$pg_pod" ]; then @@ -862,13 +920,26 @@ else redis_rc=1 printf '%s\\n' 'sub2api redis pod not found' >"$tmp/redis.err" fi -python3 - "$tmp" "$health_rc" "$root_rc" "$pg_rc" "$redis_rc" <<'PY' +if ! command -v timeout >/dev/null 2>&1; then + printf '%s\\n' 'timeout command is required for bounded cross-pod probes' >"$tmp/pg-cross.err" + printf '%s\\n' 'timeout command is required for bounded cross-pod probes' >"$tmp/redis-cross.err" + : >"$tmp/pg-cross.out" + : >"$tmp/redis-cross.out" + pg_cross_rc=127 + redis_cross_rc=127 +else + timeout 35s kubectl -n ${namespace} run "$pg_probe" --restart=Never --rm -i --image=postgres:18-alpine --image-pull-policy=IfNotPresent --command -- pg_isready -h sub2api-postgres -U sub2api -d sub2api -t 5 >"$tmp/pg-cross.out" 2>"$tmp/pg-cross.err" + pg_cross_rc=$? + timeout 35s kubectl -n ${namespace} run "$redis_probe" --restart=Never --rm -i --image=redis:8-alpine --image-pull-policy=IfNotPresent --command -- redis-cli -h sub2api-redis -p 6379 ping >"$tmp/redis-cross.out" 2>"$tmp/redis-cross.err" + redis_cross_rc=$? +fi +python3 - "$tmp" "$health_rc" "$root_rc" "$pg_rc" "$redis_rc" "$network_policy_rc" "$pg_cross_rc" "$redis_cross_rc" <<'PY' import json import os import sys tmp = sys.argv[1] -health_rc, root_rc, pg_rc, redis_rc = [int(value) for value in sys.argv[2:]] +health_rc, root_rc, pg_rc, redis_rc, network_policy_rc, pg_cross_rc, redis_cross_rc = [int(value) for value in sys.argv[2:]] def text(name, limit=4000): path = os.path.join(tmp, name) @@ -878,13 +949,44 @@ def text(name, limit=4000): return "" return data[-limit:] +def json_file(name): + path = os.path.join(tmp, name) + try: + return json.load(open(path, encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + return None + +def is_allow_all_network_policy(item): + if not isinstance(item, dict): + return False + spec = item.get("spec") or {} + return ( + item.get("metadata", {}).get("name") == "allow-all" + and spec.get("podSelector") == {} + and set(spec.get("policyTypes") or []) == {"Ingress", "Egress"} + and spec.get("ingress") == [{}] + and spec.get("egress") == [{}] + ) + health_body = text("health.body", 2000) root_body = text("root.body", 2000) +network_policy_obj = json_file("network-policy.json") +network_policy_ok = network_policy_rc == 0 and is_allow_all_network_policy(network_policy_obj) payload = { - "ok": health_rc == 0 and root_rc == 0 and pg_rc == 0 and redis_rc == 0, + "ok": health_rc == 0 and root_rc == 0 and pg_rc == 0 and redis_rc == 0 and network_policy_ok and pg_cross_rc == 0 and redis_cross_rc == 0, "namespace": "${namespace}", "serviceDns": "${serviceName}.${namespace}.svc.cluster.local:8080", "checks": { + "allowAllNetworkPolicy": { + "exitCode": network_policy_rc, + "ok": network_policy_ok, + "method": "kubectl -n ${namespace} get networkpolicy allow-all -o json", + "policyTypes": ((network_policy_obj or {}).get("spec") or {}).get("policyTypes") if isinstance(network_policy_obj, dict) else None, + "podSelector": ((network_policy_obj or {}).get("spec") or {}).get("podSelector") if isinstance(network_policy_obj, dict) else None, + "ingress": ((network_policy_obj or {}).get("spec") or {}).get("ingress") if isinstance(network_policy_obj, dict) else None, + "egress": ((network_policy_obj or {}).get("spec") or {}).get("egress") if isinstance(network_policy_obj, dict) else None, + "stderr": text("network-policy.err", 2000), + }, "sub2apiHealthViaKubernetesServiceProxy": { "exitCode": health_rc, "method": "kubectl get --raw /api/v1/namespaces/${namespace}/services/${serviceName}:8080/proxy/health", @@ -903,11 +1005,23 @@ payload = { "stdout": text("pg.out", 2000), "stderr": text("pg.err", 2000), }, + "postgresCrossPodPgIsReady": { + "exitCode": pg_cross_rc, + "method": "temporary postgres:18-alpine pod pg_isready -h sub2api-postgres -U sub2api -d sub2api -t 5, bounded by outer timeout", + "stdout": text("pg-cross.out", 2000), + "stderr": text("pg-cross.err", 2000), + }, "redisPing": { "exitCode": redis_rc, "stdout": text("redis.out", 2000), "stderr": text("redis.err", 2000), }, + "redisCrossPodPing": { + "exitCode": redis_cross_rc, + "method": "temporary redis:8-alpine pod redis-cli -h sub2api-redis -p 6379 ping, bounded by outer timeout", + "stdout": text("redis-cross.out", 2000), + "stderr": text("redis-cross.err", 2000), + }, }, } print(json.dumps(payload, ensure_ascii=False, indent=2)) diff --git a/src/components/platform-infra/sub2api/sub2api.k8s.yaml b/src/components/platform-infra/sub2api/sub2api.k8s.yaml index 45c92a3d..003cb308 100644 --- a/src/components/platform-infra/sub2api/sub2api.k8s.yaml +++ b/src/components/platform-infra/sub2api/sub2api.k8s.yaml @@ -7,6 +7,25 @@ metadata: app.kubernetes.io/managed-by: unidesk unidesk.ai/runtime-node: G14 --- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all + namespace: platform-infra + labels: + app.kubernetes.io/name: platform-infra + app.kubernetes.io/part-of: platform-infra + app.kubernetes.io/managed-by: unidesk +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: + - {} + egress: + - {} +--- apiVersion: v1 kind: ConfigMap metadata: