From c5c1ff7f58cd0a50ace6781a04d60d2f4bc93376 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 02:47:22 +0000 Subject: [PATCH 01/13] chore: sync hwlab node control-plane state Capture D601 v03 control-plane YAML/CLI changes, update CI/CD skill notes, and document the hwlab.pikapython.com admin password reset boundary with follow-up issue #319. --- .agents/skills/unidesk-cicd/SKILL.md | 5 +- config/hwlab-node-control-plane.yaml | 41 +- config/platform-db/postgres-pk01.yaml | 74 +-- docs/reference/cli.md | 4 +- scripts/src/hwlab-node-control-plane.ts | 722 ++---------------------- scripts/src/jobs.ts | 10 +- scripts/src/platform-db.ts | 7 +- scripts/tran | 9 +- scripts/trans | 9 +- 9 files changed, 59 insertions(+), 822 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 95847a3b..0a454dc2 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -86,11 +86,14 @@ bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --confirm bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm +bun scripts/cli.ts hwlab nodes control-plane infra runtime-image status --node D601 --lane v03 +bun scripts/cli.ts hwlab nodes control-plane infra runtime-image preload --node D601 --lane v03 --confirm +bun scripts/cli.ts hwlab nodes control-plane infra runtime-image logs --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm ``` -从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 输出 artifact；输入 base image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。 +从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。 --- diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 46b4d847..4c359290 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -56,34 +56,26 @@ targets: repository: pikasTech/HWLAB branch: v0.3 gitops: - branch: v0.3-gitops + branch: v0.3-d601-gitops path: deploy/gitops/node/d601/runtime-v03 gitMirror: namespace: devops-infra serviceReadName: git-mirror-http serviceWriteName: git-mirror-write cachePvcName: hwlab-git-mirror-cache - cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d601-v03-git-mirror-cache cachePvcStorage: 20Gi servicePort: 8080 - deploymentReplicas: 1 + deploymentReplicas: 0 secretName: git-mirror-github-ssh syncConfigMapName: git-mirror-sync-script syncJobPrefix: git-mirror-hwlab-d601-v03-sync-manual flushJobPrefix: git-mirror-hwlab-d601-v03-flush-manual - readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git - writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git - storage: - localPath: - namespace: kube-system - configMapName: local-path-config - provisionerDeployment: local-path-provisioner - helperImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 - imagePullPolicy: IfNotPresent + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git tekton: - pipelineName: hwlab-v03-ci-image-publish - serviceAccountName: hwlab-v03-tekton-runner - pipelineRunPrefix: hwlab-v03-ci-poll + pipelineName: hwlab-d601-v03-ci-image-publish + serviceAccountName: hwlab-d601-v03-tekton-runner + pipelineRunPrefix: hwlab-d601-v03-ci-poll toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 sourceKind: dockerfile @@ -132,22 +124,9 @@ targets: buildMode: node-local argo: namespace: argocd - projectName: hwlab-v03 - applicationName: hwlab-node-v03 - applicationFile: application-v03.yaml - repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git - obsoleteApplications: - - hwlab-d601-v03 - resourceTracking: - manageEndpointBridge: true - repositoryCredential: - enabled: true - repoURL: git@github.com:pikasTech/HWLAB.git - secretName: hwlab-node-v03-repository - sourceSecret: - namespace: hwlab-ci - name: hwlab-git-ssh - key: ssh-privatekey + projectName: hwlab-d601 + applicationName: hwlab-d601-v03 + applicationFile: application-d601-v03.yaml install: enabled: true sourceKind: url diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index 3bedd837..8d22b3cc 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -8,7 +8,6 @@ metadata: relatedIssues: - 280 - 281 - - 1119 - 297 - 300 @@ -87,7 +86,7 @@ postgres: purpose: admin-and-secret-sync - id: D601-public cidr: 36.49.29.73/32 - purpose: platform-infra-and-hwlab-v03-app + purpose: platform-infra-standby-app - id: G14-public cidr: 202.98.17.68/32 purpose: platform-infra-runtime @@ -121,11 +120,6 @@ postgres: user: sub2api address: 10.0.8.0/22 method: scram-sha-256 - - type: hostssl - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - address: 10.0.8.0/22 - method: scram-sha-256 - type: hostssl database: sub2api user: sub2api @@ -146,11 +140,6 @@ postgres: user: sub2api address: 36.49.29.73/32 method: scram-sha-256 - - type: hostssl - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - address: 36.49.29.73/32 - method: scram-sha-256 - type: hostssl database: langbot user: langbot @@ -230,20 +219,6 @@ secrets: SUB2API_DB_NAME: sub2api randomHex: SUB2API_DB_PASSWORD: 32 - - name: hwlab-d601-v03-db-credentials - sourceRef: platform-db/hwlab-d601-v03-db.env - type: env - requiredKeys: - - HWLAB_D601_V03_DB_USER - - HWLAB_D601_V03_DB_PASSWORD - - HWLAB_D601_V03_DB_NAME - createIfMissing: - enabled: true - values: - HWLAB_D601_V03_DB_USER: hwlab_d601_v03_app - HWLAB_D601_V03_DB_NAME: hwlab_d601_v03 - randomHex: - HWLAB_D601_V03_DB_PASSWORD: 32 - name: langbot-db-credentials sourceRef: platform-db/langbot-db.env type: env @@ -284,15 +259,6 @@ objects: createdb: false createrole: false superuser: false - - name: hwlab_d601_v03_app - passwordRef: - sourceRef: platform-db/hwlab-d601-v03-db.env - key: HWLAB_D601_V03_DB_PASSWORD - login: true - attributes: - createdb: false - createrole: false - superuser: false - name: langbot passwordRef: sourceRef: platform-db/langbot-db.env @@ -317,11 +283,6 @@ objects: encoding: UTF8 locale: C.UTF-8 extensions: [] - - name: hwlab_d601_v03 - owner: hwlab_d601_v03_app - encoding: UTF8 - locale: C.UTF-8 - extensions: [] - name: langbot owner: langbot encoding: UTF8 @@ -350,36 +311,6 @@ exports: - scope: platform-infra secret: sub2api-secrets key: DATABASE_URL - - name: hwlab-d601-v03-cloud-api-database-url - sourceSecretRef: platform-db/hwlab-d601-v03-db.env - render: - envKey: DATABASE_URL - format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require&uselibpqcompat=true - variables: - PGHOST: 82.156.23.220 - writeToSecretSource: - sourceRef: hwlab/d601-v03-cloud-api-db.env - key: DATABASE_URL - mode: update-or-insert - consumers: - - scope: hwlab-v03 - secret: hwlab-cloud-api-v03-db - key: database-url - - name: hwlab-d601-v03-openfga-datastore-uri - sourceSecretRef: platform-db/hwlab-d601-v03-db.env - render: - envKey: DATASTORE_URI - format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require - variables: - PGHOST: 82.156.23.220 - writeToSecretSource: - sourceRef: hwlab/d601-v03-openfga-db.env - key: DATASTORE_URI - mode: update-or-insert - consumers: - - scope: hwlab-v03 - secret: hwlab-v03-openfga - key: datastore-uri - name: langbot-database-url sourceSecretRef: platform-db/langbot-db.env render: @@ -440,9 +371,6 @@ observability: - kind: psql-app-role database: sub2api user: sub2api - - kind: psql-app-role - database: hwlab_d601_v03 - user: hwlab_d601_v03_app - kind: psql-app-role database: langbot user: langbot diff --git a/docs/reference/cli.md b/docs/reference/cli.md index c9a2fd87..d7cf9675 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -20,12 +20,12 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider` 是 v03 Code Agent / MoonBridge provider SecretRef 的受控 bootstrap 入口；`ensure` 只从集群内既有 `hwlab-v02/hwlab-v02-code-agent-provider` 复制 `openai-api-key`、`opencode-api-key` 到 lane-local Secret，输出仅披露 source/target Secret 名、key presence、decoded byte count、mutation 和后续命令，禁止打印 base64、解码值、完整 API key 或可复用凭据。OpenFGA 和 master admin API key 继续使用同一命名空间下的 `hwlab nodes secret ... --name hwlab-v03-openfga|hwlab-v03-master-server-admin-api-key`。 +`hwlab.pikapython.com` / D601 v03 的 bootstrap admin password 是 HWLAB runtime Secret 生命周期的一部分，必须收敛到 UniDesk YAML 与受控 `hwlab nodes secret ...` CLI；明文只能存在于 Git 忽略、owner-only 的 `.state/secrets/...` 来源文件，CLI、issue、日志和 trace 只能输出 presence、byte count、fingerprint、mutation 与后续命令。当前声明式重设能力缺口由 [GitHub issue #319](https://github.com/pikasTech/unidesk/issues/319) 追踪；不要把人工生成 hash、手工写 k8s Secret 或原生 `kubectl rollout` 沉淀为长期入口。 + `hwlab nodes control-plane infra plan|status|apply --node D601 --lane v03` 是 D601 HWLAB v03 节点本地 CI/CD 与 git-mirror 前置控制面的 YAML 驱动入口，配置真相源是 `config/hwlab-node-control-plane.yaml`。`plan` 只读展示 YAML target 和将渲染的 control-plane 对象；`status` 只读观察 D601 Tekton、CI namespace、git-mirror、Argo、node-local registry 和 tools image readiness；`apply --dry-run` 只输出 manifest 摘要；`apply --confirm` 只收敛 D601 control-plane bootstrap 对象，不触发 HWLAB runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。tools image 的 node-local registry 地址只能作为输出 artifact；输入 base image 必须由 YAML 声明为公开 registry 来源，缺少 output image 时应在 `status.next.blockers` 中体现，而不是把现有 node-local image 当成输入基础镜像。 `hwlab nodes control-plane infra tools-image status|build|logs --node D601 --lane v03` 是 D601 tools image 的受控入口。Dockerfile 必须由 `config/hwlab-node-control-plane.yaml` 的 `tekton.toolsImage.dockerfileInline` 声明，输入镜像必须列在 `publicBaseImages`，构建参数和网络模式也来自 YAML；confirmed build 只在 D601 后台异步构建并推送到 node-local registry，返回 status/logs 轮询命令。`hwlab nodes control-plane infra argo status|apply|logs --node D601 --lane v03` 是 D601 Argo CD 的声明式安装入口。Argo 版本、官方 manifest URL、镜像 rewrite/preload、field manager、imagePullPolicy、CRD 列表、期望 Deployment/StatefulSet 以及生成的 AppProject/Application 都必须来自同一个 YAML；`argo apply --confirm` 只执行可重复 server-side apply 和后台轮询，不把原生 `kubectl apply`、手工 Argo CLI 或临时 manifest 作为正式安装路径。 -`hwlab nodes control-plane runtime-migration --node --lane vNN [--dry-run|--allow-live-db-read --dry-run|--confirm]` 是 node-scoped runtime lane 的受控 schema migration 入口。它只通过目标 runtime namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned `cmd/hwlab-cloud-api/migrate.ts` 执行，输出 report path、source commit 和有界 stdout/stderr 摘要；不读取或打印 Secret 值、不手写 `psql`、不把 pod 内临时命令沉淀成正式流程。D601 v03 这类由 UniDesk YAML 声明的外置 PK01 PostgreSQL 切换，DB/Secret/bridge 仍以 UniDesk YAML 和 `platform-db postgres ...`、`hwlab nodes control-plane apply|trigger-current` 为 source truth；runtime migration 只负责在已发布 runtime 上补齐应用 schema。 - ## Command Model - `help` 输出命令索引，适合作为交互式入口。 diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 0baba8ad..72b8571d 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -65,29 +65,6 @@ interface ImageRewriteSpec { target: string; } -interface LocalPathStorageSpec { - namespace: string; - configMapName: string; - provisionerDeployment: string; - helperImage: string; - imagePullPolicy: "Always" | "IfNotPresent" | "Never"; -} - -interface ArgoRepositoryCredentialSpec { - enabled: boolean; - repoURL: string; - secretName: string; - sourceSecret: { - namespace: string; - name: string; - key: string; - }; -} - -interface ArgoResourceTrackingSpec { - manageEndpointBridge: boolean; -} - interface ControlPlaneTargetSpec { id: string; node: string; @@ -102,7 +79,6 @@ interface ControlPlaneTargetSpec { serviceReadName: string; serviceWriteName: string; cachePvcName: string; - cacheHostPath: string | null; cachePvcStorage: string; servicePort: number; deploymentReplicas: number; @@ -113,9 +89,6 @@ interface ControlPlaneTargetSpec { readUrl: string; writeUrl: string; }; - storage: { - localPath: LocalPathStorageSpec; - } | null; tekton: { pipelineName: string; serviceAccountName: string; @@ -139,10 +112,6 @@ interface ControlPlaneTargetSpec { projectName: string; applicationName: string; applicationFile: string; - repoURL: string; - obsoleteApplications: readonly string[]; - resourceTracking: ArgoResourceTrackingSpec; - repositoryCredential: ArgoRepositoryCredentialSpec | null; install: { enabled: boolean; sourceKind: "url"; @@ -270,17 +239,9 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta const tekton = record(components.tekton); const ciNamespace = record(components.ciNamespace); const registry = record(components.registry); - const storage = record(components.storage); - const localPath = record(storage.localPath); - const storageReady = target.storage === null || boolField(localPath, "ready"); - const argoRepositoryCredential = record(argo.repositoryCredential); - const argoResourceTracking = record(argo.resourceTracking); - const argoRepositoryCredentialReady = !boolField(argoRepositoryCredential, "required") || boolField(argoRepositoryCredential, "ready"); - const argoResourceTrackingReady = boolField(argoResourceTracking, "ready"); const ok = result.exitCode === 0 && boolField(tekton, "installed") && boolField(ciNamespace, "exists") - && storageReady && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") @@ -292,9 +253,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") - && boolField(argoInstall, "statefulSetsReady") - && argoResourceTrackingReady - && argoRepositoryCredentialReady; + && boolField(argoInstall, "statefulSetsReady"); return { ok, command: "hwlab nodes control-plane infra status", @@ -318,9 +277,6 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta argoInstalled: boolField(argo, "installed"), argoProjectExists: boolField(argo, "projectExists"), argoApplicationExists: boolField(argo, "applicationExists"), - argoResourceTrackingReady, - argoRepositoryCredentialReady, - storageLocalPathReady: storageReady, argoCrdsReady: boolField(argoInstall, "crdsReady"), argoDeploymentsReady: boolField(argoInstall, "deploymentsReady"), argoStatefulSetsReady: boolField(argoInstall, "statefulSetsReady"), @@ -328,7 +284,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta toolsImageReady: boolField(registry, "toolsImageReady"), }, result: compactCommandResult(result), - next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace, storageReady), + next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace), }; } @@ -360,7 +316,7 @@ function infraApply(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, tar next: applyNext(node, target, imageStatus), }; } - const script = applyScript(yaml, target); + const script = applyScript(yaml); const result = runTransK3s(node.kubeRoute, script, options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { @@ -477,7 +433,6 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; const argo = record(record(status.components).argo); const argoInstall = record(argo.install); - const argoResourceTracking = record(argo.resourceTracking); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "argo", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = boolField(argo, "installed") @@ -485,8 +440,7 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") - && boolField(argoInstall, "statefulSetsReady") - && boolField(argoResourceTracking, "ready"); + && boolField(argoInstall, "statefulSetsReady"); return { ok, command: "hwlab nodes control-plane infra argo status", @@ -507,7 +461,6 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge crdsReady: boolField(argoInstall, "crdsReady"), deploymentsReady: boolField(argoInstall, "deploymentsReady"), statefulSetsReady: boolField(argoInstall, "statefulSetsReady"), - resourceTrackingReady: boolField(argoResourceTracking, "ready"), }, argo, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, @@ -532,7 +485,6 @@ function argoApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, o imageRewrites: target.argo.install.imageRewrites, preloadImages: target.argo.install.preloadImages, requiredCrds: target.argo.install.requiredCrds, - obsoleteApplications: target.argo.obsoleteApplications, desired: manifestObjectSummary(desired), desiredSha256: sha256Short(desiredYaml), stateDir: remoteJobStateDir(target, "argo"), @@ -735,28 +687,6 @@ function argoInstallSpec(raw: Record, path: string): ControlPla }; } -function argoRepositoryCredentialSpec(raw: Record | undefined, path: string): ArgoRepositoryCredentialSpec | null { - if (raw === undefined) return null; - const sourceSecret = asRecord(raw.sourceSecret, `${path}.sourceSecret`); - return { - enabled: booleanField(raw, "enabled", path), - repoURL: stringField(raw, "repoURL", path), - secretName: stringField(raw, "secretName", path), - sourceSecret: { - namespace: stringField(sourceSecret, "namespace", `${path}.sourceSecret`), - name: stringField(sourceSecret, "name", `${path}.sourceSecret`), - key: stringField(sourceSecret, "key", `${path}.sourceSecret`), - }, - }; -} - -function argoResourceTrackingSpec(raw: Record | undefined, path: string): ArgoResourceTrackingSpec { - if (raw === undefined) return { manageEndpointBridge: false }; - return { - manageEndpointBridge: booleanField(raw, "manageEndpointBridge", path), - }; -} - function imageRewriteSpec(raw: Record, path: string): ImageRewriteSpec { const rewrite = { source: stringField(raw, "source", path), @@ -827,7 +757,6 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const source = asRecord(raw.source, `${path}.source`); const gitops = asRecord(raw.gitops, `${path}.gitops`); const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`); - const storage = raw.storage === undefined ? null : storageSpec(asRecord(raw.storage, `${path}.storage`), `${path}.storage`); const tekton = asRecord(raw.tekton, `${path}.tekton`); const argo = asRecord(raw.argo, `${path}.argo`); const toolsImage = asRecord(tekton.toolsImage, `${path}.tekton.toolsImage`); @@ -851,7 +780,6 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa serviceReadName, serviceWriteName, cachePvcName: stringField(gitMirror, "cachePvcName", `${path}.gitMirror`), - cacheHostPath: optionalStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`) ?? null, cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), deploymentReplicas: nonNegativeIntegerField(gitMirror, "deploymentReplicas", `${path}.gitMirror`), @@ -862,7 +790,6 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, }, - storage, tekton: { pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), @@ -874,30 +801,11 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa projectName: stringField(argo, "projectName", `${path}.argo`), applicationName: stringField(argo, "applicationName", `${path}.argo`), applicationFile: stringField(argo, "applicationFile", `${path}.argo`), - repoURL: optionalStringField(argo, "repoURL", `${path}.argo`) ?? (optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`), - obsoleteApplications: optionalStringArrayField(argo, "obsoleteApplications", `${path}.argo`), - resourceTracking: argoResourceTrackingSpec(argo.resourceTracking === undefined ? undefined : asRecord(argo.resourceTracking, `${path}.argo.resourceTracking`), `${path}.argo.resourceTracking`), - repositoryCredential: argoRepositoryCredentialSpec(argo.repositoryCredential === undefined ? undefined : asRecord(argo.repositoryCredential, `${path}.argo.repositoryCredential`), `${path}.argo.repositoryCredential`), install: argoInstallSpec(asRecord(argo.install, `${path}.argo.install`), `${path}.argo.install`), }, }; } -function storageSpec(raw: Record, path: string): ControlPlaneTargetSpec["storage"] { - const localPath = asRecord(raw.localPath, `${path}.localPath`); - const imagePullPolicy = optionalStringField(localPath, "imagePullPolicy", `${path}.localPath`) ?? "IfNotPresent"; - if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") throw new Error(`${path}.localPath.imagePullPolicy must be Always, IfNotPresent, or Never`); - return { - localPath: { - namespace: stringField(localPath, "namespace", `${path}.localPath`), - configMapName: stringField(localPath, "configMapName", `${path}.localPath`), - provisionerDeployment: stringField(localPath, "provisionerDeployment", `${path}.localPath`), - helperImage: stringField(localPath, "helperImage", `${path}.localPath`), - imagePullPolicy, - }, - }; -} - function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record[] { const labels = { "app.kubernetes.io/part-of": "hwlab-node-control-plane", @@ -924,15 +832,27 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, data: { "repositories.json": JSON.stringify([{ repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), - "server.js": gitMirrorServerJs(), - "sync.sh": gitMirrorSyncScript(target), - "flush.sh": gitMirrorFlushScript(target), + "sync.sh": "#!/bin/sh\nset -eu\necho d601-hwlab-git-mirror-sync-placeholder\ncat /etc/git-mirror/repositories.json\n", + "flush.sh": "#!/bin/sh\nset -eu\necho d601-hwlab-git-mirror-flush-placeholder\ncat /etc/git-mirror/repositories.json\n", }, }, service(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), service(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), gitMirrorDeployment(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target, "read"), gitMirrorDeployment(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target, "write"), + { + apiVersion: "tekton.dev/v1", + kind: "Pipeline", + metadata: { name: target.tekton.pipelineName, namespace: target.ciNamespace, labels }, + spec: { + params: [ + { name: "source-commit", type: "string" }, + { name: "source-branch", type: "string", default: target.source.branch }, + { name: "gitops-branch", type: "string", default: target.gitops.branch }, + ], + tasks: [{ name: "bootstrap-placeholder", taskSpec: { steps: [{ name: "notice", image: target.tekton.toolsImage.output, script: "#!/bin/sh\nset -eu\necho d601-hwlab-v03-pipeline-placeholder\n" }] } }], + }, + }, { apiVersion: "v1", kind: "Namespace", metadata: { name: target.argo.namespace, labels } }, { apiVersion: "v1", @@ -941,97 +861,13 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa data: { "project.yaml": Bun.YAML.stringify(argoProjectSkeleton(target)), [target.argo.applicationFile]: Bun.YAML.stringify(argoApplicationSkeleton(target)), - "argocd-cm.yaml": Bun.YAML.stringify(argoConfigMap(target, labels)), - "obsolete-applications.json": JSON.stringify(target.argo.obsoleteApplications, null, 2), "note.txt": "Argo CD CRDs/controller are installed by the node control-plane bootstrap path when available; this ConfigMap preserves the desired Application until Argo is ready.\n", }, }, ); - const obsoleteApplications = argoObsoleteApplicationsConfigMap(target, labels); - if (obsoleteApplications !== null) manifests.push(obsoleteApplications); return manifests; } -function argoConfigMap(target: ControlPlaneTargetSpec, labels: Record): Record { - return { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: "argocd-cm", - namespace: target.argo.namespace, - labels: { ...labels, "app.kubernetes.io/name": "argocd-cm", "app.kubernetes.io/part-of": "argocd" }, - }, - data: { - "resource.exclusions": argoResourceExclusions(target), - }, - }; -} - -function argoResourceExclusions(target: ControlPlaneTargetSpec): string { - const endpointBlock = target.argo.resourceTracking.manageEndpointBridge - ? [] - : [ - "### Network resources created by the Kubernetes control plane and excluded to reduce the number of watched events and UI clutter", - "- apiGroups:", - " - ''", - " - discovery.k8s.io", - " kinds:", - " - Endpoints", - " - EndpointSlice", - ]; - return [ - ...endpointBlock, - "### Internal Kubernetes resources excluded to reduce watch volume", - "- apiGroups:", - " - coordination.k8s.io", - " kinds:", - " - Lease", - "### Internal Kubernetes Authz/Authn resources excluded to reduce watched events", - "- apiGroups:", - " - authentication.k8s.io", - " - authorization.k8s.io", - " kinds:", - " - SelfSubjectReview", - " - TokenReview", - " - LocalSubjectAccessReview", - " - SelfSubjectAccessReview", - " - SelfSubjectRulesReview", - " - SubjectAccessReview", - "### Intermediate Certificate Request excluded to reduce watched events", - "- apiGroups:", - " - certificates.k8s.io", - " kinds:", - " - CertificateSigningRequest", - "- apiGroups:", - " - cert-manager.io", - " kinds:", - " - CertificateRequest", - "### Cilium internal resources excluded to reduce UI clutter", - "- apiGroups:", - " - cilium.io", - " kinds:", - " - CiliumIdentity", - " - CiliumEndpoint", - " - CiliumEndpointSlice", - "### Kyverno intermediate and reporting resources excluded to reduce watched events", - "- apiGroups:", - " - kyverno.io", - " - reports.kyverno.io", - " - wgpolicyk8s.io", - " kinds:", - " - PolicyReport", - " - ClusterPolicyReport", - " - EphemeralReport", - " - ClusterEphemeralReport", - " - AdmissionReport", - " - ClusterAdmissionReport", - " - BackgroundScanReport", - " - ClusterBackgroundScanReport", - " - UpdateRequest", - "", - ].join("\n"); -} - function service(name: string, namespace: string, labels: Record, port: number): Record { return { apiVersion: "v1", @@ -1042,9 +878,6 @@ function service(name: string, namespace: string, labels: Record } function gitMirrorDeployment(name: string, namespace: string, labels: Record, target: ControlPlaneTargetSpec, mode: "read" | "write"): Record { - const serverJs = gitMirrorServerJs(); - const syncScript = gitMirrorSyncScript(target); - const flushScript = gitMirrorFlushScript(target); return { apiVersion: "apps/v1", kind: "Deployment", @@ -1053,35 +886,10 @@ function gitMirrorDeployment(name: string, namespace: string, labels: Record[] { - const labels = { - "app.kubernetes.io/part-of": "hwlab-node-control-plane", - "hwlab.pikastech.local/node": target.node, - "hwlab.pikastech.local/lane": target.lane, - }; - const manifest = [argoConfigMap(target, labels), argoProjectSkeleton(target), argoApplicationSkeleton(target)]; - const obsoleteApplications = argoObsoleteApplicationsConfigMap(target, labels); - if (obsoleteApplications !== null) manifest.push(obsoleteApplications); - return manifest; -} - -function argoObsoleteApplicationsConfigMap(target: ControlPlaneTargetSpec, labels: Record): Record | null { - if (target.argo.obsoleteApplications.length === 0) return null; - return { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: `${target.argo.applicationName}-obsolete-applications`, namespace: target.argo.namespace, labels }, - data: { - "applications.json": JSON.stringify(target.argo.obsoleteApplications, null, 2), - "note.txt": "Applications listed here are deleted by UniDesk node-local Argo apply so one YAML-defined application owns the runtime resources.\n", - }, - }; + return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; } function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record { @@ -1119,7 +906,7 @@ function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record= 0 ? crlf + 4 : (lf >= 0 ? lf + 2 : -1);", - " if (headerEnd < 0) return null;", - " const headerText = buffer.slice(0, headerEnd).toString('latin1').trim();", - " const rest = buffer.slice(headerEnd);", - " const headers = {};", - " let status = 200;", - " for (const line of headerText.split(/\\r?\\n/u)) {", - " const index = line.indexOf(':');", - " if (index < 0) continue;", - " const key = line.slice(0, index).trim();", - " const value = line.slice(index + 1).trim();", - " if (key.toLowerCase() === 'status') { const parsed = Number.parseInt(value.split(' ')[0] || '', 10); if (Number.isInteger(parsed)) status = parsed; }", - " else headers[key] = value;", - " }", - " return { status, headers, rest };", - "}", - "function handleGit(req, res) {", - " const url = new URL(req.url || '/', 'http://git-mirror.local');", - " const chunks = [];", - " req.on('data', (chunk) => chunks.push(chunk));", - " req.on('error', (error) => { res.writeHead(500, { 'content-type': 'text/plain' }); res.end(String(error && error.message || error)); });", - " req.on('end', () => {", - " let body = Buffer.concat(chunks);", - " const encoding = String(req.headers['content-encoding'] || '').toLowerCase();", - " try {", - " if (encoding === 'gzip' || encoding === 'x-gzip') body = zlib.gunzipSync(body);", - " else if (encoding === 'deflate') body = zlib.inflateSync(body);", - " else if (encoding === 'br') body = zlib.brotliDecompressSync(body);", - " else if (encoding && encoding !== 'identity') throw new Error(`unsupported content-encoding: ${encoding}`);", - " } catch (error) {", - " res.writeHead(415, { 'content-type': 'text/plain' });", - " res.end(String(error && error.message || error));", - " return;", - " }", - " const env = { ...process.env, GIT_PROJECT_ROOT: projectRoot, GIT_HTTP_EXPORT_ALL: '1', PATH_INFO: decodeURIComponent(url.pathname), REQUEST_METHOD: req.method || 'GET', QUERY_STRING: url.search.slice(1), CONTENT_TYPE: req.headers['content-type'] || '', CONTENT_LENGTH: String(body.length), REMOTE_USER: 'git' };", - " delete env.HTTP_CONTENT_ENCODING;", - " const child = spawn('git', ['http-backend'], { env });", - " let pending = Buffer.alloc(0);", - " let headersSent = false;", - " child.stderr.on('data', (chunk) => process.stderr.write(chunk));", - " child.on('error', (error) => { if (!headersSent) { res.writeHead(500, { 'content-type': 'text/plain' }); headersSent = true; } res.end(String(error && error.message || error)); });", - " child.stdout.on('data', (chunk) => {", - " if (headersSent) { res.write(chunk); return; }", - " pending = Buffer.concat([pending, chunk]);", - " const parsed = parseHeaders(pending);", - " if (!parsed) return;", - " headersSent = true;", - " res.writeHead(parsed.status, parsed.headers);", - " if (parsed.rest.length) res.write(parsed.rest);", - " });", - " child.on('close', (code) => {", - " if (!headersSent) { res.writeHead(code === 0 ? 200 : 500, { 'content-type': 'text/plain' }); headersSent = true; if (pending.length) res.write(pending); }", - " res.end();", - " });", - " child.stdin.end(body);", - " });", - "}", - "http.createServer((req, res) => { if ((req.url || '').startsWith('/healthz')) return sendHealth(res); return handleGit(req, res); }).listen(port, '0.0.0.0', () => { console.log(JSON.stringify({ event: 'git-mirror-http-started', port, projectRoot, mode: process.env.GIT_MIRROR_MODE || null })); });", - "", - ].join("\n"); -} - -function gitMirrorSshSetupScriptLines(): string[] { - return [ - "key_file=${GIT_SSH_KEY_FILE:-/etc/git-mirror/ssh-privatekey}", - "if [ -s \"$key_file\" ]; then", - " chmod 600 \"$key_file\"", - " export GIT_SSH_COMMAND=\"ssh -i $key_file -o StrictHostKeyChecking=accept-new -p 443\"", - "else", - " export GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new -p 443\"", - "fi", - ]; -} - -function gitMirrorSyncScript(target: ControlPlaneTargetSpec): string { - return [ - "#!/bin/sh", - "set -eu", - "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", - "mkdir -p /cache/pikasTech", - ...gitMirrorSshSetupScriptLines(), - `repository=${shQuote(target.source.repository)}`, - `source_branch=${shQuote(target.source.branch)}`, - `gitops_branch=${shQuote(target.gitops.branch)}`, - "repo=\"/cache/${repository}.git\"", - "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", - "mkdir -p \"$(dirname \"$repo\")\"", - "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", - " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", - "else", - " rm -rf \"$repo\"", - " git init --bare \"$repo\"", - " git --git-dir=\"$repo\" remote add origin \"$remote\"", - "fi", - "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", - "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", - "git --git-dir=\"$repo\" config http.receivepack true", - "timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", - "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", - "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", - "if timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", - " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - " if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then", - " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", - " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", - " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", - " fi", - "fi", - "git --git-dir=\"$repo\" update-server-info", - "local_source=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${source_branch}^{commit}\" 2>/dev/null || true)", - "github_source=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\" 2>/dev/null || true)", - "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", - "json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }", - "cat > /cache/HWLAB.last-sync.json </dev/null || true)", - "if [ -n \"$local_gitops\" ]; then", - " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", - " git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", - "fi", - "git --git-dir=\"$repo\" update-server-info", - "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", - "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", - "json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }", - "cat > /cache/HWLAB.last-flush.json </dev/null || printf '{}') -if [ "$local_path_enabled" = true ]; then - kubectl -n "$local_path_ns" get cm "$local_path_configmap" -o 'go-template={{ index .data "helperPod.yaml" }}' >/tmp/hwlab-local-path-helper.yaml 2>/tmp/hwlab-local-path-helper.err || true - python3 - "$local_path_ns" "$local_path_configmap" "$local_path_deployment" "$local_path_helper_image" "$local_path_pull_policy" /tmp/hwlab-local-path-helper.yaml <<'PY' >/tmp/hwlab-local-path-fragment.json -import json, pathlib, re, subprocess, sys -ns, configmap, deployment, expected_image, expected_policy, helper_path = sys.argv[1:7] -text = pathlib.Path(helper_path).read_text(errors="replace") if pathlib.Path(helper_path).exists() else "" -def run(args): - return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -def exists(args): - return run(args).returncode == 0 -def ready_deploy(): - data = run(["kubectl", "-n", ns, "get", "deployment", deployment, "-o", "json"]) - if data.returncode != 0: - return False - obj=json.loads(data.stdout) - desired=int(obj.get("spec", {}).get("replicas") or 0) - ready=int(obj.get("status", {}).get("readyReplicas") or 0) - return desired > 0 and ready == desired -image_match = re.search(r"(?m)^\\s*image:\\s*['\\\"]?([^'\\\"\\s]+)", text) -policy_match = re.search(r"(?m)^\\s*imagePullPolicy:\\s*['\\\"]?([^'\\\"\\s]+)", text) -current_image = image_match.group(1) if image_match else None -current_policy = policy_match.group(1) if policy_match else None -config_exists = exists(["kubectl", "-n", ns, "get", "configmap", configmap]) -deployment_exists = exists(["kubectl", "-n", ns, "get", "deployment", deployment]) -payload = { - "enabled": True, - "namespace": ns, - "configMap": configmap, - "provisionerDeployment": deployment, - "configMapExists": config_exists, - "deploymentExists": deployment_exists, - "deploymentReady": ready_deploy(), - "expectedHelperImage": expected_image, - "helperImage": current_image, - "expectedImagePullPolicy": expected_policy, - "imagePullPolicy": current_policy, - "ready": config_exists and deployment_exists and ready_deploy() and current_image == expected_image and current_policy == expected_policy, -} -print(json.dumps(payload)) -PY -else - printf '{"enabled":false,"ready":true}\\n' >/tmp/hwlab-local-path-fragment.json -fi -local_path_fragment=$(cat /tmp/hwlab-local-path-fragment.json 2>/dev/null || printf '{"enabled":false,"ready":true}') -if [ "$argo_repo_credential_enabled" = true ]; then - python3 - "$argo_ns" "$argo_repo_credential_secret" "$argo_repo_credential_url" "$argo_repo_credential_source_ns" "$argo_repo_credential_source_name" "$argo_repo_credential_source_key" <<'PY' >/tmp/hwlab-argo-repository-credential-fragment.json -import json, subprocess, sys -argo_ns, secret, repo_url, source_ns, source_name, source_key = sys.argv[1:7] -def run(args): - return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -repo = run(["kubectl", "-n", argo_ns, "get", "secret", secret, "-o", "json"]) -source = run(["kubectl", "-n", source_ns, "get", "secret", source_name, "-o", "json"]) -repo_payload = json.loads(repo.stdout) if repo.returncode == 0 and repo.stdout.strip() else {} -source_payload = json.loads(source.stdout) if source.returncode == 0 and source.stdout.strip() else {} -labels = repo_payload.get("metadata", {}).get("labels", {}) if repo_payload else {} -data = repo_payload.get("data", {}) if repo_payload else {} -source_data = source_payload.get("data", {}) if source_payload else {} -payload = { - "required": True, - "ready": repo.returncode == 0 and source.returncode == 0 and labels.get("argocd.argoproj.io/secret-type") == "repository" and all(key in data for key in ["type", "url", "sshPrivateKey"]) and source_key in source_data, - "secret": secret, - "repoURL": repo_url, - "sourceSecret": {"namespace": source_ns, "name": source_name, "key": source_key, "present": source.returncode == 0 and source_key in source_data}, - "valuesPrinted": False, -} -print(json.dumps(payload)) -PY -else - printf '{"required":false,"ready":true,"valuesPrinted":false}\\n' >/tmp/hwlab-argo-repository-credential-fragment.json -fi -argo_repository_credential_fragment=$(cat /tmp/hwlab-argo-repository-credential-fragment.json 2>/dev/null || printf '{"required":false,"ready":true,"valuesPrinted":false}') -python3 - "$argo_ns" "$argo_manage_endpoint_bridge" <<'PY' >/tmp/hwlab-argo-resource-tracking-fragment.json -import json, subprocess, sys -namespace, manage = sys.argv[1:3] -expect_manage = manage == "true" -result = subprocess.run(["kubectl", "-n", namespace, "get", "configmap", "argocd-cm", "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -payload = {"manageEndpointBridge": expect_manage, "configMap": "argocd-cm", "exists": result.returncode == 0} -def endpoint_resources_excluded(exclusions): - for item in exclusions.split("- apiGroups:"): - if "kinds:" not in item: - continue - kinds = [line.strip()[2:].strip() for line in item.splitlines() if line.strip().startswith("- ")] - if "Endpoints" in kinds or "EndpointSlice" in kinds: - return True - return False -if result.returncode == 0: - data = json.loads(result.stdout).get("data", {}) - exclusions = data.get("resource.exclusions", "") - payload["endpointResourcesExcluded"] = endpoint_resources_excluded(exclusions) - payload["endpointIgnoreUpdates"] = "resource.customizations.ignoreResourceUpdates.Endpoints" in data - payload["endpointSliceIgnoreUpdates"] = "resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice" in data -else: - payload["endpointResourcesExcluded"] = None - payload["endpointIgnoreUpdates"] = None - payload["endpointSliceIgnoreUpdates"] = None -payload["ready"] = payload["exists"] and (not expect_manage or (payload["endpointResourcesExcluded"] is False and payload["endpointIgnoreUpdates"] is False and payload["endpointSliceIgnoreUpdates"] is False)) -print(json.dumps(payload)) -PY -argo_resource_tracking_fragment=$(cat /tmp/hwlab-argo-resource-tracking-fragment.json 2>/dev/null || printf '{"ready":false,"parseError":true}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"storage":{"localPath":$local_path_fragment},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"resourceTracking":$argo_resource_tracking_fragment,"repositoryCredential":$argo_repository_credential_fragment,"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"tekton":{"installed":$(kubectl get crd pipelines.tekton.dev pipelineruns.tekton.dev >/dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} JSON `; } -function applyScript(yaml: string, target: ControlPlaneTargetSpec): string { +function applyScript(yaml: string): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); - const localPath = target.storage?.localPath ?? null; - const helperPod = localPath === null ? "" : localPathHelperPodYaml(localPath); - const helperEncoded = Buffer.from(helperPod, "utf8").toString("base64"); - const argoRepositoryCredential = target.argo.repositoryCredential; return ` set +e manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml) printf %s ${shQuote(encoded)} | base64 -d >"$manifest" kubectl apply --server-side --field-manager=unidesk-hwlab-node-control-plane -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err rc=$? -storage_rc=0 -repo_credential_rc=0 -if [ "$rc" -eq 0 ] && [ ${localPath === null ? "false" : "true"} = true ]; then - local_path_ns=${shQuote(localPath?.namespace ?? "")} - local_path_configmap=${shQuote(localPath?.configMapName ?? "")} - local_path_deployment=${shQuote(localPath?.provisionerDeployment ?? "")} - helper_path=$(mktemp /tmp/hwlab-local-path-helper.XXXXXX.yaml) - printf %s ${shQuote(helperEncoded)} | base64 -d >"$helper_path" - patch_path=$(mktemp /tmp/hwlab-local-path-helper.XXXXXX.json) - python3 - "$helper_path" "$patch_path" <<'PY' -import json, pathlib, sys -helper=pathlib.Path(sys.argv[1]).read_text() -pathlib.Path(sys.argv[2]).write_text(json.dumps({"data":{"helperPod.yaml": helper}}, ensure_ascii=False)) -PY - ( - kubectl -n "$local_path_ns" patch configmap "$local_path_configmap" --type merge -p "$(cat "$patch_path")" - patch_rc=$? - if [ "$patch_rc" -eq 0 ]; then - kubectl -n "$local_path_ns" rollout restart deployment "$local_path_deployment" || true - kubectl -n "$local_path_ns" get pod -o name | grep '^pod/helper-pod-create-pvc-' | xargs -r kubectl -n "$local_path_ns" delete --ignore-not-found=true - fi - exit "$patch_rc" - ) >/tmp/hwlab-node-local-path.out 2>/tmp/hwlab-node-local-path.err - storage_rc=$? - rm -f "$helper_path" "$patch_path" -fi -if [ "$rc" -eq 0 ] && [ ${argoRepositoryCredential?.enabled === true ? "true" : "false"} = true ]; then - argo_ns=${shQuote(target.argo.namespace)} - repo_secret=${shQuote(argoRepositoryCredential?.secretName ?? "")} - repo_url=${shQuote(argoRepositoryCredential?.repoURL ?? "")} - source_ns=${shQuote(argoRepositoryCredential?.sourceSecret.namespace ?? "")} - source_secret=${shQuote(argoRepositoryCredential?.sourceSecret.name ?? "")} - source_key=${shQuote(argoRepositoryCredential?.sourceSecret.key ?? "")} - repo_key_file=$(mktemp /tmp/hwlab-argo-repository-key.XXXXXX) - ( - set -eu - kubectl -n "$source_ns" get secret "$source_secret" -o "jsonpath={.data.$source_key}" | base64 -d >"$repo_key_file" - kubectl -n "$argo_ns" create secret generic "$repo_secret" --from-literal=type=git --from-literal=url="$repo_url" --from-file=sshPrivateKey="$repo_key_file" --dry-run=client -o yaml \\ - | kubectl label --local -f - argocd.argoproj.io/secret-type=repository app.kubernetes.io/part-of=hwlab-node-control-plane hwlab.pikastech.local/node=${shQuote(target.node)} hwlab.pikastech.local/lane=${shQuote(target.lane)} -o yaml \\ - | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-argo-repository -f - - ) >/tmp/hwlab-node-argo-repository.out 2>/tmp/hwlab-node-argo-repository.err - repo_credential_rc=$? - rm -f "$repo_key_file" -fi -python3 - "$rc" "$storage_rc" "$repo_credential_rc" <<'PY' +python3 - "$rc" <<'PY' import json, pathlib, sys out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else '' err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else '' -storage_out=pathlib.Path('/tmp/hwlab-node-local-path.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-local-path.out').exists() else '' -storage_err=pathlib.Path('/tmp/hwlab-node-local-path.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-local-path.err').exists() else '' -repo_out=pathlib.Path('/tmp/hwlab-node-argo-repository.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-argo-repository.out').exists() else '' -repo_err=pathlib.Path('/tmp/hwlab-node-argo-repository.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-argo-repository.err').exists() else '' -print(json.dumps({'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'storage': {'localPathApplyExitCode': int(sys.argv[2]), 'stdoutPreview': storage_out[-2000:], 'stderrPreview': storage_err[-2000:]}, 'argoRepositoryCredential': {'applyExitCode': int(sys.argv[3]), 'stdoutPreview': repo_out[-2000:], 'stderrPreview': repo_err[-2000:], 'valuesPrinted': False}, 'runtimeRolloutTriggered': False, 'pk01Touched': False}, ensure_ascii=False)) +print(json.dumps({'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}, ensure_ascii=False)) PY rm -f "$manifest" -if [ "$rc" -ne 0 ]; then exit "$rc"; fi -if [ "$storage_rc" -ne 0 ]; then exit "$storage_rc"; fi -exit "$repo_credential_rc" +exit "$rc" `; } -function localPathHelperPodYaml(spec: LocalPathStorageSpec): string { - return [ - "apiVersion: v1", - "kind: Pod", - "metadata:", - " name: helper-pod", - "spec:", - " containers:", - " - name: helper-pod", - ` image: "${spec.helperImage}"`, - ` imagePullPolicy: ${spec.imagePullPolicy}`, - "", - ].join("\n"); -} - function toolsImageStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, timeoutSeconds: number): { registryReady: boolean; toolsImageReady: boolean; @@ -1719,7 +1134,6 @@ function statusNext( gitMirror: Record, argo: Record, ciNamespace: Record, - storageReady: boolean, ): Record { const bootstrapMissing = !boolField(ciNamespace, "exists") || !boolField(gitMirror, "namespaceExists") @@ -1729,7 +1143,6 @@ function statusNext( const blockers: string[] = []; if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); - if (!storageReady) blockers.push("local-path-storage-not-ready"); if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); const argoInstall = record(argo.install); if (!boolField(argo, "installed")) blockers.push("argocd-not-installed"); @@ -1749,9 +1162,6 @@ function statusNext( if (!boolField(registry, "toolsImageReady")) { next.buildToolsImage = "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。"; } - if (!storageReady) { - next.fixLocalPathStorage = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; - } if (!boolField(argo, "installed")) { next.installArgo = "准备受控 D601 Argo CD 安装入口后再进入 runtime rollout。"; } @@ -1857,7 +1267,6 @@ function argoApplyStartScript(node: ControlPlaneNodeSpec, target: ControlPlaneTa const desiredEncoded = Buffer.from(desiredYaml, "utf8").toString("base64"); const rewritesEncoded = Buffer.from(JSON.stringify(target.argo.install.imageRewrites), "utf8").toString("base64"); const preloadEncoded = Buffer.from(JSON.stringify(target.argo.install.preloadImages), "utf8").toString("base64"); - const obsoleteApplications = shellSingleQuotedLines(target.argo.obsoleteApplications); return ` set -eu state_dir=${shQuote(stateDir)} @@ -1874,19 +1283,11 @@ namespace=${shQuote(target.argo.namespace)} manifest_url=${shQuote(target.argo.install.manifestUrl)} field_manager=${shQuote(target.argo.install.fieldManager)} readiness_timeout=${shQuote(String(target.argo.install.readinessTimeoutSeconds))} -repo_credential_enabled=${target.argo.repositoryCredential?.enabled === true ? "true" : "false"} -repo_secret=${shQuote(target.argo.repositoryCredential?.secretName ?? "")} -repo_url=${shQuote(target.argo.repositoryCredential?.repoURL ?? "")} -source_ns=${shQuote(target.argo.repositoryCredential?.sourceSecret.namespace ?? "")} -source_secret=${shQuote(target.argo.repositoryCredential?.sourceSecret.name ?? "")} -source_key=${shQuote(target.argo.repositoryCredential?.sourceSecret.key ?? "")} -active_app=${shQuote(target.argo.applicationName)} log="$state_dir/job.log" status="$state_dir/status.json" install_yaml="$state_dir/install.yaml" rendered_yaml="$state_dir/install.rendered.yaml" desired_yaml="$state_dir/desired.yaml" -obsolete_apps_file="$state_dir/obsolete-applications.txt" write_status() { state="$1"; shift message="$1"; shift || true @@ -1902,23 +1303,11 @@ ${proxyExportBlock(node)} printf %s ${shQuote(desiredEncoded)} | base64 -d >"$desired_yaml" printf %s ${shQuote(rewritesEncoded)} | base64 -d >"$state_dir/image-rewrites.json" printf %s ${shQuote(preloadEncoded)} | base64 -d >"$state_dir/preload-images.json" - cat >"$obsolete_apps_file" <<'OBSOLETE_APPS' -${obsoleteApplications} -OBSOLETE_APPS kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply --server-side --field-manager="$field_manager" -f - || exit "$?" python3 - "$state_dir/preload-images.json" "$state_dir/image-rewrites.json" <<'PY' >"$state_dir/pull-images.sh" import json, pathlib, shlex, sys preload=json.loads(pathlib.Path(sys.argv[1]).read_text()) rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) -def registry_probe(image): - prefix="127.0.0.1:5000/" - if not image.startswith(prefix): - return None - repo_tag=image[len(prefix):] - if ":" not in repo_tag: - return None - repo, tag=repo_tag.rsplit(":", 1) - return "http://127.0.0.1:5000/v2/" + repo + "/manifests/" + tag print("#!/bin/sh") print("set -eu") seen=set() @@ -1928,16 +1317,6 @@ for item in rewrites: if target in seen: continue seen.add(target) - probe=registry_probe(target) - if probe: - print("if curl -fsS --max-time 5 " + shlex.quote(probe) + " >/dev/null 2>&1; then") - print(" echo " + shlex.quote("image already present: " + target)) - print("else") - print(" docker pull " + shlex.quote(pull)) - print(" docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) - print(" docker push " + shlex.quote(target)) - print("fi") - continue print("docker pull " + shlex.quote(pull)) print("docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) print("docker push " + shlex.quote(target)) @@ -1965,30 +1344,7 @@ PY sleep 5 done kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null || exit "$?" - if [ "$repo_credential_enabled" = true ]; then - repo_key_file=$(mktemp /tmp/hwlab-argo-repository-key.XXXXXX) - kubectl -n "$source_ns" get secret "$source_secret" -o "jsonpath={.data.$source_key}" | base64 -d >"$repo_key_file" || exit "$?" - kubectl -n "$namespace" create secret generic "$repo_secret" --from-literal=type=git --from-literal=url="$repo_url" --from-file=sshPrivateKey="$repo_key_file" --dry-run=client -o yaml \\ - | kubectl label --local -f - argocd.argoproj.io/secret-type=repository app.kubernetes.io/part-of=hwlab-node-control-plane hwlab.pikastech.local/node=${shQuote(target.node)} hwlab.pikastech.local/lane=${shQuote(target.lane)} -o yaml \\ - | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-argo-repository -f - || exit "$?" - rm -f "$repo_key_file" - fi - kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" - while IFS= read -r obsolete_app; do - [ -n "$obsolete_app" ] || continue - if [ "$obsolete_app" = "$active_app" ]; then - echo "refusing to delete active Argo application listed as obsolete: $obsolete_app" >&2 - exit 42 - fi - kubectl -n "$namespace" delete application "$obsolete_app" --ignore-not-found=true || exit "$?" - done <"$obsolete_apps_file" - kubectl -n "$namespace" rollout restart deployment argocd-repo-server || exit "$?" - kubectl -n "$namespace" rollout status deployment argocd-repo-server --timeout=180s || exit "$?" - kubectl -n "$namespace" rollout restart statefulset argocd-application-controller || exit "$?" - if ! kubectl -n "$namespace" rollout status statefulset argocd-application-controller --timeout=120s; then - kubectl -n "$namespace" delete pod -l app.kubernetes.io/name=argocd-application-controller --ignore-not-found=true --wait=false || exit "$?" - kubectl -n "$namespace" rollout status statefulset argocd-application-controller --timeout=180s || exit "$?" - fi + kubectl apply --server-side --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" write_status succeeded argocd-install-applied } >>"$log" 2>&1 || { rc=$? @@ -2065,7 +1421,7 @@ function manifestObjectSummary(manifest: readonly Record[]): Re } function runTransK3s(kubeRoute: string, script: string, timeoutSeconds: number): CommandResult { - return runCommand([rootPath("scripts", "trans"), kubeRoute, "script", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); + return runCommand(["/root/.local/bin/trans", kubeRoute, "script", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); } function proxyExportBlock(node: ControlPlaneNodeSpec): string { @@ -2094,15 +1450,6 @@ function shellJsonArray(items: readonly string[]): string { return JSON.stringify([...items]); } -function shellSingleQuotedLines(items: readonly string[]): string { - for (const item of items) validateKubernetesName(item, "argo.obsoleteApplications"); - return items.join("\n"); -} - -function validateKubernetesName(value: string, path: string): void { - if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} contains invalid Kubernetes metadata.name: ${value}`); -} - function parseRemoteJson(text: string): unknown { const trimmed = text.trim(); if (trimmed.length === 0) return null; @@ -2168,11 +1515,6 @@ function stringArrayField(obj: Record, key: string, path: strin return [...value] as string[]; } -function optionalStringArrayField(obj: Record, key: string, path: string): string[] { - if (obj[key] === undefined) return []; - return stringArrayField(obj, key, path); -} - function stringRecordField(obj: Record, path: string): Record { const result: Record = {}; for (const [key, value] of Object.entries(obj)) { diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index 464a35af..d916cf93 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -418,8 +418,6 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri const stageStatus = stringField(lastEvent.status); const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); - const node = stringField(lastEvent.node) ?? commandOption(job.command, "--node") ?? "G14"; - const lane = stringField(lastEvent.lane) ?? commandOption(job.command, "--lane") ?? "v03"; const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail) ? true : stage === "create-pipelinerun" && stageStatus === "failed" @@ -467,19 +465,13 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri slow ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand: pipelineRun - ? `bun scripts/cli.ts hwlab nodes control-plane status --node ${node} --lane ${lane} --pipeline-run ${pipelineRun}` + ? `bun scripts/cli.ts hwlab nodes control-plane status --node ${stringField(lastEvent.node) ?? "G14"} --lane ${stringField(lastEvent.lane) ?? "v03"} --pipeline-run ${pipelineRun}` : job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; } -function commandOption(command: readonly string[], name: string): string | null { - const index = command.indexOf(name); - const value = index >= 0 ? command[index + 1] : undefined; - return typeof value === "string" && value.trim().length > 0 && !value.startsWith("--") ? value.trim() : null; -} - function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary { const nowMs = Date.now(); const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000); diff --git a/scripts/src/platform-db.ts b/scripts/src/platform-db.ts index dba3bbba..96b0249d 100644 --- a/scripts/src/platform-db.ts +++ b/scripts/src/platform-db.ts @@ -878,7 +878,7 @@ function connectionStringExport(item: Record, path: string): Co consumers: arrayOfRecords(item.consumers, `${path}.consumers`).map((consumer, index) => ({ scope: stringField(consumer, "scope", `${path}.consumers[${index}]`), secret: stringField(consumer, "secret", `${path}.consumers[${index}]`), - key: kubernetesSecretKey(stringField(consumer, "key", `${path}.consumers[${index}]`), `${path}.consumers[${index}].key`), + key: envKey(stringField(consumer, "key", `${path}.consumers[${index}]`), `${path}.consumers[${index}].key`), })), }; } @@ -909,11 +909,6 @@ function envKey(value: string, path: string): string { return value; } -function kubernetesSecretKey(value: string, path: string): string { - if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`); - return value; -} - function configSummary(pg: PostgresHostConfig): Record { return { path: pg.configPath, diff --git a/scripts/tran b/scripts/tran index 992cef87..67c18135 100755 --- a/scripts/tran +++ b/scripts/tran @@ -1,11 +1,10 @@ #!/bin/sh set -eu -self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -self_repo=$(CDPATH= cd -- "$self_dir/.." && pwd) -repo=${UNIDESK_TRAN_REPO_ROOT:-$self_repo} -if [ ! -f "$repo/scripts/cli.ts" ] && [ -f /root/unidesk/scripts/cli.ts ]; then - repo=/root/unidesk +repo=${UNIDESK_TRAN_REPO_ROOT:-/root/unidesk} +if [ ! -f "$repo/scripts/cli.ts" ]; then + self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + repo=$(CDPATH= cd -- "$self_dir/.." && pwd) fi tran_timeout_seconds() { diff --git a/scripts/trans b/scripts/trans index 7c4a1f23..e0e68a83 100755 --- a/scripts/trans +++ b/scripts/trans @@ -1,11 +1,10 @@ #!/bin/sh set -eu -self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -self_repo=$(CDPATH= cd -- "$self_dir/.." && pwd) -repo=${UNIDESK_TRANS_REPO_ROOT:-$self_repo} -if [ ! -f "$repo/scripts/cli.ts" ] && [ -f /root/unidesk/scripts/cli.ts ]; then - repo=/root/unidesk +repo=${UNIDESK_TRANS_REPO_ROOT:-/root/unidesk} +if [ ! -f "$repo/scripts/cli.ts" ]; then + self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + repo=$(CDPATH= cd -- "$self_dir/.." && pwd) fi exec bun "$repo/scripts/cli.ts" ssh "$@" From c29ca99a50c300e4c7229d1687fc9a9b5e58a354 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 03:59:07 +0000 Subject: [PATCH 02/13] docs: add yaml-first distributed ops spec --- AGENTS.md | 2 + docs/reference/yaml-first-ops.md | 128 +++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 docs/reference/yaml-first-ops.md diff --git a/AGENTS.md b/AGENTS.md index fa75281f..4ebda137 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - P0: UniDesk 自有配置一律优先使用 YAML（`.yaml`/`.yml`），包括 `config/` 下的运行面、平台基础设施、节点/lane、部署参数和可调版本配置；除非外部工具硬性要求 JSON/TOML/ENV 等格式，禁止新增 JSON 作为 UniDesk 自有配置真相。 - P0: 需要代码读取的 YAML 配置必须显式校验格式、字段类型和必填项；配置校验只保证“能被正确读取和渲染”，不得把业务策略、调度策略或数值选择写成代码硬编码、schema 硬范围、合同测试或隐藏默认值。后续版本、镜像、namespace、endpoint、容量、冷却时间、退避窗口等可调项必须从 YAML 配置进入受控 CLI，具体数值以 YAML 为准。 +- P0: YAML-first 异构分布式运维架构、现有 YAML 归属优先、禁止硬编码 node/service、公共 ops 层抽取和薄 domain CLI 规则见 `docs/reference/yaml-first-ops.md`。 ## P0 最高优先级：G14 platform-infra 规则 @@ -278,6 +279,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `docs/reference/hwlab.md`：HWLAB 指挥侧固定 workspace、G14 主运行面、D601 legacy/硬件桥接边界、最小 device-agent/gateway 桥接模型和受控发布边界。 - `docs/reference/g14.md`：G14 provider 节点、k3s 控制桥、legacy DEV/PROD 退役边界、当前 HWLAB runtime lane、device-agent 手动实验边界、Code Queue/CI 候选目标和节点本地 VPN proxy bootstrap 边界。 - `docs/reference/pk01.md`：PK01 腾讯云 provider-gateway、pikanode/MET Docker workload、SSH 透传、磁盘 GC 和 pikanode temp 长效 retention 边界。 +- `docs/reference/yaml-first-ops.md`：YAML-first 异构分布式运维架构、现有 YAML 归属优先、公共 ops 层抽取、禁止硬编码 node/service 和薄 domain CLI 规则。 - `docs/reference/platform-infra.md`：G14 `platform-infra` namespace、YAML-first shared service 配置、Sub2API/Codex pool、FRP 暴露和 on-demand availability probe 开发边界；Sub2API 日常操作统一见 `$unidesk-sub2api`（`.agents/skills/unidesk-sub2api/SKILL.md`）。 - `docs/reference/master-server-ops.md`：主 server 本机 Codex profile wrapper、ACX/GOCX/Moon Bridge 路由边界、默认模型、真实调用验收和 MiniMax session recovery 规则。 - `docs/reference/g14-observability-infra.md`：G14 原生 k3s 上 Prometheus Operator、`devops-infra` 监控基础设施、跨 namespace scrape 声明和安全边界。 diff --git a/docs/reference/yaml-first-ops.md b/docs/reference/yaml-first-ops.md new file mode 100644 index 00000000..b5cf3d42 --- /dev/null +++ b/docs/reference/yaml-first-ops.md @@ -0,0 +1,128 @@ +# YAML-First Heterogeneous Distributed Ops + +This document defines the UniDesk architecture for YAML-first heterogeneous distributed operations. It is the long-term reference for turning node, lane, service, Secret, exposure, database, rollout and probe decisions into declared configuration plus reusable CLI execution. Concrete values belong in YAML under `config/`; this document defines ownership and architecture only. + +## Scope + +YAML-first ops applies to UniDesk-owned distributed runtime management across heterogeneous targets: host services, k3s namespaces, public exposure bridges, external databases, app runtime Secrets, CI/CD control-plane bootstrap, workflow services and managed service probes. + +It is not a new global orchestrator. Existing domain ownership stays intact: + +- Platform shared services keep their truth in the existing platform infra YAML family. +- Platform database state keeps its truth in platform database YAML. +- Runtime lane services keep their truth in their existing node/lane YAML. +- Agent execution infrastructure keeps its truth in its own infrastructure YAML. + +Add a new top-level YAML registry only after multiple existing domains share the same lifecycle, owner and command model, and after the common blocks have already proven reusable. The default path is to extend the owning domain YAML and shared ops helpers, not to create another parallel control plane. + +## Source Of Truth + +UniDesk-owned distributed ops choices must enter through YAML: + +- target route and execution plane +- namespace, workload, service, Secret and ConfigMap identifiers +- image references, versions and pull policy +- public URL, DNS expectation, FRP/Caddy edge settings and probe endpoints +- database host, role/database declarations, Secret exports and connection mode +- Secret source references, key mappings, transforms and rollout triggers +- readiness, validation and smoke probe shape +- retention, cadence, timeout and policy values when they are UniDesk-owned choices + +Code may validate that YAML is present, typed, syntactically valid and renderable. Code must not become the hidden source for node names, service names, namespaces, ports, image tags, Secret names, URLs, account lists, capacities, cooldowns or retry windows. These values must be read from YAML or from explicit external tool/runtime APIs. + +External formats such as JSON, TOML, env files, Kubernetes YAML, Caddyfile, systemd units or app-specific config files may still be generated or consumed at the edge when the external tool requires them. They are inputs or rendered artifacts, not UniDesk desired-state truth. + +## Architecture Layers + +YAML-first ops uses five layers. + +1. Domain YAML + +The owning `config/**/*.yaml` file declares the desired runtime state and all tunable values. A domain YAML may contain reusable blocks such as `publicExposure`, `externalDatabase`, `runtimeSecrets`, `rollout`, `probes`, `staging`, `retention` or `controlPlane`, but the exact block is owned by the domain until it is promoted into a shared helper. + +2. Domain Parser + +Each domain has a parser that resolves a selected target and validates only shape, field type, required fields and renderability. It may validate generic syntax such as Kubernetes resource names, route token format, URL shape, image reference shape, relative source references and key names. It must not hard-code current policy values or silently fill business defaults that should live in YAML. + +3. Common Ops Library + +Shared behavior belongs in reusable modules under `scripts/src/`, not in service-specific command files. The existing reusable seeds are the platform infra public-service helpers and the platform infra ops library. New common helpers should be extracted when the same operation appears in more than one domain, especially for: + +- route execution and bounded capture +- YAML parsing primitives +- redacted output, fingerprints and compact evidence +- Secret source loading and source path redaction +- Kubernetes Secret apply from local source material +- rollout restart/status from YAML-declared workload refs +- public exposure rendering through FRP/Caddy +- manifest staging, dry-run and server-side apply wrappers +- probe execution and response summarization +- async job submission and short polling for long operations + +4. Thin Domain CLI + +The domain CLI resolves the target from YAML, calls shared helpers and prints structured JSON. It should not contain large inline shell bodies, duplicated secret-sync scripts, hard-coded service names or app-specific operational workflows. A domain CLI may keep a stable command namespace for compatibility and discoverability, but the implementation should delegate to common helpers. + +5. Runtime Executor + +Runtime mutation goes through UniDesk CLI and `trans` route execution. Direct `kubectl`, raw SSH, hand-written Caddy edits, direct GitHub API calls or ad hoc shell scripts may be diagnostic or emergency recovery tools only. Repeated operational writes must be promoted into a controlled CLI command that reads YAML and reports redacted structured output. + +## Common Block Rules + +Reusable blocks must describe operations in data, not in service-specific code branches. + +### Target Blocks + +A target block should declare the route, execution plane, namespace and any workload refs required by the operation. Code must not infer these from a node id, lane id or service id by concatenating strings unless that concatenation rule itself is explicitly declared and stable for the domain. + +### Secret Blocks + +A runtime Secret block should declare source reference, source key, target Secret, target key, optional transform and rollout trigger. Secret values must stay in git-ignored owner-only source files or external Secret stores. CLI output may show sourceRef, target object names, key names, presence, byte counts, fingerprints, mutation and next commands; it must not print secret values, full tokens, decoded base64, passwords or complete connection strings. + +App-specific transforms are allowed only as isolated named transform functions. The transform name is data in YAML; the implementation belongs in a shared transform registry or a small domain adapter, not in a one-off reset command. + +### Exposure Blocks + +Public exposure must be declared as an edge topology, including DNS expectation, public base URL, bridge settings, edge host route and target service. The existing FRP/Caddy path is a reusable public-service primitive. New public exposure code should extend that primitive instead of adding per-service Caddy or FRP scripts. + +### Database Blocks + +External database consumers must reference the YAML-owned platform database source and exported Secret shape. A consumer should not deploy a new database, copy connection strings by hand, or derive credentials from live runtime objects unless the owning database YAML declares that export. + +### Probe Blocks + +Probes are validation data, not hidden policy. YAML should declare what endpoint or runtime object proves the operation for that service. CLI code may execute the probe, bound output and classify failure, but should not hard-code current URLs, credentials, namespaces or service paths. + +## Refactoring Rule + +When adding YAML-first ops to an existing domain, follow this order: + +1. Inventory the existing YAML, CLI commands and helper modules. +2. Choose the owning domain YAML; do not start with a new global registry. +3. Add or refine a reusable block in that YAML with all concrete values declared there. +4. Extend the domain parser with shape/type/renderability validation only. +5. Extract common execution into shared helper modules before adding domain-specific code. +6. Keep the domain CLI as a thin adapter over the common helper. +7. Validate with the narrowest syntax check and command-shape or original-entry runtime check required by the change. + +Large domain command files must be split by responsibility before receiving more operational logic. Typical split boundaries are target resolution, manifest rendering, Secret sync, public exposure, database bridge, rollout, probes, cleanup and status summarization. + +## Anti-Patterns + +Avoid these patterns: + +- creating a per-service reset script when a YAML-declared Secret sync plus rollout block is enough +- adding a second control plane for a service that already has an owning YAML and CLI namespace +- hard-coding node ids, service ids, namespaces, ports, URLs, Secret names or workload names in code +- deriving live state by string conventions when YAML can declare the object directly +- keeping repeated `kubectl apply`, Caddy edits, FRP edits or rollout restarts as runbook shell snippets +- printing secret values, complete env files, full `DATABASE_URL` values or reusable API keys +- writing long-term docs that duplicate current YAML values as prose +- using contract tests or hidden guards to freeze policy values that should remain YAML-controlled +- preserving legacy command branches after the latest YAML-first path supersedes them + +## Documentation Boundary + +Long-term references should point to this architecture for common YAML-first ops rules, then document only domain-specific ownership and entrypoints. They should not repeat common Secret, exposure, target, redaction or no-hardcoding rules unless a domain adds a stricter constraint. + +When a recurring operation becomes stable, update the owning reference document and the relevant skill with the domain entrypoint and decision boundary. Do not document one-off manual recovery as the standard path; manual repair remains recovery evidence until the YAML and CLI path exists. From 5a7afb52367c5c4f88e4cf86619ac7591222cf0d Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 04:01:31 +0000 Subject: [PATCH 03/13] chore: make sub2api d601 primary --- .agents/skills/unidesk-sub2api/SKILL.md | 20 +- config/platform-infra/sub2api-codex-pool.yaml | 2 +- config/platform-infra/sub2api.yaml | 14 +- docs/reference/platform-infra.md | 24 +- scripts/src/platform-infra-sub2api-codex.ts | 66 +++-- scripts/src/platform-infra.ts | 276 ++++++++++++++++-- 6 files changed, 331 insertions(+), 71 deletions(-) diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index bbafe531..60b91678 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -5,7 +5,7 @@ description: UniDesk Sub2API 平台运维技能。用户提到 Sub2API、sub2api # UniDesk Sub2API -UniDesk 在 k3s `platform-infra` namespace 运维 Sub2API。G14 是默认 active runtime；D601 由同一 YAML/CLI 控制，可保持 standby predeploy，也可在外置 DB、镜像、FRP 和 egress proxy 条件就绪后作为 external-active target 运行。日常操作统一使用 UniDesk CLI，不直接写 Kubernetes 资源或手工调用 Sub2API 管理 API。 +UniDesk 在 k3s `platform-infra` namespace 运维 Sub2API。D601 是当前默认 active runtime，使用外置 PostgreSQL 和目标级 public exposure；G14 由同一 YAML/CLI 控制为 standby predeploy，默认缩容且不运行 sentinel、FRP 或 HTTPS egress proxy。日常操作统一使用 UniDesk CLI，不直接写 Kubernetes 资源或手工调用 Sub2API 管理 API。 **固定入口**: `cd /root/unidesk && bun scripts/cli.ts platform-infra sub2api ...` @@ -33,31 +33,31 @@ bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ` / `auth.json.` 并在 YAML 里声明。 +- 默认 `~/.codex/config.toml` 和 `~/.codex/auth.json` 只作为统一 Sub2API consumer 使用；`config.toml` 必须指向 YAML-selected active target 的 consumer URL，当前由 `codex-pool configure-local --confirm` 写入 D601 target-level public URL；`auth.json` 必须使用统一 pool API key。新增上游账号不得覆盖这两个默认文件，只能新增 `config.toml.` / `auth.json.` 并在 YAML 里声明。 - 输出只能包含 Secret 路径、长度、preview/fingerprint；禁止打印完整 API key、admin password、JWT secret、TOTP key。 ## 部署与状态 ```bash bun scripts/cli.ts platform-infra sub2api plan -bun scripts/cli.ts platform-infra sub2api plan --target D601 +bun scripts/cli.ts platform-infra sub2api plan --target G14 bun scripts/cli.ts platform-infra sub2api apply --dry-run -bun scripts/cli.ts platform-infra sub2api apply --target D601 --dry-run +bun scripts/cli.ts platform-infra sub2api apply --target G14 --dry-run bun scripts/cli.ts platform-infra sub2api apply --confirm -bun scripts/cli.ts platform-infra sub2api apply --target D601 --confirm +bun scripts/cli.ts platform-infra sub2api apply --target G14 --confirm bun scripts/cli.ts platform-infra sub2api status -bun scripts/cli.ts platform-infra sub2api status --target D601 +bun scripts/cli.ts platform-infra sub2api status --target G14 bun scripts/cli.ts platform-infra sub2api validate -bun scripts/cli.ts platform-infra sub2api validate --target D601 +bun scripts/cli.ts platform-infra sub2api validate --target G14 ``` - `plan` 读取 `config/platform-infra/sub2api.yaml`，渲染 `src/components/platform-infra/sub2api/sub2api.k8s.yaml`，检查 no Ingress/NodePort/LoadBalancer/hostPort/hostNetwork/resource limits，并要求 `NetworkPolicy/allow-all` 随 manifest 受控创建。 - `apply --confirm` 默认创建异步 job；按返回的 `job status` 命令轮询，再跑 `status` 和 `validate`。 - `status --full|--raw` 只在需要展开远端 stdout/stderr 或原始 JSON 时使用。 -- `validate` 是按需验收，不是连续可用性探针。对 D601 standby，`validate --target D601` 验证预部署形态，不要求外置 DB 当前可连接；对 D601 external-active，必须验证外置 DB、ephemeral Redis、Sub2API service、YAML egress proxy 和目标级 public exposure。 +- `validate` 是按需验收，不是连续可用性探针。对 standby target，`validate --target ` 验证预部署形态，不要求外置 DB 当前可连接；对 external-active target，必须验证外置 DB、ephemeral Redis、Sub2API service、YAML egress proxy 和目标级 public exposure。 ## 镜像升级 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 279369d9..1af9019d 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -119,7 +119,7 @@ profiles: configFile: config.toml.socap authFile: auth.json.socap publicExposure: - enabled: true + enabled: false proxyName: platform-infra-sub2api configMapName: sub2api-frpc-config deploymentName: sub2api-frpc diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index 9742976d..1ac8c328 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -6,16 +6,16 @@ targets: - id: G14 route: G14:k3s namespace: platform-infra - role: active + role: standby enabled: true - databaseMode: bundled - redisMode: bundled-persistent - appReplicas: 1 - redisReplicas: 1 + databaseMode: external-pending + redisMode: local-ephemeral + appReplicas: 0 + redisReplicas: 0 - id: D601 route: D601:k3s namespace: platform-infra - role: active-standby + role: active enabled: true databaseMode: external-active redisMode: local-ephemeral @@ -118,7 +118,7 @@ runtime: sentinel: mode: singleton enabledOnTargets: - - G14 + - D601 security: urlAllowlist: enabled: false diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index cfb336ef..f23260a3 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -1,6 +1,6 @@ # Platform Infra -`platform-infra` is the k3s namespace for UniDesk-operated shared platform services. G14 is the default active runtime for this namespace; D601 may host explicitly declared standby or externally backed active targets when the service needs node-local preparation, cutover capacity, or a direct public edge. It is separate from HWLAB runtime lanes, AgentRun lanes, D601 user services, and legacy `devops-infra` control-plane helpers. New shared infra should land here first; old `devops-infra` resources migrate gradually only when a concrete owner and validation path exists. +`platform-infra` is the k3s namespace for UniDesk-operated shared platform services. Runtime placement is service-specific and YAML-selected. For Sub2API, D601 is the active externally backed target and G14 is a predeployed standby target scaled to zero; other platform services may still declare G14 as their active runtime in their own YAML. It is separate from HWLAB runtime lanes, AgentRun lanes, D601 user services, and legacy `devops-infra` control-plane helpers. New shared infra should land here first; old `devops-infra` resources migrate gradually only when a concrete owner and validation path exists. ## Source Of Truth @@ -12,15 +12,15 @@ ## Sub2API Deployment Boundary - Sub2API is a platform service operated by UniDesk in namespace `platform-infra`. It is not a HWLAB lane workload, AgentRun workload, D601 user service, or master server daemon. -- The canonical deployment entrypoint is `bun scripts/cli.ts platform-infra sub2api plan|apply|status|validate|codex-pool`. Runtime targets are selected with `--target`; `G14` is the default active target and `D601` is controlled by the same YAML as either standby predeploy or externally backed active runtime. Daily operation procedures live in `$unidesk-sub2api` at `.agents/skills/unidesk-sub2api/SKILL.md`. This reference keeps only development boundaries and project-specific source-of-truth rules. +- The canonical deployment entrypoint is `bun scripts/cli.ts platform-infra sub2api plan|apply|status|validate|codex-pool`. Runtime targets are selected with `--target`; the Sub2API active target is the target whose YAML role/database mode enables active replicas, currently `D601`, and `G14` is kept as a standby predeploy. Daily operation procedures live in `$unidesk-sub2api` at `.agents/skills/unidesk-sub2api/SKILL.md`. This reference keeps only development boundaries and project-specific source-of-truth rules. - Raw `kubectl` through `trans :k3s` is only for bounded diagnosis and evidence, not a formal mutate path. - The image version is controlled by `config/platform-infra/sub2api.yaml`. Image update procedures are daily operations owned by `$unidesk-sub2api`; the development boundary is that image choices remain YAML-controlled. - Sub2API should stay ClusterIP-only by default. Do not add Ingress, NodePort, LoadBalancer, or broad FRP exposure unless a YAML-controlled public exposure decision exists. - Sub2API currently has no resource limits by design. Do not add CPU or memory limits unless a later explicit decision changes that policy and stores the new policy in YAML. - Master server is a consumer/control host, not the runtime location. Do not deploy Sub2API, PostgreSQL, Redis, or heavy validation loops on master server. -- D601 Sub2API is selected by YAML, not by ad hoc runtime patches. In standby mode it must render without a local PostgreSQL StatefulSet, keep the Sub2API app and local Redis cache scaled to zero, and use only ephemeral Redis storage when Redis is later activated. In externally backed active mode it connects directly to the YAML-declared external PostgreSQL endpoint with `sslmode=require`, keeps durable app state outside the D601 k3s node, and uses local Redis only as ephemeral cache. Activation must be applied through the same `platform-infra sub2api --target D601` CLI path. +- Sub2API active/standby placement is selected by YAML, not by ad hoc runtime patches. A standby target must render without a local PostgreSQL StatefulSet, keep the Sub2API app and local Redis cache scaled to zero, use only ephemeral Redis storage if Redis is later activated, and omit public FRP, HTTPS egress proxy, and account sentinel resources unless YAML explicitly promotes that target. An externally backed active target connects directly to the YAML-declared external PostgreSQL endpoint with `sslmode=require`, keeps durable app state outside the k3s node, and uses local Redis only as ephemeral cache. Promotion or failback must be applied by editing `config/platform-infra/sub2api.yaml` and running the same `platform-infra sub2api --target ` CLI path. - External platform PostgreSQL endpoints for Sub2API are produced by the platform DB YAML and its `platform-db postgres` CLI. Cross-node Sub2API consumers connect directly to that endpoint; the master server is not a PostgreSQL data-plane relay. DNS aliases are optional when the exported `DATABASE_URL` uses a reachable IP with `sslmode=require`; current PK01-specific rules live in `docs/reference/pk01.md`. -- Sub2API account sentinel and public exposure are target-scoped YAML decisions. Do not create a second sentinel, FRP client, public management surface, or edge proxy by hand; enable or move those resources only through the target YAML and the `platform-infra sub2api` / `codex-pool --target` CLI paths. +- Sub2API account sentinel, public exposure, and HTTPS egress proxy are target-scoped YAML decisions. The active target may run them when YAML enables them; the standby G14 target must stay deployed but inactive until YAML promotion. Do not create a second sentinel, FRP client, public management surface, or edge proxy by hand; enable or move those resources only through the target YAML and the `platform-infra sub2api` / `codex-pool --target` CLI paths. ## LangBot Deployment Boundary @@ -68,7 +68,7 @@ - Codex accounts selected by YAML do not declare `schedulable` as durable configuration. `schedulable=true` is a `codex-pool sync --confirm` process-control baseline for UniDesk-managed accounts that are not under sentinel quarantine, not a YAML field. - `codex-pool sync --confirm` preserves UniDesk-managed accounts that are absent from YAML by default; explicit upstream retirement requires `codex-pool sync --confirm --prune-removed`. This keeps account deletion out of the normal availability-recovery path and prevents temporary YAML edits from becoming destructive runtime changes. - `profiles.entries` selects local Codex profile files from `~/.codex/` and maps them to Sub2API account names. -- The unsuffixed master `~/.codex/config.toml` and `~/.codex/auth.json` are reserved for the unified Sub2API consumer. `config.toml` must keep `base_url = "https://sub2api.74-48-78-17.nip.io/"`, and `auth.json` must contain the unified pool API key from `pool.apiKeySecretName` / `pool.apiKeySecretKey`. Do not replace these two files with direct upstream account credentials. +- The unsuffixed master `~/.codex/config.toml` and `~/.codex/auth.json` are reserved for the unified Sub2API consumer. `config.toml` must keep the YAML-selected consumer base URL written by `codex-pool configure-local --target --confirm`, and `auth.json` must contain the unified pool API key from `pool.apiKeySecretName` / `pool.apiKeySecretKey` on that active target. Do not replace these two files with direct upstream account credentials. - Additional upstream accounts must use suffixed local profile files such as `config.toml.` and `auth.json.`, then be declared through `profiles.entries` in `config/platform-infra/sub2api-codex-pool.yaml`. - `profiles.entries[].capacity` optionally overrides `pool.defaultAccountCapacity` for one account. Capacity is a YAML-controlled routing input; concrete current values belong only in `config/platform-infra/sub2api-codex-pool.yaml` and runtime validation output, not in long-term reference prose. Code constants, Secrets, ad-hoc runtime patches, or stale tests must not override YAML source of truth. - `profiles.entries[].loadFactor` optionally overrides `pool.defaultAccountLoadFactor` for one account and is rendered to Sub2API `load_factor`. Treat it as routing policy: values belong in YAML and `codex-pool validate` output, not code constants, Secrets, or ad-hoc runtime patches. @@ -78,7 +78,7 @@ - Codex account-state, quota prompts, model-routing failures, encrypted-content affinity failures, gateway wrappers, and timeout-like upstream errors must be handled by the generic temporary-unschedulable/failover path plus the external marker sentinel. Do not change membership, priority, capacity, load factor, WebSocket mode, `pool_mode`, or a specific provider's status merely to work around those errors. If a matching upstream failure still logs `openai.forward_failed` without `openai.upstream_failover_switching`, the missing fix is in Sub2API's HTTP `/responses` failover classification/error propagation, not in account pinning. - `profiles.entries[].openaiResponsesWebSocketsV2Mode` is the account-level Responses WebSocket v2 switch for OpenAI-compatible upstreams that require WebSocket transport. Allowed values are `off`, `ctx_pool`, and `passthrough`; omit the field unless that upstream needs it. - `profiles.entries[].upstreamUserAgent` is an optional account-level upstream request User-Agent override. Use it only for upstreams that require a Codex CLI compatible User-Agent; keep the value YAML-controlled and newline-free. -- `publicExposure` in `config/platform-infra/sub2api-codex-pool.yaml` controls the default Codex-pool public bridge from master server to the G14 ClusterIP service. Target-level `publicExposure` in `config/platform-infra/sub2api.yaml` controls non-master exposure such as a D601-to-PK01 edge. +- `publicExposure` in `config/platform-infra/sub2api-codex-pool.yaml` controls the legacy Codex-pool public bridge from master server to the G14 ClusterIP service and should stay disabled unless that bridge is explicitly reintroduced. Target-level `publicExposure` in `config/platform-infra/sub2api.yaml` controls the active public edge such as D601-to-PK01. - `publicExposure.masterCaddy.responseHeaderTimeoutSeconds` controls the master Caddy `response_header_timeout` for the public Sub2API site. It must be long enough for Codex `/responses/compact` requests; otherwise Caddy can return a client-visible 504 before Sub2API finishes the upstream compact request, and that edge timeout is not an account-level upstream failure that Sub2API can use for temporary-unschedulable failover. The numeric value belongs only in `config/platform-infra/sub2api-codex-pool.yaml`; after changing it, use `codex-pool expose --confirm` to reload Caddy and verify the rendered `response_header_timeout`. Requests that were already in flight before the reload may still finish with the previous timeout, so post-change evidence should check only requests that started after the reload. - `publicExposure.masterCaddy.edgeRetry` controls the master Caddy reverse-proxy retry window for the public Sub2API site. This belongs at the edge because FRP remotePort listener loss, `connection refused`, EOF, or connection reset can happen before a request reaches Sub2API, so Sub2API account failover and sentinel logic cannot observe or recover that request. Keep retry scope narrow, especially for non-idempotent POST traffic: connection-attempt failures may be retried by the reverse proxy, while round-trip retry after an upstream connection was established should be limited by YAML `retryMatch` to paths that are safe to repeat, such as compact. Retry durations and intervals belong only in YAML; after changing them, run `codex-pool expose --confirm` and verify the rendered Caddyfile contains the expected `lb_try_duration`, `lb_try_interval`, and `lb_retry_match`. - `localCodex` controls how the master server's current `~/.codex` consumer files are backed up and rewritten. Keep `supportsWebSockets` and `responsesWebSocketsV2` in the same state, and enable them only when at least one YAML-managed account has a current direct Codex WSv2 smoke that passes. If no upstream profile can sustain Responses WSv2, the honest long-term state is `false/false` so Codex uses HTTP Responses directly instead of repeatedly reconnecting before `response.completed`. `localCodex.responsesSmokeModel` is the YAML-declared model used by `codex-pool validate` for the lightweight `POST /v1/responses` smoke. @@ -125,15 +125,15 @@ If the YAML success cadence maximum is lowered or an account changes trust class Operational observation for this sentinel should use the read-only `codex-pool sentinel-report` table or its `--raw` form. It is the canonical low-noise view for per-account probe count, trust class, marker result, HTTP/error diagnostics, freeze TTL, success cadence, success cadence maximum, next probe time, and recent CronJob runs; raw ConfigMap dumps and ad hoc log scraping are fallback diagnostics, not the primary state surface. -The default G14 Codex-pool request path is: +The active Codex-pool request path follows the YAML-selected active target: -1. A client sends an OpenAI-compatible request to the configured consumer base URL, normally `https://sub2api.74-48-78-17.nip.io/v1/...`, with the unified API key. -2. master `frps` forwards the TCP connection to `platform-infra/sub2api-frpc` when `publicExposure.enabled` is true. -3. `sub2api-frpc` forwards to `sub2api.platform-infra.svc.cluster.local:8080`. +1. A client sends an OpenAI-compatible request to the configured consumer base URL with the unified API key. +2. The target-level public edge forwards traffic to that target's `sub2api-frpc` when `config/platform-infra/sub2api.yaml` enables `publicExposure`. +3. `sub2api-frpc` forwards to `sub2api.platform-infra.svc.cluster.local:8080` inside the active target namespace. 4. Sub2API validates the unified key and resolves its `group_id`. 5. Accounts listed in `profiles.entries` are bound to the same group via `group_ids`, so Sub2API dispatches through that group using its own account selection semantics. -The D601 externally backed request path is different when target-level `publicExposure.enabled=true` in `config/platform-infra/sub2api.yaml`: client traffic reaches PK01 Caddy, PK01 forwards to the YAML-declared FRP remote port, D601 `sub2api-frpc` connects directly to PK01 `frps`, and FRP forwards to `sub2api.platform-infra.svc.cluster.local:8080` on D601. This path does not pass through the master server or the pikanode reverse proxy. `api.pikapython.com` must resolve to the YAML-declared PK01 public address before Caddy can obtain or renew the public certificate; when DNS is missing, PK01 local FRP probes and public-IP remote-port probes may prove the edge path, but they are not a substitute for final `https://api.pikapython.com` validation. +For the current D601 externally backed active target, client traffic reaches PK01 Caddy, PK01 forwards to the YAML-declared FRP remote port, D601 `sub2api-frpc` connects directly to PK01 `frps`, and FRP forwards to `sub2api.platform-infra.svc.cluster.local:8080` on D601. This path does not pass through the master server or the pikanode reverse proxy. `api.pikapython.com` must resolve to the YAML-declared PK01 public address before Caddy can obtain or renew the public certificate; when DNS is missing, PK01 local FRP probes and public-IP remote-port probes may prove the edge path, but they are not a substitute for final `https://api.pikapython.com` validation. When target-level `egressProxy.enabled=true`, the D601 target renders an in-cluster HTTP(S) proxy client from the master VPN subscription source declared in YAML. The CLI injects the resulting proxy URL and `NO_PROXY` into Sub2API and, when requested by YAML, the Codex account sentinel. `platform-infra sub2api validate --target D601 --full` must prove the proxy Deployment/Service is ready and that an app pod can complete the YAML-declared health probe through the proxy. Subscription contents and generated proxy configs are Secret material and must not be printed. @@ -208,4 +208,4 @@ spec: This policy must be included in the `sub2api plan` / `apply` manifest rendering so that it is created as part of the normal deployment flow, not maintained as a manual one-off. -`platform-infra sub2api status` must report whether `NetworkPolicy/allow-all` exists and still has `podSelector: {}`, `policyTypes: [Ingress, Egress]`, `ingress: [{}]`, and `egress: [{}]`. For active bundled targets, `platform-infra sub2api validate` must also run temporary in-namespace probe pods that connect to `sub2api-postgres:5432` and `sub2api-redis:6379`; local `pg_isready` inside the PostgreSQL pod alone is insufficient because it does not exercise kube-router cross-pod policy evaluation. For external-DB standby targets, `validate --target` checks the predeployment shape: no local PostgreSQL, app replicas zero, ClusterIP services, allow-all NetworkPolicy, and local Redis declared as ephemeral cache with readiness required only when Redis replicas are above zero. For external-DB active targets, `validate --target` checks that the app uses the external database endpoint, local Redis is ephemeral, no local PostgreSQL StatefulSet exists, and any YAML-declared egress proxy and public exposure resources are present and probed through their configured paths. +`platform-infra sub2api status` must report whether `NetworkPolicy/allow-all` exists and still has `podSelector: {}`, `policyTypes: [Ingress, Egress]`, `ingress: [{}]`, and `egress: [{}]`. For active bundled targets, `platform-infra sub2api validate` must also run temporary in-namespace probe pods that connect to `sub2api-postgres:5432` and `sub2api-redis:6379`; local `pg_isready` inside the PostgreSQL pod alone is insufficient because it does not exercise kube-router cross-pod policy evaluation. For external-DB standby targets, `validate --target` checks the predeployment shape: no local PostgreSQL, app replicas zero, ClusterIP services, allow-all NetworkPolicy, local Redis declared as ephemeral cache with readiness required only when Redis replicas are above zero, and no standby-disabled public FRP, egress proxy, or sentinel CronJob remains. For external-DB active targets, `validate --target` checks that the app uses the external database endpoint, local Redis is ephemeral, no local PostgreSQL StatefulSet exists, and any YAML-declared egress proxy and public exposure resources are present and probed through their configured paths. diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index 7c384792..75ed3ec5 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -17,7 +17,8 @@ import { import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; const g14K3sRoute = "G14:k3s"; -const defaultTargetId = "G14"; +const defaultTargetId = "D601"; +const defaultTargetRoute = "D601:k3s"; const namespace = "platform-infra"; const serviceName = "sub2api"; const serviceDns = `${serviceName}.${namespace}.svc.cluster.local:8080`; @@ -79,6 +80,7 @@ interface CodexPoolRuntimeTarget { namespace: string; serviceName: string; serviceDns: string; + publicBaseUrl: string | null; appSecretName: string; egressProxy: { enabled: boolean; @@ -549,7 +551,7 @@ function codexPoolRuntimeTarget(targetId: string = defaultTargetId): CodexPoolRu const raw = parsed.targets.find((item) => isRecord(item) && String(item.id ?? "").toLowerCase() === targetId.toLowerCase()); if (!isRecord(raw)) throw new Error(`${sub2apiConfigPath}.targets does not contain target ${targetId}`); const id = stringValue(raw.id) ?? targetId; - const route = stringValue(raw.route) ?? (id === defaultTargetId ? g14K3sRoute : ""); + const route = stringValue(raw.route) ?? (id === defaultTargetId ? defaultTargetRoute : ""); const targetNamespace = stringValue(raw.namespace) ?? namespace; if (route.length === 0) throw new Error(`${sub2apiConfigPath}.targets[${id}].route is required`); validateKubernetesName(targetNamespace, `${sub2apiConfigPath}.targets[${id}].namespace`, true); @@ -571,12 +573,19 @@ function codexPoolRuntimeTarget(targetId: string = defaultTargetId): CodexPoolRu }; } + let publicBaseUrl: string | null = null; + if (isRecord(raw.publicExposure) && raw.publicExposure.enabled === true) { + publicBaseUrl = normalizeBaseUrl(stringValue(raw.publicExposure.publicBaseUrl)); + if (publicBaseUrl === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].publicExposure.publicBaseUrl must be a valid http(s) URL when target public exposure is enabled`); + } + return { id, route, namespace: targetNamespace, serviceName, serviceDns: `${serviceName}.${targetNamespace}.svc.cluster.local:8080`, + publicBaseUrl, appSecretName, egressProxy, }; @@ -591,6 +600,7 @@ function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false, t const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const profiles = collectCodexProfiles(); const ok = profiles.length > 0 && profiles.every((profile) => profile.ok); + const consumerBaseUrl = codexConsumerBaseUrl(pool, runtimeTarget); return { ok, action: "platform-infra-sub2api-codex-pool-plan", @@ -614,8 +624,10 @@ function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false, t ? `Account sentinel is enabled as k8s CronJob ${runtimeTarget.namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.` : "Account sentinel monitoring is disabled by YAML.", publicExposure: pool.publicExposure.enabled - ? `Default Codex consumers use ${codexConsumerBaseUrl(pool)}; bounded master-local probes may use ${pool.publicExposure.masterBaseUrl}. FRP proxy ${pool.publicExposure.proxyName} maps public ${pool.publicExposure.publicBaseUrl} to ${pool.publicExposure.localIP}:${pool.publicExposure.localPort}.` - : "Public FRP exposure is disabled by YAML.", + ? `Default Codex consumers use ${consumerBaseUrl}; bounded master-local probes may use ${pool.publicExposure.masterBaseUrl}. Legacy Codex-pool FRP proxy ${pool.publicExposure.proxyName} maps public ${pool.publicExposure.publicBaseUrl} to ${pool.publicExposure.localIP}:${pool.publicExposure.localPort}.` + : runtimeTarget.publicBaseUrl === null + ? "Public FRP exposure is disabled by YAML." + : `Legacy Codex-pool FRP exposure is disabled by YAML; Codex consumers for target ${runtimeTarget.id} use target-level public exposure ${consumerBaseUrl}.`, idempotency: "sync reuses the group, account names, and k3s Secret when they already exist; credentials are updated from the current local Codex files; managed accounts missing from YAML are preserved unless --prune-removed is explicitly provided.", configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.", }, @@ -994,6 +1006,8 @@ async function codexPoolExpose(config: UniDeskConfig, options: ConfirmOptions): async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOptions): Promise> { const pool = readCodexPoolConfig(); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); + const consumerExposureAvailable = pool.publicExposure.enabled || runtimeTarget.publicBaseUrl !== null; const codexDir = join(homedir(), ".codex"); const configPath = join(codexDir, "config.toml"); const authPath = join(codexDir, "auth.json"); @@ -1010,7 +1024,7 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp authPath, backupConfigPath, backupAuthPath, - baseUrl: codexConsumerBaseUrl(pool), + baseUrl: consumerExposureAvailable ? codexConsumerBaseUrl(pool, runtimeTarget) : null, providerName: pool.localCodex.providerName, wireApi: pool.localCodex.wireApi, modelContextWindow: pool.localCodex.modelContextWindow, @@ -1021,12 +1035,12 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp valuesPrinted: false, }, next: { - confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm", + confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool configure-local${targetFlag(runtimeTarget)} --confirm`, }, }; } - if (!pool.publicExposure.enabled) throw new Error(`${codexPoolConfigPath}.publicExposure.enabled must be true before configure-local`); - const keyResult = await fetchPoolApiKey(config, pool); + if (!consumerExposureAvailable) throw new Error(`${codexPoolConfigPath}.publicExposure.enabled is false and ${sub2apiConfigPath}.targets[${runtimeTarget.id}].publicExposure.enabled is not true; configure-local needs one consumer URL`); + const keyResult = await fetchPoolApiKey(config, pool, runtimeTarget); if (keyResult.apiKey === null) { return { ok: false, @@ -1035,8 +1049,8 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp valuesPrinted: false, }; } - const writeResult = writeLocalCodexConfig(pool, keyResult.apiKey); - const validateResult = await validatePublicGatewayWithKey(pool, keyResult.apiKey); + const writeResult = writeLocalCodexConfig(pool, keyResult.apiKey, runtimeTarget); + const validateResult = await validatePublicGatewayWithKey(pool, keyResult.apiKey, runtimeTarget); return { ok: writeResult.ok && validateResult.ok, action: "platform-infra-sub2api-codex-pool-configure-local", @@ -1044,7 +1058,7 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp local: writeResult, validation: validateResult, apiKey: { - secret: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, + secret: `${runtimeTarget.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, keyPreview: apiKeyPreview(keyResult.apiKey), apiKeyFingerprint: fingerprint(keyResult.apiKey), valuesPrinted: false, @@ -2633,6 +2647,8 @@ function poolTarget(pool = readCodexPoolConfig(), target = codexPoolRuntimeTarge namespace: target.namespace, service: serviceName, serviceDns: target.serviceDns, + publicBaseUrl: target.publicBaseUrl, + consumerBaseUrl: pool.publicExposure.enabled || target.publicBaseUrl !== null ? codexConsumerBaseUrl(pool, target) : null, configPath: codexPoolConfigPath, groupName: pool.groupName, apiKeyName: pool.apiKeyName, @@ -3070,7 +3086,7 @@ kubectl -n ${target.namespace} get secret ${pool.apiKeySecretName} -o json } } -function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string): Record { +function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget(defaultTargetId)): Record { const codexDir = join(homedir(), ".codex"); mkdirSync(codexDir, { recursive: true, mode: 0o700 }); const configPath = join(codexDir, "config.toml"); @@ -3082,7 +3098,7 @@ function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string): Record> { - const probe = await probePublicModels(pool, "with-api-key", apiKey, "public"); +async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: string, target = codexPoolRuntimeTarget(defaultTargetId)): Promise> { + const probeBase = target.publicBaseUrl === null ? "public" : "target-public"; + const probe = await probePublicModels(pool, "with-api-key", apiKey, probeBase, target); return { ...probe, ok: probe.ok === true, @@ -3248,8 +3266,12 @@ async function validatePublicGatewayWithKey(pool: CodexPoolConfig, apiKey: strin }; } -async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, base: "master" | "public" = "master"): Promise> { - const baseUrl = base === "public" ? pool.publicExposure.publicBaseUrl : pool.publicExposure.masterBaseUrl; +async function probePublicModels(pool: CodexPoolConfig, mode: "with-api-key" | "without-api-key", apiKey?: string, base: "master" | "public" | "target-public" = "master", target = codexPoolRuntimeTarget(defaultTargetId)): Promise> { + const baseUrl = base === "target-public" + ? (target.publicBaseUrl ?? pool.publicExposure.publicBaseUrl) + : base === "public" + ? pool.publicExposure.publicBaseUrl + : pool.publicExposure.masterBaseUrl; const url = `${baseUrl.replace(/\/+$/u, "")}/v1/models`; const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 8000); diff --git a/scripts/src/platform-infra.ts b/scripts/src/platform-infra.ts index ea4c0ba7..61f79cf5 100644 --- a/scripts/src/platform-infra.ts +++ b/scripts/src/platform-infra.ts @@ -7,12 +7,13 @@ import { startJob } from "./jobs"; import type { RenderedCliResult } from "./output"; import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; -const defaultTargetId = "G14"; +const defaultTargetId = "D601"; const namespace = "platform-infra"; const serviceName = "sub2api"; const fieldManager = "unidesk-platform-infra"; const manifestPath = rootPath("src", "components", "platform-infra", "sub2api", "sub2api.k8s.yaml"); const configPath = rootPath("config", "platform-infra", "sub2api.yaml"); +const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml"); const repoRoot = rootPath(); const secretName = "sub2api-secrets"; const requiredSecretKeys = ["POSTGRES_PASSWORD", "ADMIN_PASSWORD", "JWT_SECRET", "TOTP_ENCRYPTION_KEY"] as const; @@ -189,6 +190,33 @@ interface EgressProxySecretMaterial { valuesPrinted: false; } +interface ManagedResourceCleanupPlan { + externalDbState: boolean; + redisPersistentState: boolean; + publicExposure: { + enabled: boolean; + deploymentName: string; + configMapName: string; + secretName: string; + }; + egressProxy: { + enabled: boolean; + deploymentName: string; + serviceName: string; + secretName: string; + }; + sentinel: { + enabled: boolean; + cronJobName: string; + configMapName: string; + credentialsSecretName: string; + stateConfigMapName: string; + serviceAccountName: string; + roleName: string; + roleBindingName: string; + }; +} + export function platformInfraHelp(): unknown { return { command: "platform-infra sub2api|langbot|n8n|wechat-archive ...", @@ -898,6 +926,68 @@ function configHashFor(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): str .slice(0, 16); } +function targetHasSentinel(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): boolean { + return sub2api.runtime.sentinel.enabledOnTargets.some((id) => id.toLowerCase() === target.id.toLowerCase()); +} + +function managedResourceCleanupPlan(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): ManagedResourceCleanupPlan { + const sentinel = codexPoolSentinelResourceNames(); + return { + externalDbState: isExternalTarget(target), + redisPersistentState: target.redisMode === "local-ephemeral", + publicExposure: { + enabled: target.publicExposure?.enabled === true, + deploymentName: target.publicExposure?.frpc.deploymentName ?? "sub2api-frpc", + configMapName: "sub2api-frpc-config", + secretName: target.publicExposure?.frpc.secretName ?? "sub2api-frpc-secrets", + }, + egressProxy: { + enabled: target.egressProxy?.enabled === true, + deploymentName: target.egressProxy?.deploymentName ?? "sub2api-egress-proxy", + serviceName: target.egressProxy?.serviceName ?? "sub2api-egress-proxy", + secretName: target.egressProxy?.secretName ?? "sub2api-egress-proxy-config", + }, + sentinel: { + ...sentinel, + enabled: targetHasSentinel(sub2api, target), + }, + }; +} + +function codexPoolSentinelResourceNames(): ManagedResourceCleanupPlan["sentinel"] { + const defaults: ManagedResourceCleanupPlan["sentinel"] = { + enabled: false, + cronJobName: "sub2api-account-sentinel", + configMapName: "sub2api-account-sentinel-config", + credentialsSecretName: "sub2api-account-sentinel-profiles", + stateConfigMapName: "sub2api-account-sentinel-state", + serviceAccountName: "sub2api-account-sentinel", + roleName: "sub2api-account-sentinel", + roleBindingName: "sub2api-account-sentinel", + }; + if (!existsSync(codexPoolConfigPath)) return defaults; + const parsed = Bun.YAML.parse(readFileSync(codexPoolConfigPath, "utf8")) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return defaults; + const sentinel = (parsed as Record).sentinel; + if (typeof sentinel !== "object" || sentinel === null || Array.isArray(sentinel)) return defaults; + const record = sentinel as Record; + const text = (key: string, fallback: string): string => { + const value = record[key]; + return typeof value === "string" && value.length > 0 ? value : fallback; + }; + const serviceAccountName = text("serviceAccountName", defaults.serviceAccountName); + return { + enabled: false, + cronJobName: text("cronJobName", defaults.cronJobName), + configMapName: text("configMapName", defaults.configMapName), + credentialsSecretName: text("credentialsSecretName", defaults.credentialsSecretName), + stateConfigMapName: text("stateConfigMapName", defaults.stateConfigMapName), + serviceAccountName, + roleName: serviceAccountName, + roleBindingName: serviceAccountName, + }; +} + function externalPendingManifest(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { const urlAllowlist = sub2api.security.urlAllowlist; const database = sub2api.runtime.database; @@ -1473,13 +1563,13 @@ function plan(options: TargetOptions): Record { decision: { owner: "UniDesk", namespace: target.namespace, - reason: target.id === "G14" - ? "Sub2API remains the active G14 platform-infra deployment." + reason: target.databaseMode === "external-pending" + ? `${target.id} is a standby Sub2API platform-infra target prepared through YAML; it stays scaled to zero until this target is promoted in YAML.` : target.databaseMode === "external-active" - ? "D601 is activated as a platform-infra Sub2API target against the external PK01 PostgreSQL runtime while keeping app state outside the k3s node." - : "D601 is a standby Sub2API platform-infra target prepared through YAML; it does not run until the external Pika01/PK01 database secret is ready.", + ? `${target.id} is activated as a platform-infra Sub2API target against the external PK01 PostgreSQL runtime while keeping app state outside the k3s node.` + : `${target.id} is a bundled active Sub2API platform-infra target.`, exposure: target.publicExposure?.enabled - ? `Public HTTPS ${target.publicExposure.publicBaseUrl} through PK01 Caddy and D601 frpc; no master server forwarding and no Kubernetes Ingress/NodePort/LoadBalancer.` + ? `Public HTTPS ${target.publicExposure.publicBaseUrl} through PK01 Caddy and ${target.id} frpc; no master server forwarding and no Kubernetes Ingress/NodePort/LoadBalancer.` : "ClusterIP only; no public ingress or node-level exposure.", resourcePolicy: "No Kubernetes CPU/memory requests or limits, matching issue #220.", imageVersionControl: "Sub2API image repository/tag/pullPolicy are controlled by config/platform-infra/sub2api.yaml in the UniDesk repository.", @@ -1488,7 +1578,7 @@ function plan(options: TargetOptions): Record { publicExposure: target.publicExposure?.enabled ? { mode: "pk01-caddy-frp-direct", - dataPath: "client -> PK01 Caddy -> PK01 frps remotePort -> D601 frpc -> Sub2API", + dataPath: `client -> PK01 Caddy -> PK01 frps remotePort -> ${target.id} frpc -> Sub2API`, pikanodeRole: "pikapython.com upstream only; api.pikapython.com does not pass through pikanode Express", publicBaseUrl: target.publicExposure.publicBaseUrl, hostname: target.publicExposure.dns.hostname, @@ -1497,7 +1587,7 @@ function plan(options: TargetOptions): Record { : null, egressProxy: target.egressProxy?.enabled ? { - mode: "D601 in-cluster HTTP proxy client to the master VPN subscription", + mode: `${target.id} in-cluster HTTP proxy client to the master VPN subscription`, service: `${target.egressProxy.serviceName}.${target.namespace}.svc.cluster.local:${target.egressProxy.listenPort}`, sourceRef: target.egressProxy.sourceRef, applyToSub2Api: target.egressProxy.applyToSub2Api, @@ -1508,7 +1598,7 @@ function plan(options: TargetOptions): Record { dataStores: isExternalTarget(target) ? [ target.databaseMode === "external-active" ? "External PostgreSQL active from platform-db/PK01" : "External PostgreSQL pending from platform-db/Pika01", - target.redisReplicas === 0 ? "D601 local Redis 8 ephemeral cache, scaled to zero until activation" : "D601 local Redis 8 ephemeral cache", + target.redisReplicas === 0 ? `${target.id} local Redis 8 ephemeral cache, scaled to zero until activation` : `${target.id} local Redis 8 ephemeral cache`, ] : ["PostgreSQL 18", "Redis 8"], appPoolCaps: { @@ -1517,6 +1607,16 @@ function plan(options: TargetOptions): Record { redisPoolSize: 32, redisMinIdleConns: 2, }, + standbyActivation: target.databaseMode === "external-pending" + ? { + target: target.id, + currentMode: "predeployed-standby", + enableByYaml: `change target ${target.id} databaseMode to external-active and set appReplicas/redisReplicas to 1, then apply --target ${target.id} --confirm`, + sharedExternalDb: `${sub2api.runtime.database.host}:${sub2api.runtime.database.port}/${sub2api.runtime.database.dbName}`, + sentinelEnabled: targetHasSentinel(sub2api, target), + } + : null, + cleanup: managedResourceCleanupPlan(sub2api, target), }, policy, next: { @@ -1587,7 +1687,7 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise"$ns_out" 2>"$ns_err" ns_rc=$? secret_action="unknown" @@ -2803,7 +2907,98 @@ else : >"$apply_out" printf '%s\\n' 'skipped because namespace, app secret, public exposure secret, or egress proxy secret step failed' >"$apply_err" fi -python3 - "$ns_rc" "$secret_rc" "$exposure_secret_rc" "$egress_secret_rc" "$apply_rc" "$secret_action" "$exposure_secret_action" "$egress_secret_action" "$ns_out" "$ns_err" "$secret_out" "$secret_err" "$exposure_secret_out" "$exposure_secret_err" "$egress_secret_out" "$egress_secret_err" "$apply_out" "$apply_err" <<'PY' +cleanup_rc=0 +if [ "$apply_rc" -eq 0 ]; then + python3 - "$cleanup_out" "$cleanup_err" <<'PY' +import json +import subprocess +import sys +import time + +out_path, err_path = sys.argv[1:3] +namespace = ${JSON.stringify(target.namespace)} +plan = json.loads(${JSON.stringify(JSON.stringify(cleanupPlan))}) + +def run(args): + proc = subprocess.run(["kubectl", "-n", namespace, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return { + "args": ["kubectl", "-n", namespace, *args], + "exitCode": proc.returncode, + "stdout": proc.stdout.decode("utf-8", errors="replace")[-2000:], + "stderr": proc.stderr.decode("utf-8", errors="replace")[-2000:], + } + +def delete(kind, name): + return run(["delete", kind, name, "--ignore-not-found=true", "--wait=false"]) + +items = [] +if plan["externalDbState"]: + items.append({"name": "legacy-postgres-statefulset", **delete("statefulset", "sub2api-postgres")}) + items.append({"name": "legacy-postgres-service", **delete("service", "sub2api-postgres")}) + items.append({"name": "legacy-postgres-pvc", **delete("pvc", "postgres-data-sub2api-postgres-0")}) + items.append({"name": "legacy-app-data-pvc", **delete("pvc", "sub2api-data")}) +if plan["redisPersistentState"]: + items.append({"name": "legacy-redis-pvc", **delete("pvc", "sub2api-redis-data")}) +if not plan["publicExposure"]["enabled"]: + items.append({"name": "disabled-public-frpc-deployment", **delete("deployment", plan["publicExposure"]["deploymentName"])}) + items.append({"name": "disabled-public-frpc-configmap", **delete("configmap", plan["publicExposure"]["configMapName"])}) + items.append({"name": "disabled-public-frpc-secret", **delete("secret", plan["publicExposure"]["secretName"])}) +if not plan["egressProxy"]["enabled"]: + items.append({"name": "disabled-egress-proxy-deployment", **delete("deployment", plan["egressProxy"]["deploymentName"])}) + items.append({"name": "disabled-egress-proxy-service", **delete("service", plan["egressProxy"]["serviceName"])}) + items.append({"name": "disabled-egress-proxy-secret", **delete("secret", plan["egressProxy"]["secretName"])}) +if not plan["sentinel"]["enabled"]: + items.append({"name": "disabled-sentinel-cronjob", **delete("cronjob", plan["sentinel"]["cronJobName"])}) + items.append({"name": "disabled-sentinel-jobs", **run(["delete", "job", "-l", f"app.kubernetes.io/name={plan['sentinel']['cronJobName']}", "--ignore-not-found=true", "--wait=false"])}) + items.append({"name": "disabled-sentinel-configmap", **delete("configmap", plan["sentinel"]["configMapName"])}) + items.append({"name": "disabled-sentinel-credentials-secret", **delete("secret", plan["sentinel"]["credentialsSecretName"])}) + items.append({"name": "disabled-sentinel-serviceaccount", **delete("serviceaccount", plan["sentinel"]["serviceAccountName"])}) + items.append({"name": "disabled-sentinel-role", **delete("role", plan["sentinel"]["roleName"])}) + items.append({"name": "disabled-sentinel-rolebinding", **delete("rolebinding", plan["sentinel"]["roleBindingName"])}) + +watch = [] +if plan["externalDbState"]: + watch.extend([("statefulset", "sub2api-postgres"), ("service", "sub2api-postgres"), ("pvc", "postgres-data-sub2api-postgres-0"), ("pvc", "sub2api-data")]) +if plan["redisPersistentState"]: + watch.append(("pvc", "sub2api-redis-data")) +if not plan["publicExposure"]["enabled"]: + watch.append(("deployment", plan["publicExposure"]["deploymentName"])) +if not plan["egressProxy"]["enabled"]: + watch.append(("deployment", plan["egressProxy"]["deploymentName"])) + watch.append(("service", plan["egressProxy"]["serviceName"])) +if not plan["sentinel"]["enabled"]: + watch.append(("cronjob", plan["sentinel"]["cronJobName"])) + +deadline = time.time() + 90 +remaining = [] +while True: + remaining = [] + for kind, name in watch: + result = run(["get", kind, name]) + if result["exitCode"] == 0: + remaining.append({"kind": kind, "name": name}) + if not remaining or time.time() >= deadline: + break + time.sleep(3) + +payload = { + "ok": all(item["exitCode"] == 0 for item in items) and not remaining, + "plan": plan, + "items": items, + "remainingAfterWait": remaining, + "valuesPrinted": False, +} +open(out_path, "w", encoding="utf-8").write(json.dumps(payload, ensure_ascii=False, indent=2)) +open(err_path, "w", encoding="utf-8").write("") +sys.exit(0 if payload["ok"] else 1) +PY + cleanup_rc=$? +else + : >"$cleanup_out" + printf '%s\\n' 'skipped because apply failed' >"$cleanup_err" + cleanup_rc=1 +fi +python3 - "$ns_rc" "$secret_rc" "$exposure_secret_rc" "$egress_secret_rc" "$apply_rc" "$cleanup_rc" "$secret_action" "$exposure_secret_action" "$egress_secret_action" "$ns_out" "$ns_err" "$secret_out" "$secret_err" "$exposure_secret_out" "$exposure_secret_err" "$egress_secret_out" "$egress_secret_err" "$apply_out" "$apply_err" "$cleanup_out" "$cleanup_err" <<'PY' import json import sys ns_rc = int(sys.argv[1]) @@ -2811,17 +3006,23 @@ secret_rc = int(sys.argv[2]) exposure_secret_rc = int(sys.argv[3]) egress_secret_rc = int(sys.argv[4]) apply_rc = int(sys.argv[5]) -secret_action = sys.argv[6] -exposure_secret_action = sys.argv[7] -egress_secret_action = sys.argv[8] -paths = sys.argv[9:] +cleanup_rc = int(sys.argv[6]) +secret_action = sys.argv[7] +exposure_secret_action = sys.argv[8] +egress_secret_action = sys.argv[9] +paths = sys.argv[10:] def text(path): try: return open(path, encoding="utf-8").read() except FileNotFoundError: return "" +def parsed(path): + try: + return json.load(open(path, encoding="utf-8")) + except Exception: + return None payload = { - "ok": ns_rc == 0 and secret_rc == 0 and exposure_secret_rc == 0 and egress_secret_rc == 0 and apply_rc == 0, + "ok": ns_rc == 0 and secret_rc == 0 and exposure_secret_rc == 0 and egress_secret_rc == 0 and apply_rc == 0 and cleanup_rc == 0, "target": "${target.id}", "namespace": "${target.namespace}", "databaseMode": "${target.databaseMode}", @@ -2843,6 +3044,7 @@ payload = { "publicExposureSecret": {"exitCode": exposure_secret_rc, "action": exposure_secret_action, "stdout": text(paths[4])[-4000:], "stderr": text(paths[5])[-4000:]}, "egressProxySecret": {"exitCode": egress_secret_rc, "action": egress_secret_action, "stdout": text(paths[6])[-4000:], "stderr": text(paths[7])[-4000:]}, "apply": {"exitCode": apply_rc, "stdout": text(paths[8])[-8000:], "stderr": text(paths[9])[-4000:]}, + "cleanup": {"exitCode": cleanup_rc, "summary": parsed(paths[10]), "stdout": text(paths[10])[-8000:], "stderr": text(paths[11])[-4000:]}, }, } print(json.dumps(payload, ensure_ascii=False, indent=2)) @@ -2878,6 +3080,7 @@ capture_json pvc kubectl -n ${target.namespace} get pvc -l app.kubernetes.io/par capture_json secrets kubectl -n ${target.namespace} get secret ${secretName} capture_json configmap kubectl -n ${target.namespace} get configmap sub2api-config capture_json networkpolicies kubectl -n ${target.namespace} get networkpolicy +capture_json cronjobs kubectl -n ${target.namespace} get cronjob -l app.kubernetes.io/part-of=platform-infra capture_json ingresses kubectl -n ${target.namespace} get ingress capture_json quotas kubectl -n ${target.namespace} get resourcequota capture_json limitranges kubectl -n ${target.namespace} get limitrange @@ -3074,6 +3277,7 @@ services = items("services") pods = items("pods") pvcs = items("pvc") networkpolicies = items("networkpolicies") +cronjobs = items("cronjobs") secret = load("secrets") configmap = load("configmap") configmap_data = (configmap or {}).get("data") or {} @@ -3169,6 +3373,14 @@ statefulset_summaries = [statefulset_summary(item) for item in statefulsets] workload_ready = all(d["ready"] for d in deployment_summaries) and all(s["ready"] for s in statefulset_summaries) local_postgres_present = any(item.get("metadata", {}).get("name") == "sub2api-postgres" for item in statefulsets + services + pvcs) redis_pvc_present = any(item.get("metadata", {}).get("name") == "sub2api-redis-data" for item in pvcs) +public_exposure_deployment_present = any(item.get("metadata", {}).get("name") == "sub2api-frpc" for item in deployments) +egress_proxy_deployment_present = any(item.get("metadata", {}).get("name") == "sub2api-egress-proxy" for item in deployments) +sentinel_cronjob_present = any(item.get("metadata", {}).get("name") == "${codexPoolSentinelResourceNames().cronJobName}" for item in cronjobs) +standby_disabled_resources_ok = not external_pending or ( + not public_exposure_deployment_present + and not egress_proxy_deployment_present + and not sentinel_cronjob_present +) secret_ready = len(missing_secret_keys) == 0 secret_ok = secret_ready or external_pending if external_pending: @@ -3179,6 +3391,7 @@ if external_pending: and redis_desired_aligned and expected_app_replicas == 0 and expected_redis_replicas == 0 + and standby_disabled_resources_ok ) elif external_active: state_model_ok = ( @@ -3223,6 +3436,10 @@ payload = { "redisPvcPresent": redis_pvc_present, "sub2apiDesiredReplicasAligned": sub2api_desired_aligned, "redisDesiredReplicasAligned": redis_desired_aligned, + "standbyDisabledResourcesOk": standby_disabled_resources_ok, + "publicExposureDeploymentPresent": public_exposure_deployment_present, + "egressProxyDeploymentPresent": egress_proxy_deployment_present, + "sentinelCronJobPresent": sentinel_cronjob_present, "ok": state_model_ok, }, "imageControl": { @@ -3702,7 +3919,9 @@ capture_json() { capture_json ns kubectl get namespace ${target.namespace} capture_json app kubectl -n ${target.namespace} get deployment ${serviceName} capture_json redis kubectl -n ${target.namespace} get deployment ${redisService} +capture_json deployments kubectl -n ${target.namespace} get deployments capture_json services kubectl -n ${target.namespace} get service +capture_json cronjobs kubectl -n ${target.namespace} get cronjob capture_json statefulsets kubectl -n ${target.namespace} get statefulsets capture_json pvc kubectl -n ${target.namespace} get pvc capture_json configmap kubectl -n ${target.namespace} get configmap sub2api-config @@ -3772,6 +3991,8 @@ def deployment_summary(item): } services = items("services") +deployments = items("deployments") +cronjobs = items("cronjobs") statefulsets = items("statefulsets") pvcs = items("pvc") configmap = load("configmap") @@ -3785,6 +4006,16 @@ network_policy_ok = rc("networkpolicy") == 0 and is_allow_all_network_policy(net service_names = sorted(item.get("metadata", {}).get("name") for item in services) local_postgres_present = any(item.get("metadata", {}).get("name") == "sub2api-postgres" for item in services + statefulsets + pvcs) redis_pvc_present = any(item.get("metadata", {}).get("name") == "sub2api-redis-data" for item in pvcs) +public_exposure_deployment_present = any(item.get("metadata", {}).get("name") == "sub2api-frpc" for item in deployments) +egress_proxy_deployment_present = any(item.get("metadata", {}).get("name") == "sub2api-egress-proxy" for item in deployments) +egress_proxy_service_present = "sub2api-egress-proxy" in service_names +sentinel_cronjob_present = any(item.get("metadata", {}).get("name") == "${codexPoolSentinelResourceNames().cronJobName}" for item in cronjobs) +standby_disabled_ok = ( + not public_exposure_deployment_present + and not egress_proxy_deployment_present + and not egress_proxy_service_present + and not sentinel_cronjob_present +) redis_ephemeral = any(volume.get("emptyDir") == {} for volume in redis_summary["volumes"]) configmap_aligned = ( configmap_data.get("DATABASE_HOST") == "${database.host}" @@ -3798,7 +4029,7 @@ app_scaled_to_zero = app_summary["exists"] and app_summary["desired"] == ${targe image_aligned = "${expectedImage}" in app_summary["images"] redis_ready = redis_summary["exists"] and redis_summary["desired"] == ${target.redisReplicas} and redis_ephemeral and (redis_summary["ready"] or ${target.redisReplicas} == 0) payload = { - "ok": rc("ns") == 0 and network_policy_ok and app_scaled_to_zero and image_aligned and redis_ready and configmap_aligned and not local_postgres_present and not redis_pvc_present and "${serviceName}" in service_names and "${redisService}" in service_names, + "ok": rc("ns") == 0 and network_policy_ok and app_scaled_to_zero and image_aligned and redis_ready and configmap_aligned and not local_postgres_present and not redis_pvc_present and standby_disabled_ok and "${serviceName}" in service_names and "${redisService}" in service_names, "target": "${target.id}", "namespace": "${target.namespace}", "status": "pending-external-db", @@ -3825,6 +4056,13 @@ payload = { "redisReadyEphemeral": {"ok": redis_ready, "summary": redis_summary, "redisPvcPresent": redis_pvc_present, "readinessRequired": ${target.redisReplicas} > 0}, "serviceBoundary": {"serviceNames": service_names, "servicePresent": "${serviceName}" in service_names, "redisServicePresent": "${redisService}" in service_names}, "externalDbOnly": {"ok": not local_postgres_present, "localPostgresPresent": local_postgres_present}, + "standbyDisabled": { + "ok": standby_disabled_ok, + "publicExposureDeploymentPresent": public_exposure_deployment_present, + "egressProxyDeploymentPresent": egress_proxy_deployment_present, + "egressProxyServicePresent": egress_proxy_service_present, + "sentinelCronJobPresent": sentinel_cronjob_present, + }, "configMapAligned": {"ok": configmap_aligned, "databaseHost": configmap_data.get("DATABASE_HOST"), "redisHost": configmap_data.get("REDIS_HOST")}, }, "next": { From 792e62e03f7383d23ce6f593da2e62f35fa8d5ac Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 04:18:43 +0000 Subject: [PATCH 04/13] fix: harden sub2api codex failover stability --- config/platform-infra/sub2api-codex-pool.yaml | 2 +- scripts/src/platform-infra-sub2api-codex.ts | 97 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 1af9019d..1d21c5e5 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -161,7 +161,7 @@ sentinel: monitor: enabled: true actions: - enabled: false + enabled: true schedule: "*/1 * * * *" image: python:3.12-alpine sdk: diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index 75ed3ec5..710a9c39 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -2008,6 +2008,8 @@ function compactGatewayResponsesRecent(block: unknown): unknown { slowFinalErrorCount: block.slowFinalErrorCount, contextCanceledCount: block.contextCanceledCount, ignoredProbeNoiseCount: block.ignoredProbeNoiseCount, + failoverBudgetExhaustedCount: Array.isArray(block.failoverBudgetExhausted) ? block.failoverBudgetExhausted.length : undefined, + failoverBudgetExhausted: Array.isArray(block.failoverBudgetExhausted) ? block.failoverBudgetExhausted.slice(-3).reverse() : undefined, recentFailovers: Array.isArray(block.recentFailovers) ? block.recentFailovers.slice(-4).reverse() : undefined, recentForwardFailures: Array.isArray(block.recentForwardFailures) ? block.recentForwardFailures.slice(-4).reverse().map((item) => ({ ...item, @@ -5134,6 +5136,51 @@ def ignored_probe_noise(items, section): result.append(probe) return result +def group_by_request_id(items): + grouped = {} + for item in items: + request_id = item.get("requestId") + if not isinstance(request_id, str) or not request_id: + continue + grouped.setdefault(request_id, []).append(item) + return grouped + +def failover_budget_exhausted_evidence(failovers, final_errors): + final_by_request = {} + for item in final_errors: + request_id = item.get("requestId") + if isinstance(request_id, str) and request_id: + final_by_request[request_id] = item + exhausted = [] + for request_id, request_failovers in group_by_request_id(failovers).items(): + final = final_by_request.get(request_id) + if not final: + continue + last = request_failovers[-1] + switch_count = last.get("switchCount") + max_switches = last.get("maxSwitches") + final_status = final.get("statusCode") + if ( + isinstance(switch_count, int) + and isinstance(max_switches, int) + and max_switches > 0 + and switch_count >= max_switches + and isinstance(final_status, int) + and final_status >= 500 + ): + exhausted.append({ + "requestId": request_id, + "clientRequestId": final.get("clientRequestId") or last.get("clientRequestId"), + "path": final.get("path") or last.get("path"), + "finalAccountId": final.get("accountId"), + "finalStatusCode": final_status, + "switchCount": switch_count, + "maxSwitches": max_switches, + "lastFailoverAccountId": last.get("accountId"), + "lastUpstreamStatus": last.get("upstreamStatus"), + }) + return exhausted + def recent_responses_gateway_evidence(): proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"]) stdout = proc.stdout.decode("utf-8", errors="replace") @@ -5189,6 +5236,7 @@ def recent_responses_gateway_evidence(): visible_final_errors = filter_ignored_probe_noise(final_errors) visible_slow_final_errors = filter_ignored_probe_noise(slow_final_errors) visible_context_canceled = filter_ignored_probe_noise(context_canceled) + failover_budget_exhausted = failover_budget_exhausted_evidence(visible_failovers, visible_final_errors) probe_noise = ( ignored_probe_noise(failovers, "failovers") + ignored_probe_noise(forward_failures, "forwardFailures") @@ -5207,6 +5255,7 @@ def recent_responses_gateway_evidence(): "slowFinalErrorCount": len(visible_slow_final_errors), "contextCanceledCount": len(visible_context_canceled), "ignoredProbeNoiseCount": len(probe_noise), + "failoverBudgetExhausted": failover_budget_exhausted[-8:], "rawCounts": { "failoverCount": len(failovers), "forwardFailureCount": len(forward_failures), @@ -5990,6 +6039,8 @@ def trace_reason(events, final_event): failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + if failover_budget_exhausted(failovers, final_event): + return "failover-budget-exhausted" if failovers and select_failures: return "failover-attempted-no-candidate" if failovers: @@ -6004,6 +6055,47 @@ def trace_reason(events, final_event): return "completed" return "unknown" +def failover_budget_exhausted(failovers, final_event): + if not failovers or not isinstance(final_event, dict): + return False + last = failovers[-1] + switch_count = last.get("switchCount") + max_switches = last.get("maxSwitches") + final_status = final_event.get("statusCode") + return ( + isinstance(switch_count, int) + and isinstance(max_switches, int) + and max_switches > 0 + and switch_count >= max_switches + and isinstance(final_status, int) + and final_status >= 500 + ) + +def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot): + if not failover_budget_exhausted(failovers, final_event): + return [] + tried = set() + for item in failovers: + account_id = item.get("accountId") + if isinstance(account_id, int): + tried.add(account_id) + final_account = final_event.get("accountId") if isinstance(final_event, dict) else None + if isinstance(final_account, int): + tried.add(final_account) + result = [] + for item in account_snapshot: + account_id = item.get("accountId") + if not isinstance(account_id, int) or account_id in tried: + continue + if item.get("schedulable") is True and item.get("status") == "active" and item.get("tempUnschedulableSet") is not True: + result.append({ + "accountId": account_id, + "accountName": item.get("accountName"), + "priority": item.get("priority"), + "concurrency": item.get("concurrency"), + }) + return result + def run_trace(): payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} request_id = payload.get("requestId") @@ -6063,6 +6155,7 @@ def run_trace(): final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400] window_failovers = [item for item in window_events if item.get("type") == "failover"] window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] + untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) reason = trace_reason(events, final_event) if not matched: outcome = "not-found" @@ -6112,6 +6205,10 @@ def run_trace(): "tempUnschedulableCount": len(temp_unsched), "adminSchedulableCount": len(admin_sched), }, + "diagnostics": { + "failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}), + "untriedSchedulableAccounts": untried_schedulable_accounts, + }, "accountSnapshot": account_snapshot, "accountSnapshotError": account_snapshot_error, "rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [], From 687310d83c2d44d6c77f392af60df089b885ba5e Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 04:21:57 +0000 Subject: [PATCH 05/13] feat: add yaml-first AgentRun lane ops --- config/agentrun.yaml | 130 +++++++++ config/platform-db/postgres-pk01.yaml | 76 +++++ scripts/src/agentrun-lanes.ts | 393 ++++++++++++++++++++++++++ scripts/src/agentrun.ts | 333 +++++++++++++++++++++- 4 files changed, 928 insertions(+), 4 deletions(-) create mode 100644 scripts/src/agentrun-lanes.ts diff --git a/config/agentrun.yaml b/config/agentrun.yaml index a1990499..b910350b 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -28,3 +28,133 @@ auth: client: role: render-only transport: direct-http + +controlPlane: + default: + node: G14 + lane: v01 + + nodes: + G14: + route: G14 + kubeRoute: G14:k3s + D601: + route: D601 + kubeRoute: D601:k3s + + lanes: + v01: + node: G14 + version: v0.1 + source: + repository: pikasTech/agentrun + branch: v0.1 + remote: git@github.com:pikasTech/agentrun.git + workspace: /root/agentrun-v01 + runtime: + namespace: agentrun-v01 + managerDeployment: agentrun-mgr + managerService: agentrun-mgr + managerPort: 8080 + internalBaseUrl: http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080 + ci: + namespace: agentrun-ci + pipeline: agentrun-v01-ci-image-publish + pipelineRunPrefix: agentrun-v01-ci + serviceAccountName: agentrun-v01-tekton-runner + registryPrefix: 127.0.0.1:5000/agentrun + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + gitops: + branch: v0.1-gitops + path: deploy/gitops/g14/runtime-v01 + argoNamespace: argocd + argoApplication: agentrun-g14-v01 + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git + gitMirror: + namespace: devops-infra + readService: git-mirror-http + writeService: git-mirror-write + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/agentrun.git + cachePvc: git-mirror-cache + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + syncJobPrefix: git-mirror-agentrun-sync-manual + flushJobPrefix: git-mirror-agentrun-flush-manual + repositories: + - key: agentrun + repository: pikasTech/agentrun + sourceBranch: v0.1 + gitopsBranch: v0.1-gitops + - key: unidesk + repository: pikasTech/unidesk + sourceBranch: master + - key: agent_skills + repository: pikasTech/agent_skills + sourceBranch: master + database: + mode: local-postgres + secretRef: + name: agentrun-v01-mgr-db + key: DATABASE_URL + localPostgresExpectedAbsent: false + + v02: + node: D601 + version: v0.2 + source: + repository: pikasTech/agentrun + branch: v0.2 + remote: git@github.com:pikasTech/agentrun.git + workspace: /home/ubuntu/workspace/agentrun-v02 + runtime: + namespace: agentrun-v02 + managerDeployment: agentrun-mgr + managerService: agentrun-mgr + managerPort: 8080 + internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 + ci: + namespace: agentrun-ci + pipeline: agentrun-v02-ci-image-publish + pipelineRunPrefix: agentrun-v02-ci + serviceAccountName: agentrun-v02-tekton-runner + registryPrefix: 127.0.0.1:5000/agentrun + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + gitops: + branch: v0.2-gitops + path: deploy/gitops/node/d601/runtime-v02 + argoNamespace: argocd + argoApplication: agentrun-d601-v02 + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + gitMirror: + namespace: devops-infra + readService: git-mirror-http + writeService: git-mirror-write + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + cachePvc: hwlab-git-mirror-cache + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + syncJobPrefix: git-mirror-agentrun-d601-v02-sync-manual + flushJobPrefix: git-mirror-agentrun-d601-v02-flush-manual + repositories: + - key: agentrun + repository: pikasTech/agentrun + sourceBranch: v0.2 + gitopsBranch: v0.2-gitops + - key: unidesk + repository: pikasTech/unidesk + sourceBranch: master + - key: agent_skills + repository: pikasTech/agent_skills + sourceBranch: master + database: + mode: external-postgres + provider: PK01 + configRef: config/platform-db/postgres-pk01.yaml + database: agentrun_v02 + user: agentrun_v02 + sslmode: require + secretSourceRef: agentrun/d601-v02-mgr-db.env + secretRef: + name: agentrun-v02-mgr-db + key: DATABASE_URL + localPostgresExpectedAbsent: true diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index 8d22b3cc..5434c07f 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -200,6 +200,36 @@ postgres: user: n8n address: 202.98.17.68/32 method: scram-sha-256 + - type: hostssl + database: agentrun_v02 + user: agentrun_v02 + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: postgres + user: agentrun_v02 + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: agentrun_v02 + user: agentrun_v02 + address: 36.49.29.73/32 + method: scram-sha-256 + - type: hostssl + database: postgres + user: agentrun_v02 + address: 36.49.29.73/32 + method: scram-sha-256 + - type: hostssl + database: agentrun_v02 + user: agentrun_v02 + address: 74.48.78.17/32 + method: scram-sha-256 + - type: hostssl + database: postgres + user: agentrun_v02 + address: 74.48.78.17/32 + method: scram-sha-256 secrets: source: master-local @@ -247,6 +277,20 @@ secrets: N8N_DB_NAME: n8n randomHex: N8N_DB_PASSWORD: 32 + - name: agentrun-v02-db-credentials + sourceRef: platform-db/agentrun-v02-db.env + type: env + requiredKeys: + - AGENTRUN_V02_DB_USER + - AGENTRUN_V02_DB_PASSWORD + - AGENTRUN_V02_DB_NAME + createIfMissing: + enabled: true + values: + AGENTRUN_V02_DB_USER: agentrun_v02 + AGENTRUN_V02_DB_NAME: agentrun_v02 + randomHex: + AGENTRUN_V02_DB_PASSWORD: 32 objects: roles: @@ -277,6 +321,15 @@ objects: createdb: false createrole: false superuser: false + - name: agentrun_v02 + passwordRef: + sourceRef: platform-db/agentrun-v02-db.env + key: AGENTRUN_V02_DB_PASSWORD + login: true + attributes: + createdb: false + createrole: false + superuser: false databases: - name: sub2api owner: sub2api @@ -293,6 +346,11 @@ objects: encoding: UTF8 locale: C.UTF-8 extensions: [] + - name: agentrun_v02 + owner: agentrun_v02 + encoding: UTF8 + locale: C.UTF-8 + extensions: [] exports: connectionStrings: @@ -341,6 +399,21 @@ exports: - scope: platform-infra secret: n8n-secrets key: DATABASE_URL + - name: agentrun-v02-database-url + sourceSecretRef: platform-db/agentrun-v02-db.env + render: + envKey: DATABASE_URL + format: postgresql://$(AGENTRUN_V02_DB_USER):$(AGENTRUN_V02_DB_PASSWORD)@$(PGHOST):5432/$(AGENTRUN_V02_DB_NAME)?sslmode=require + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: agentrun/d601-v02-mgr-db.env + key: DATABASE_URL + mode: update-or-insert + consumers: + - scope: agentrun-v02 + secret: agentrun-v02-mgr-db + key: DATABASE_URL backup: phase: minimum-restoreable @@ -377,6 +450,9 @@ observability: - kind: psql-app-role database: n8n user: n8n + - kind: psql-app-role + database: agentrun_v02 + user: agentrun_v02 - kind: disk-free path: /var/lib/postgresql/16/main minFreeGiB: 10 diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts new file mode 100644 index 00000000..fd2ebfb4 --- /dev/null +++ b/scripts/src/agentrun-lanes.ts @@ -0,0 +1,393 @@ +import { readFileSync } from "node:fs"; +import { rootPath } from "./config"; + +export const AGENTRUN_CONFIG_PATH = "config/agentrun.yaml"; + +export interface AgentRunGitMirrorRepositorySpec { + readonly key: string; + readonly repository: string; + readonly sourceBranch: string; + readonly gitopsBranch?: string; +} + +export interface AgentRunLaneSpec { + readonly lane: string; + readonly nodeId: string; + readonly nodeRoute: string; + readonly nodeKubeRoute: string; + readonly version: string; + readonly source: { + readonly repository: string; + readonly branch: string; + readonly remote: string; + readonly workspace: string; + }; + readonly runtime: { + readonly namespace: string; + readonly managerDeployment: string; + readonly managerService: string; + readonly managerPort: number; + readonly internalBaseUrl: string; + }; + readonly ci: { + readonly namespace: string; + readonly pipeline: string; + readonly pipelineRunPrefix: string; + readonly serviceAccountName: string; + readonly registryPrefix: string; + readonly toolsImage: string; + }; + readonly gitops: { + readonly branch: string; + readonly path: string; + readonly argoNamespace: string; + readonly argoApplication: string; + readonly repoURL: string; + }; + readonly gitMirror: { + readonly namespace: string; + readonly readService: string; + readonly writeService: string; + readonly readUrl: string; + readonly writeUrl: string; + readonly cachePvc: string; + readonly toolsImage: string; + readonly syncJobPrefix: string; + readonly flushJobPrefix: string; + readonly repositories: readonly AgentRunGitMirrorRepositorySpec[]; + }; + readonly database: { + readonly mode: "local-postgres" | "external-postgres"; + readonly provider: string | null; + readonly configRef: string | null; + readonly database: string | null; + readonly user: string | null; + readonly sslmode: "require" | null; + readonly secretSourceRef: string | null; + readonly secretRef: { readonly name: string; readonly key: string }; + readonly localPostgresExpectedAbsent: boolean; + }; +} + +export interface AgentRunLaneTarget { + readonly configPath: string; + readonly spec: AgentRunLaneSpec; +} + +interface AgentRunNodeSpec { + readonly id: string; + readonly route: string; + readonly kubeRoute: string; +} + +interface AgentRunControlPlaneConfig { + readonly sourcePath: string; + readonly defaultTarget: { + readonly node: string; + readonly lane: string; + }; + readonly nodes: Record; + readonly lanes: Record; +} + +export function resolveAgentRunLaneTarget(options: { node?: string | null; lane?: string | null }, env: NodeJS.ProcessEnv = process.env): AgentRunLaneTarget { + const config = readAgentRunControlPlaneConfig(env); + const requestedNode = options.node ?? null; + const requestedLane = options.lane ?? null; + if (requestedLane !== null) { + const spec = config.lanes[requestedLane]; + if (spec === undefined) throw new Error(`${config.sourcePath}: controlPlane.lanes.${requestedLane} is not declared`); + if (requestedNode !== null && requestedNode !== spec.nodeId) throw new Error(`--node ${requestedNode} does not match controlPlane.lanes.${requestedLane}.node=${spec.nodeId}`); + return { configPath: config.sourcePath, spec }; + } + if (requestedNode !== null) { + const candidates = Object.values(config.lanes).filter((lane) => lane.nodeId === requestedNode); + if (candidates.length === 0) throw new Error(`${config.sourcePath}: no AgentRun lane is declared for node ${requestedNode}`); + if (candidates.length > 1) throw new Error(`--lane is required because node ${requestedNode} has multiple AgentRun lanes: ${candidates.map((lane) => lane.lane).join(", ")}`); + return { configPath: config.sourcePath, spec: candidates[0] }; + } + const spec = config.lanes[config.defaultTarget.lane]; + if (spec === undefined) throw new Error(`${config.sourcePath}: default controlPlane lane ${config.defaultTarget.lane} is not declared`); + if (spec.nodeId !== config.defaultTarget.node) throw new Error(`${config.sourcePath}: controlPlane.default.node does not match default lane node`); + return { configPath: config.sourcePath, spec }; +} + +export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record { + return { + node: { + id: spec.nodeId, + route: spec.nodeRoute, + kubeRoute: spec.nodeKubeRoute, + }, + lane: spec.lane, + version: spec.version, + source: { + repository: spec.source.repository, + branch: spec.source.branch, + workspace: spec.source.workspace, + }, + runtime: { + namespace: spec.runtime.namespace, + managerDeployment: spec.runtime.managerDeployment, + managerService: spec.runtime.managerService, + managerPort: spec.runtime.managerPort, + internalBaseUrl: spec.runtime.internalBaseUrl, + }, + ci: { + namespace: spec.ci.namespace, + pipeline: spec.ci.pipeline, + pipelineRunPrefix: spec.ci.pipelineRunPrefix, + serviceAccountName: spec.ci.serviceAccountName, + registryPrefix: spec.ci.registryPrefix, + }, + gitops: { + branch: spec.gitops.branch, + path: spec.gitops.path, + argoNamespace: spec.gitops.argoNamespace, + argoApplication: spec.gitops.argoApplication, + repoURL: spec.gitops.repoURL, + }, + gitMirror: { + namespace: spec.gitMirror.namespace, + readService: spec.gitMirror.readService, + writeService: spec.gitMirror.writeService, + readUrl: spec.gitMirror.readUrl, + writeUrl: spec.gitMirror.writeUrl, + cachePvc: spec.gitMirror.cachePvc, + repositories: spec.gitMirror.repositories.map((repo) => ({ + key: repo.key, + repository: repo.repository, + sourceBranch: repo.sourceBranch, + gitopsBranch: repo.gitopsBranch ?? null, + })), + }, + database: { + mode: spec.database.mode, + provider: spec.database.provider, + configRef: spec.database.configRef, + database: spec.database.database, + user: spec.database.user, + sslmode: spec.database.sslmode, + secretSourceRef: spec.database.secretSourceRef, + secretRef: spec.database.secretRef, + localPostgresExpectedAbsent: spec.database.localPostgresExpectedAbsent, + valuesPrinted: false, + }, + }; +} + +export function agentRunPipelineRunName(spec: AgentRunLaneSpec, sourceCommit: string): string { + return `${spec.ci.pipelineRunPrefix}-${sourceCommit.slice(0, 12)}`; +} + +function readAgentRunControlPlaneConfig(env: NodeJS.ProcessEnv): AgentRunControlPlaneConfig { + const configPath = env.AGENTRUN_CONTROL_PLANE_CONFIG ?? rootPath(AGENTRUN_CONFIG_PATH); + const root = asRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown, configPath); + const controlPlane = recordField(root, "controlPlane", configPath); + const defaultTarget = recordField(controlPlane, "default", `${configPath}.controlPlane`); + const nodes = parseNodes(recordField(controlPlane, "nodes", `${configPath}.controlPlane`), configPath); + const lanes = parseLanes(recordField(controlPlane, "lanes", `${configPath}.controlPlane`), nodes, configPath); + return { + sourcePath: configPath, + defaultTarget: { + node: stringField(defaultTarget, "node", `${configPath}.controlPlane.default`), + lane: stringField(defaultTarget, "lane", `${configPath}.controlPlane.default`), + }, + nodes, + lanes, + }; +} + +function parseNodes(input: Record, configPath: string): Record { + const result: Record = {}; + for (const [id, raw] of Object.entries(input)) { + validateSimpleId(id, `${configPath}.controlPlane.nodes`); + const node = asRecord(raw, `${configPath}.controlPlane.nodes.${id}`); + result[id] = { + id, + route: stringField(node, "route", `${configPath}.controlPlane.nodes.${id}`), + kubeRoute: stringField(node, "kubeRoute", `${configPath}.controlPlane.nodes.${id}`), + }; + } + if (Object.keys(result).length === 0) throw new Error(`${configPath}.controlPlane.nodes must declare at least one node`); + return result; +} + +function parseLanes(input: Record, nodes: Record, configPath: string): Record { + const result: Record = {}; + for (const [lane, raw] of Object.entries(input)) { + validateSimpleId(lane, `${configPath}.controlPlane.lanes`); + const item = asRecord(raw, `${configPath}.controlPlane.lanes.${lane}`); + const nodeId = stringField(item, "node", `${configPath}.controlPlane.lanes.${lane}`); + const node = nodes[nodeId]; + if (node === undefined) throw new Error(`${configPath}.controlPlane.lanes.${lane}.node references undeclared node ${nodeId}`); + result[lane] = parseLane(lane, node, item, configPath); + } + if (Object.keys(result).length === 0) throw new Error(`${configPath}.controlPlane.lanes must declare at least one lane`); + return result; +} + +function parseLane(lane: string, node: AgentRunNodeSpec, input: Record, configPath: string): AgentRunLaneSpec { + const path = `${configPath}.controlPlane.lanes.${lane}`; + const source = recordField(input, "source", path); + const runtime = recordField(input, "runtime", path); + const ci = recordField(input, "ci", path); + const gitops = recordField(input, "gitops", path); + const gitMirror = recordField(input, "gitMirror", path); + const database = recordField(input, "database", path); + return { + lane, + nodeId: node.id, + nodeRoute: node.route, + nodeKubeRoute: node.kubeRoute, + version: stringField(input, "version", path), + source: { + repository: stringField(source, "repository", `${path}.source`), + branch: stringField(source, "branch", `${path}.source`), + remote: stringField(source, "remote", `${path}.source`), + workspace: absolutePathField(source, "workspace", `${path}.source`), + }, + runtime: { + namespace: stringField(runtime, "namespace", `${path}.runtime`), + managerDeployment: stringField(runtime, "managerDeployment", `${path}.runtime`), + managerService: stringField(runtime, "managerService", `${path}.runtime`), + managerPort: integerField(runtime, "managerPort", `${path}.runtime`), + internalBaseUrl: urlField(runtime, "internalBaseUrl", `${path}.runtime`), + }, + ci: { + namespace: stringField(ci, "namespace", `${path}.ci`), + pipeline: stringField(ci, "pipeline", `${path}.ci`), + pipelineRunPrefix: stringField(ci, "pipelineRunPrefix", `${path}.ci`), + serviceAccountName: stringField(ci, "serviceAccountName", `${path}.ci`), + registryPrefix: stringField(ci, "registryPrefix", `${path}.ci`), + toolsImage: stringField(ci, "toolsImage", `${path}.ci`), + }, + gitops: { + branch: stringField(gitops, "branch", `${path}.gitops`), + path: relativePathField(gitops, "path", `${path}.gitops`), + argoNamespace: stringField(gitops, "argoNamespace", `${path}.gitops`), + argoApplication: stringField(gitops, "argoApplication", `${path}.gitops`), + repoURL: urlField(gitops, "repoURL", `${path}.gitops`), + }, + gitMirror: { + namespace: stringField(gitMirror, "namespace", `${path}.gitMirror`), + readService: stringField(gitMirror, "readService", `${path}.gitMirror`), + writeService: stringField(gitMirror, "writeService", `${path}.gitMirror`), + readUrl: urlField(gitMirror, "readUrl", `${path}.gitMirror`), + writeUrl: urlField(gitMirror, "writeUrl", `${path}.gitMirror`), + cachePvc: stringField(gitMirror, "cachePvc", `${path}.gitMirror`), + toolsImage: stringField(gitMirror, "toolsImage", `${path}.gitMirror`), + syncJobPrefix: stringField(gitMirror, "syncJobPrefix", `${path}.gitMirror`), + flushJobPrefix: stringField(gitMirror, "flushJobPrefix", `${path}.gitMirror`), + repositories: arrayField(gitMirror, "repositories", `${path}.gitMirror`).map((repo, index) => parseGitMirrorRepository(repo, `${path}.gitMirror.repositories[${index}]`)), + }, + database: parseDatabase(database, `${path}.database`), + }; +} + +function parseGitMirrorRepository(input: Record, path: string): AgentRunGitMirrorRepositorySpec { + const gitopsBranch = optionalStringField(input, "gitopsBranch", path); + return { + key: stringField(input, "key", path), + repository: stringField(input, "repository", path), + sourceBranch: stringField(input, "sourceBranch", path), + ...(gitopsBranch === undefined ? {} : { gitopsBranch }), + }; +} + +function parseDatabase(input: Record, path: string): AgentRunLaneSpec["database"] { + const mode = enumField(input, "mode", path, ["local-postgres", "external-postgres"]); + const sslmode = optionalStringField(input, "sslmode", path); + if (sslmode !== undefined && sslmode !== "require") throw new Error(`${path}.sslmode must be require when set`); + return { + mode, + provider: optionalStringField(input, "provider", path) ?? null, + configRef: optionalStringField(input, "configRef", path) ?? null, + database: optionalStringField(input, "database", path) ?? null, + user: optionalStringField(input, "user", path) ?? null, + sslmode: sslmode === undefined ? null : "require", + secretSourceRef: optionalStringField(input, "secretSourceRef", path) ?? null, + secretRef: parseSecretRef(recordField(input, "secretRef", path), `${path}.secretRef`), + localPostgresExpectedAbsent: booleanField(input, "localPostgresExpectedAbsent", path), + }; +} + +function parseSecretRef(input: Record, path: string): { name: string; key: string } { + return { + name: stringField(input, "name", path), + key: stringField(input, "key", path), + }; +} + +function asRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be a YAML object`); + return value as Record; +} + +function recordField(obj: Record, key: string, path: string): Record { + return asRecord(obj[key], `${path}.${key}`); +} + +function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value.trim(); +} + +function optionalStringField(obj: Record, key: string, path: string): string | undefined { + const value = obj[key]; + if (value === undefined || value === null) return undefined; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string when set`); + return value.trim(); +} + +function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +function integerField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (!Number.isInteger(value)) throw new Error(`${path}.${key} must be an integer`); + return Number(value); +} + +function arrayField(obj: Record, key: string, path: string): Record[] { + const value = obj[key]; + if (!Array.isArray(value)) throw new Error(`${path}.${key} must be a YAML array`); + return value.map((item, index) => asRecord(item, `${path}.${key}[${index}]`)); +} + +function enumField(obj: Record, key: string, path: string, values: readonly T[]): T { + const value = stringField(obj, key, path); + if (!values.includes(value as T)) throw new Error(`${path}.${key} must be one of ${values.join(", ")}`); + return value as T; +} + +function absolutePathField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!value.startsWith("/") || value.includes("..")) throw new Error(`${path}.${key} must be an absolute path without ..`); + return value; +} + +function relativePathField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (value.startsWith("/") || value.includes("..")) throw new Error(`${path}.${key} must be a relative path without ..`); + return value; +} + +function urlField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + try { + const parsed = new URL(value); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("unsupported protocol"); + } catch { + throw new Error(`${path}.${key} must be an http(s) URL`); + } + return value; +} + +function validateSimpleId(value: string, path: string): void { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path}.${value} must use a simple id`); +} diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index b2db4a0e..d3bc5909 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -5,6 +5,13 @@ import type { RenderedCliResult } from "./output"; import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; import { runRemoteSshCommandCapture } from "./remote"; import { startJob } from "./jobs"; +import { + AGENTRUN_CONFIG_PATH, + agentRunLaneSummary, + agentRunPipelineRunName, + resolveAgentRunLaneTarget, + type AgentRunLaneSpec, +} from "./agentrun-lanes"; const g14SourceRoute = "G14:/root/agentrun-v01"; const g14K3sRoute = "G14:k3s"; @@ -56,6 +63,8 @@ export function agentRunHelp(): unknown { "bun scripts/cli.ts agentrun apply -f - --dry-run", "bun scripts/cli.ts agentrun send session/ --aipod Artificer --prompt-stdin", "bun scripts/cli.ts agentrun explain task", + "bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", + "bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", "bun scripts/cli.ts agentrun control-plane status", "bun scripts/cli.ts agentrun control-plane status --full", "bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", @@ -92,6 +101,7 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str } if (config === null) throw new Error("agentrun control-plane and git-mirror commands require UniDesk config"); if (group === "control-plane") { + if (action === "plan") return await controlPlanePlan(config, parseStatusOptions(actionArgs)); if (action === "status") return await status(config, parseStatusOptions(actionArgs)); if (action === "expose") return await exposeAgentRun(config, parseConfirmOptions(actionArgs)); if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(actionArgs)); @@ -217,8 +227,10 @@ function agentRunHelpText(args: string[]): string { return [ "Usage: bun scripts/cli.ts agentrun control-plane [options]", "", - "Actions: status, expose, trigger-current, refresh, cleanup-runs, cleanup-released-pvs", + "Actions: plan, status, expose, trigger-current, refresh, cleanup-runs, cleanup-released-pvs", "Examples:", + " bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", + " bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", " bun scripts/cli.ts agentrun control-plane status", " bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", " bun scripts/cli.ts agentrun control-plane expose --dry-run", @@ -1645,6 +1657,8 @@ interface DisclosureOptions { } interface StatusOptions extends DisclosureOptions { + node: string | null; + lane: string | null; sourceCommit: string | null; pipelineRun: string | null; targetMode: "latest-source-head" | "source-commit" | "pipeline-run"; @@ -1656,7 +1670,14 @@ interface TimedValue { } function parseDisclosureOptions(args: string[]): DisclosureOptions { - for (const arg of args) { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--node" || arg === "--lane") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + index += 1; + continue; + } if (arg !== "--full" && arg !== "--raw") throw new Error(`unsupported status option: ${arg}`); } const raw = args.includes("--raw"); @@ -1664,6 +1685,8 @@ function parseDisclosureOptions(args: string[]): DisclosureOptions { } function parseStatusOptions(args: string[]): StatusOptions { + let node: string | null = null; + let lane: string | null = null; let sourceCommit: string | null = null; let pipelineRun: string | null = null; let full = false; @@ -1679,6 +1702,20 @@ function parseStatusOptions(args: string[]): StatusOptions { full = true; continue; } + if (arg === "--node") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); + node = value; + index += 1; + continue; + } + if (arg === "--lane") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); + lane = value; + index += 1; + continue; + } if (arg === "--source-commit") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--source-commit requires a value"); @@ -1690,7 +1727,7 @@ function parseStatusOptions(args: string[]): StatusOptions { if (arg === "--pipeline-run") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--pipeline-run requires a value"); - if (!isAgentRunPipelineRunName(value)) throw new Error("--pipeline-run must be an agentrun-v01-ci-<12+ hex> PipelineRun name"); + if (!isAgentRunPipelineRunName(value)) throw new Error("--pipeline-run must be an agentrun-vNN-ci-<12+ hex> PipelineRun name"); pipelineRun = value; index += 1; continue; @@ -1701,6 +1738,8 @@ function parseStatusOptions(args: string[]): StatusOptions { return { full, raw, + node, + lane, sourceCommit, pipelineRun, targetMode: pipelineRun !== null ? "pipeline-run" : sourceCommit !== null ? "source-commit" : "latest-source-head", @@ -1779,7 +1818,42 @@ function positiveIntegerOption(args: string[], name: string, defaultValue: numbe return Math.min(value, maxValue); } +async function controlPlanePlan(_config: UniDeskConfig, options: StatusOptions): Promise> { + const target = resolveAgentRunLaneTarget(options); + const spec = target.spec; + return { + ok: true, + command: "agentrun control-plane plan", + mode: "yaml-declared-node-lane", + configPath: target.configPath, + target: agentRunLaneSummary(spec), + plannedChecks: [ + "source-branch-exists", + "source-worktree-exists-and-clean", + "git-mirror-services-ready", + "ci-namespace-pipeline-serviceaccount", + "argo-application-alignment", + "runtime-namespace-manager-service", + "database-secretref-present", + "local-postgres-absent-when-external", + ], + deploymentBoundary: { + mutation: false, + note: "plan/status are read-only. Long writes must use controlled AgentRun/Platform DB commands and this YAML target.", + }, + next: { + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, + postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, + }, + valuesPrinted: false, + }; +} + async function status(config: UniDeskConfig, options: StatusOptions): Promise> { + if (options.node !== null || options.lane !== null) { + return await statusYamlLane(config, options, resolveAgentRunLaneTarget(options)); + } const sourceProbe = await timedStatusStage("source", () => capture(config, g14SourceRoute, ["script", "--", [ "cd /root/agentrun-v01", "git fetch origin v0.1 >/dev/null 2>&1 || true", @@ -1930,6 +2004,103 @@ async function status(config: UniDeskConfig, options: StatusOptions): Promise> { + const spec = target.spec; + const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["script", "--", yamlLaneSourceStatusScript(spec)])); + const sourcePayload = captureJsonPayload(sourceProbe.value); + const sourceCommit = options.sourceCommit + ?? (options.pipelineRun !== null ? null : stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead)); + const pipelineRun = options.pipelineRun ?? (sourceCommit ? agentRunPipelineRunName(spec, sourceCommit) : null); + const [runtimeProbe, mirrorProbe] = await Promise.all([ + timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneRuntimeStatusScript(spec, pipelineRun)])), + timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneGitMirrorStatusScript(spec)])), + ]); + const runtimePayload = captureJsonPayload(runtimeProbe.value); + const mirrorPayload = captureJsonPayload(mirrorProbe.value); + const pipeline = record(runtimePayload.pipeline); + const argo = record(runtimePayload.argo); + const manager = record(runtimePayload.manager); + const database = record(runtimePayload.database); + const localPostgres = record(runtimePayload.localPostgres); + const blockers = [ + ...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]), + ...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]), + ...(sourcePayload.workspaceClean === true || sourcePayload.workspaceExists !== true ? [] : ["source-worktree-dirty"]), + ...(mirrorPayload.readReady === true ? [] : ["git-mirror-read-not-ready"]), + ...(mirrorPayload.writeReady === true ? [] : ["git-mirror-write-not-ready"]), + ...(mirrorPayload.cachePvcExists === true ? [] : ["git-mirror-cache-pvc-missing"]), + ...(runtimePayload.ciNamespaceExists === true ? [] : ["ci-namespace-missing"]), + ...(pipeline.exists === true ? [] : ["pipeline-missing"]), + ...(runtimePayload.serviceAccountExists === true ? [] : ["ci-serviceaccount-missing"]), + ...(argo.exists === true ? [] : ["argo-application-missing"]), + ...(runtimePayload.runtimeNamespaceExists === true ? [] : ["runtime-namespace-missing"]), + ...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]), + ...(manager.serviceExists === true ? [] : ["manager-service-missing"]), + ...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []), + ...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []), + ]; + return { + ok: sourceProbe.value.exitCode === 0 && runtimeProbe.value.exitCode === 0 && mirrorProbe.value.exitCode === 0 && blockers.length === 0, + command: "agentrun control-plane status", + mode: "yaml-declared-node-lane", + configPath: target.configPath, + target: agentRunLaneSummary(spec), + summary: { + aligned: blockers.length === 0, + blockers, + sourceCommit, + expectedPipelineRun: pipelineRun, + source: { + workspaceExists: sourcePayload.workspaceExists ?? false, + workspaceClean: sourcePayload.workspaceClean ?? null, + localHead: sourcePayload.localHead ?? null, + remoteBranchExists: sourcePayload.remoteBranchExists ?? false, + remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, + }, + gitMirror: { + readReady: mirrorPayload.readReady ?? false, + writeReady: mirrorPayload.writeReady ?? false, + cachePvcExists: mirrorPayload.cachePvcExists ?? false, + repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null, + }, + ci: { + namespaceExists: runtimePayload.ciNamespaceExists ?? false, + serviceAccountExists: runtimePayload.serviceAccountExists ?? false, + pipeline, + pipelineRun: record(runtimePayload.pipelineRun), + }, + argo, + runtime: { + namespaceExists: runtimePayload.runtimeNamespaceExists ?? false, + manager, + database, + localPostgres, + }, + }, + timings: { + sourceMs: sourceProbe.elapsedMs, + runtimeMs: runtimeProbe.elapsedMs, + gitMirrorMs: mirrorProbe.elapsedMs, + totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs), + }, + source: sourcePayload, + runtime: runtimePayload, + gitMirror: mirrorPayload, + captures: { + source: compactCapture(sourceProbe.value, { full: options.full || options.raw || sourceProbe.value.exitCode !== 0 }), + runtime: compactCapture(runtimeProbe.value, { full: options.full || options.raw || runtimeProbe.value.exitCode !== 0 }), + gitMirror: compactCapture(mirrorProbe.value, { full: options.full || options.raw || mirrorProbe.value.exitCode !== 0 }), + }, + next: { + plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`, + postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, + postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, + statusFull: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --full`, + }, + valuesPrinted: false, + }; +} + async function exposeAgentRun(_config: UniDeskConfig, options: ConfirmOptions): Promise> { const clientConfig = readAgentRunClientConfig(); const exposure = clientConfig.publicExposure; @@ -2406,6 +2577,160 @@ function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions): string { ].join("\n"); } +function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string { + return [ + "set +e", + `expected_workspace=${shQuote(spec.source.workspace)}`, + `source_branch=${shQuote(spec.source.branch)}`, + "workspace_exists=false", + "workspace_clean=null", + "local_head=null", + "branch=null", + "remote_url=null", + "remote_branch_exists=false", + "remote_branch_commit=null", + "status_short=''", + "if [ -d .git ] || git rev-parse --show-toplevel >/dev/null 2>&1; then", + " actual_workspace=$(pwd)", + " workspace_exists=true", + " branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)", + " remote_url=$(git remote get-url origin 2>/dev/null || true)", + " local_head=$(git rev-parse HEAD 2>/dev/null || true)", + " status_short=$(git status --short 2>/dev/null || true)", + " if [ -z \"$status_short\" ]; then workspace_clean=true; else workspace_clean=false; fi", + " git fetch origin \"$source_branch\" >/dev/null 2>&1 || true", + " remote_branch_commit=$(git rev-parse \"refs/remotes/origin/$source_branch\" 2>/dev/null || true)", + " if [ -n \"$remote_branch_commit\" ]; then remote_branch_exists=true; fi", + "else", + " actual_workspace=$(pwd)", + "fi", + "export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace", + "node <<'NODE'", + "function nullable(value) { return value && value !== 'null' ? value : null; }", + "function booleanValue(value) { if (value === 'true') return true; if (value === 'false') return false; return null; }", + "const env = process.env;", + "console.log(JSON.stringify({", + " ok: env.workspace_exists === 'true',", + " expectedWorkspace: env.expected_workspace,", + " actualWorkspace: env.actual_workspace || null,", + " workspaceExists: env.workspace_exists === 'true',", + " workspaceClean: booleanValue(env.workspace_clean),", + " branch: nullable(env.branch),", + " remoteUrl: nullable(env.remote_url),", + " localHead: nullable(env.local_head),", + " remoteBranch: env.source_branch,", + " remoteBranchExists: env.remote_branch_exists === 'true',", + " remoteBranchCommit: nullable(env.remote_branch_commit),", + " statusShort: nullable(env.status_short),", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); +} + +function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string | null): string { + return [ + "set +e", + `runtime_namespace=${shQuote(spec.runtime.namespace)}`, + `ci_namespace=${shQuote(spec.ci.namespace)}`, + `pipeline_name=${shQuote(spec.ci.pipeline)}`, + `pipeline_run=${pipelineRun === null ? "''" : shQuote(pipelineRun)}`, + `service_account=${shQuote(spec.ci.serviceAccountName)}`, + `argo_namespace=${shQuote(spec.gitops.argoNamespace)}`, + `argo_application=${shQuote(spec.gitops.argoApplication)}`, + `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, + `manager_service=${shQuote(spec.runtime.managerService)}`, + `database_secret=${shQuote(spec.database.secretRef.name)}`, + `database_key=${shQuote(spec.database.secretRef.key)}`, + "export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key", + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null", + "runtime_ns_exit=$?", + "kubectl get ns \"$ci_namespace\" -o json > \"$tmp_dir/ci-ns.json\" 2>/dev/null", + "ci_ns_exit=$?", + "kubectl -n \"$ci_namespace\" get pipeline \"$pipeline_name\" -o json > \"$tmp_dir/pipeline.json\" 2>/dev/null", + "pipeline_exit=$?", + "kubectl -n \"$ci_namespace\" get serviceaccount \"$service_account\" -o json > \"$tmp_dir/serviceaccount.json\" 2>/dev/null", + "sa_exit=$?", + "if [ -n \"$pipeline_run\" ]; then kubectl -n \"$ci_namespace\" get pipelinerun \"$pipeline_run\" -o json > \"$tmp_dir/pipelinerun.json\" 2>/dev/null; pr_exit=$?; else pr_exit=2; fi", + "kubectl -n \"$argo_namespace\" get application \"$argo_application\" -o json > \"$tmp_dir/argo.json\" 2>/dev/null", + "argo_exit=$?", + "kubectl -n \"$runtime_namespace\" get deploy \"$manager_deployment\" -o json > \"$tmp_dir/manager-deploy.json\" 2>/dev/null", + "manager_deploy_exit=$?", + "kubectl -n \"$runtime_namespace\" get svc \"$manager_service\" -o json > \"$tmp_dir/manager-svc.json\" 2>/dev/null", + "manager_svc_exit=$?", + "kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null", + "db_secret_exit=$?", + "kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null", + "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'", + "const fs = require('node:fs');", + "const path = require('node:path');", + "const dir = process.env.NODE_TMP;", + "function readJson(name) { try { return JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); } catch { return null; } }", + "function exists(exitName) { return process.env[exitName] === '0'; }", + "function condition(obj) { return obj?.status?.conditions?.[0] || null; }", + "function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }", + "const pipelineRun = readJson('pipelinerun.json');", + "const argo = readJson('argo.json');", + "const managerDeploy = readJson('manager-deploy.json');", + "const managerSvc = readJson('manager-svc.json');", + "const dbSecret = readJson('db-secret.json');", + "let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}", + "const c = condition(pipelineRun);", + "console.log(JSON.stringify({", + " ok: exists('RUNTIME_NS_EXIT') && exists('CI_NS_EXIT'),", + " runtimeNamespaceExists: exists('RUNTIME_NS_EXIT'),", + " ciNamespaceExists: exists('CI_NS_EXIT'),", + " serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),", + " pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },", + " pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },", + " argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },", + " manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },", + " database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },", + " localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); +} + +function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { + return [ + "set +e", + `namespace=${shQuote(spec.gitMirror.namespace)}`, + `read_service=${shQuote(spec.gitMirror.readService)}`, + `write_service=${shQuote(spec.gitMirror.writeService)}`, + `cache_pvc=${shQuote(spec.gitMirror.cachePvc)}`, + `repositories_json=${shQuote(JSON.stringify(spec.gitMirror.repositories))}`, + "kubectl -n \"$namespace\" get svc \"$read_service\" -o json >/tmp/agentrun-gitmirror-read.json 2>/dev/null", + "read_exit=$?", + "kubectl -n \"$namespace\" get svc \"$write_service\" -o json >/tmp/agentrun-gitmirror-write.json 2>/dev/null", + "write_exit=$?", + "kubectl -n \"$namespace\" get pvc \"$cache_pvc\" -o json >/tmp/agentrun-gitmirror-cache.json 2>/dev/null", + "cache_exit=$?", + "kubectl -n \"$namespace\" get deploy,svc,pvc -o name > /tmp/agentrun-gitmirror-names.txt 2>/dev/null", + "NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" REPOSITORIES_JSON=\"$repositories_json\" node <<'NODE'", + "const fs = require('node:fs');", + "const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');", + "let names = ''; try { names = fs.readFileSync('/tmp/agentrun-gitmirror-names.txt', 'utf8'); } catch {}", + "const readReady = process.env.READ_EXIT === '0';", + "const writeReady = process.env.WRITE_EXIT === '0';", + "const cachePvcExists = process.env.CACHE_EXIT === '0';", + "console.log(JSON.stringify({", + " ok: readReady && writeReady && cachePvcExists,", + " namespace: process.env.NAMESPACE,", + " readReady,", + " writeReady,", + " cachePvcExists,", + " resources: names.split(/\\r?\\n/).filter(Boolean).slice(0, 40),", + " repositories,", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); +} + function cleanupRunsPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); @@ -4429,7 +4754,7 @@ function pipelineRunName(sourceCommit: string): string { } function isAgentRunPipelineRunName(value: string): boolean { - return /^agentrun-v01-ci-[0-9a-f]{12,40}(?:-[a-z0-9-]+)?$/u.test(value); + return /^agentrun-v[0-9]+-ci-[0-9a-f]{12,40}(?:-[a-z0-9-]+)?$/u.test(value); } function statusTargetArg(options: StatusOptions, target: Record): string { From 90fae0c1c46c95a920f0da0a91cabd9b37bc0992 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 04:32:07 +0000 Subject: [PATCH 06/13] feat: sync AgentRun lane database secret --- scripts/src/agentrun.ts | 170 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 1 deletion(-) diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index d3bc5909..ddad76e7 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -1,4 +1,6 @@ +import { createHash } from "node:crypto"; import { chmodSync, copyFileSync, existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { rootPath, type UniDeskConfig } from "./config"; import type { RenderedCliResult } from "./output"; @@ -65,6 +67,8 @@ export function agentRunHelp(): unknown { "bun scripts/cli.ts agentrun explain task", "bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", "bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", + "bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", + "bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane status", "bun scripts/cli.ts agentrun control-plane status --full", "bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", @@ -103,6 +107,7 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str if (group === "control-plane") { if (action === "plan") return await controlPlanePlan(config, parseStatusOptions(actionArgs)); if (action === "status") return await status(config, parseStatusOptions(actionArgs)); + if (action === "secret-sync") return await secretSync(config, parseSecretSyncOptions(actionArgs)); if (action === "expose") return await exposeAgentRun(config, parseConfirmOptions(actionArgs)); if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(actionArgs)); if (action === "refresh") return await refresh(config, parseConfirmOptions(actionArgs)); @@ -227,10 +232,11 @@ function agentRunHelpText(args: string[]): string { return [ "Usage: bun scripts/cli.ts agentrun control-plane [options]", "", - "Actions: plan, status, expose, trigger-current, refresh, cleanup-runs, cleanup-released-pvs", + "Actions: plan, status, secret-sync, expose, trigger-current, refresh, cleanup-runs, cleanup-released-pvs", "Examples:", " bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", " bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", + " bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane status", " bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", " bun scripts/cli.ts agentrun control-plane expose --dry-run", @@ -1635,6 +1641,11 @@ interface ConfirmOptions { dryRun: boolean; } +interface SecretSyncOptions extends ConfirmOptions { + node: string | null; + lane: string | null; +} + interface GitMirrorOptions extends ConfirmOptions { timeoutSeconds: number; wait: boolean; @@ -1751,6 +1762,32 @@ function parseTriggerOptions(args: string[]): TriggerOptions { return parseConfirmOptions(args); } +function parseSecretSyncOptions(args: string[]): SecretSyncOptions { + const base = parseConfirmOptions(args); + let node: string | null = null; + let lane: string | null = null; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--confirm" || arg === "--dry-run") continue; + if (arg === "--node") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); + node = value; + index += 1; + continue; + } + if (arg === "--lane") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); + lane = value; + index += 1; + continue; + } + throw new Error(`unsupported secret-sync option: ${arg}`); + } + return { ...base, node, lane }; +} + function parseConfirmOptions(args: string[]): ConfirmOptions { if (args.includes("--confirm") && args.includes("--dry-run")) throw new Error("accepts only one of --confirm or --dry-run"); return { @@ -2101,6 +2138,65 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar }; } +async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Promise> { + const { configPath, spec } = resolveAgentRunLaneTarget(options); + if (spec.database.secretSourceRef === null) { + return { + ok: false, + command: "agentrun control-plane secret-sync", + mode: "yaml-declared-node-lane", + configPath, + target: agentRunLaneSummary(spec), + degradedReason: "database-secret-source-not-declared", + valuesPrinted: false, + }; + } + const source = readSecretSourceValue(spec, spec.database.secretSourceRef, spec.database.secretRef.key); + const plan = { + namespace: spec.runtime.namespace, + secret: spec.database.secretRef.name, + key: spec.database.secretRef.key, + sourceRef: spec.database.secretSourceRef, + sourcePath: source.redactedPath, + fingerprint: source.fingerprint, + valueBytes: source.valueBytes, + valuesPrinted: false, + }; + if (options.dryRun || !options.confirm) { + return { + ok: true, + command: "agentrun control-plane secret-sync", + mode: "dry-run", + mutation: false, + configPath, + target: agentRunLaneSummary(spec), + plan, + next: { + confirm: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; + } + const result = await capture(config, spec.nodeKubeRoute, ["script", "--", secretSyncScript(spec, source.value)]); + const payload = captureJsonPayload(result); + return { + ok: result.exitCode === 0 && payload.ok !== false, + command: "agentrun control-plane secret-sync", + mode: "confirmed-sync", + mutation: true, + configPath, + target: agentRunLaneSummary(spec), + plan, + result: payload, + capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), + next: { + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; +} + async function exposeAgentRun(_config: UniDeskConfig, options: ConfirmOptions): Promise> { const clientConfig = readAgentRunClientConfig(); const exposure = clientConfig.publicExposure; @@ -2731,6 +2827,78 @@ function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { ].join("\n"); } +function readSecretSourceValue(spec: AgentRunLaneSpec, sourceRef: string, key: string): { redactedPath: string; value: string; valueBytes: number; fingerprint: string } { + if (sourceRef.startsWith("/") || sourceRef.includes("..")) throw new Error(`secret sourceRef must be relative without ..: ${sourceRef}`); + const secretRoot = resolveSecretSourceRoot(spec); + const sourcePath = join(secretRoot, ...sourceRef.split("/")); + if (!existsSync(sourcePath)) throw new Error(`secret source ${sourceRef} is missing; run platform-db postgres apply --config config/platform-db/postgres-pk01.yaml --confirm first`); + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const value = values.get(key); + if (value === undefined || value.length === 0) throw new Error(`secret source ${sourceRef} is missing required key ${key}`); + return { + redactedPath: `.state/secrets/${sourceRef}`, + value, + valueBytes: Buffer.byteLength(value, "utf8"), + fingerprint: `sha256:${createHash("sha256").update(value).digest("hex")}`, + }; +} + +function resolveSecretSourceRoot(spec: AgentRunLaneSpec): string { + if (spec.database.configRef === null) return rootPath(".state", "secrets"); + const configPath = spec.database.configRef.startsWith("/") ? spec.database.configRef : rootPath(spec.database.configRef); + const configRoot = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); + const secrets = record(configRoot.secrets); + const root = stringOrNull(secrets.root); + if (root === null || !root.startsWith("/") || root.includes("..")) throw new Error(`${spec.database.configRef}.secrets.root must be an absolute path without ..`); + return root; +} + +function parseEnvFile(text: string): Map { + const result = new Map(); + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const index = line.indexOf("="); + if (index <= 0) continue; + const key = line.slice(0, index).trim(); + let value = line.slice(index + 1).trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + result.set(key, value); + } + return result; +} + +function secretSyncScript(spec: AgentRunLaneSpec, value: string): string { + const encoded = Buffer.from(value, "utf8").toString("base64"); + return [ + "set -eu", + `namespace=${shQuote(spec.runtime.namespace)}`, + `secret_name=${shQuote(spec.database.secretRef.name)}`, + `secret_key=${shQuote(spec.database.secretRef.key)}`, + `secret_value_b64=${shQuote(encoded)}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "secret_file=\"$tmp_dir/secret-value\"", + "printf '%s' \"$secret_value_b64\" | base64 -d > \"$secret_file\"", + "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply --server-side --field-manager=unidesk-agentrun-secret-sync -f - >/dev/null", + "kubectl -n \"$namespace\" create secret generic \"$secret_name\" --from-file=\"$secret_key=$secret_file\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-agentrun-secret-sync -f - >/dev/null", + "rm -f \"$secret_file\"", + "kubectl -n \"$namespace\" get secret \"$secret_name\" -o json > \"$tmp_dir/secret.json\"", + "NAMESPACE=\"$namespace\" SECRET_NAME=\"$secret_name\" SECRET_KEY=\"$secret_key\" SECRET_JSON=\"$tmp_dir/secret.json\" node <<'NODE'", + "const fs = require('node:fs');", + "const crypto = require('node:crypto');", + "const secret = JSON.parse(fs.readFileSync(process.env.SECRET_JSON, 'utf8'));", + "const data = secret.data || {};", + "const raw = data[process.env.SECRET_KEY] || '';", + "const bytes = raw ? Buffer.from(raw, 'base64').length : 0;", + "const fingerprint = raw ? 'sha256:' + crypto.createHash('sha256').update(Buffer.from(raw, 'base64')).digest('hex') : null;", + "console.log(JSON.stringify({ ok: Boolean(raw), namespace: process.env.NAMESPACE, secret: process.env.SECRET_NAME, key: process.env.SECRET_KEY, valueBytes: bytes, fingerprint, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + function cleanupRunsPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); From d99f57cf83444b81f548531cd1ff451b7a63eb90 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 05:05:31 +0000 Subject: [PATCH 07/13] fix: protect primary sub2api codex account --- .agents/skills/unidesk-sub2api/SKILL.md | 5 +- config/platform-infra/sub2api-codex-pool.yaml | 6 + .../platform-infra-sub2api-codex-sentinel.ts | 156 +++++++++++++++++- scripts/src/platform-infra-sub2api-codex.ts | 127 +++++++++++++- 4 files changed, 289 insertions(+), 5 deletions(-) diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 60b91678..3dfd5cad 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -105,6 +105,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 - `profiles.entries[].capacity`: 可选 per-account concurrency override；不写则使用 `pool.defaultAccountCapacity`。具体数值只以 `config/platform-infra/sub2api-codex-pool.yaml` 为准，skill 和长期参考只描述规则，不重复写当前值。 - `profiles.entries[].loadFactor`: 可选 per-account Sub2API `load_factor` override；不写则使用 `pool.defaultAccountLoadFactor`。具体数值只以 `config/platform-infra/sub2api-codex-pool.yaml` 为准，修改后必须 `codex-pool sync --confirm` 和 `codex-pool validate`。 - `profiles.entries[].trustUpstream`: 可选账号级哨兵信任标记；默认 `false`。可信账号使用 `sentinel.cadence.trustedSuccessMaxIntervalMinutes` 作为连续成功后的最大探测退避，不可信账号使用 `sentinel.cadence.untrustedSuccessMaxIntervalMinutes`。它只影响哨兵探测频率和状态可见性，不改变 Sub2API account priority/capacity/loadFactor。 +- `profiles.entries[].sentinelProtect`: 可选账号级哨兵保护策略；默认关闭，只给需要避免偶发 marker 波动误屏蔽的高价值账号启用。启用时必须显式声明 `consecutiveFailures`、`initialRetryDelaySeconds`、`maxRetryDelaySeconds` 和 `backoffMultiplier`；marker probe 或 gateway failure 触发冻结前会先按该策略做连续 marker 确认，只有全部失败才进入冻结状态机。它只影响哨兵冻结判定和 `sentinel-report` 可见性，不改变 Sub2API account priority/capacity/loadFactor。 - 除非用户明确要求修改配置，不要仅凭推断改账号 membership、priority、capacity、loadFactor、WebSocket mode 或其他调度策略；先保留 YAML，完成 provenance/runtime evidence 溯源，并把结论写回相关 issue 或 runbook 后再提出变更。 - Sub2API 是 UniDesk 可读源码和可观测运行面的受控组件；排查 Sub2API 调度、failover、错误传播、临时不可调度或 account selection 时，默认先读当前 Sub2API 源码实现，再用真实 request id、Sub2API 日志和原入口流量验证。不要用 mock upstream、临时 probe account 或测试桩作为默认结论来源；这类探针最多是显式 debug 辅助，不能替代源码链路和真实运行证据。 - `profiles.entries[].tempUnschedulable`: 可选 per-account Sub2API 内置临时不可调度覆盖；只用于明确偏离 pool 默认规则，不用它给某个账号特殊优先级或临时绕过通用 failover。 @@ -127,7 +128,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 `trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。默认输出请求开始/最终状态、failover、`account_select_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 -`sentinel-report` 是只读低噪声报表，不触发 probe、不修改账号。默认输出类似 `ps` 的文本表，展示每个账号的探测次数、最近 marker/HTTP/动作、冻结 TTL、成功退避、下一次 probe 和最近 run 事件；需要机器处理时使用 `sentinel-report --raw`。 +`sentinel-report` 是只读低噪声报表，不触发 probe、不修改账号。默认输出类似 `ps` 的文本表，展示每个账号的探测次数、最近 marker/HTTP/动作、冻结 TTL、成功退避、下一次 probe 和最近 run 事件；`PROT` 展示账号级保护阈值，`P_FAIL` 展示最近一次保护确认中的失败次数/阈值；需要机器处理时使用 `sentinel-report --raw`。 `sync --confirm` 和 `validate` 可能超过单次 SSH/runtime 短连接窗口。必须继续使用 `bun scripts/cli.ts platform-infra sub2api codex-pool ...`，由 CLI 在 G14 远端提交作业并短轮询状态；不要改用裸 `trans G14:k3s script` 等一个长连接等待完整结果。若看到 `UNIDESK_SSH_RUNTIME_TIMEOUT`，先按 `docs/reference/platform-infra.md` 的规则处理为控制面可见性问题，修 CLI/job/poll 或重跑受控命令，不要手工 patch Sub2API credentials 或源码。 @@ -141,7 +142,7 @@ Codex 启动时反复出现 WebSocket reconnect、HTTPS fallback、`websocket cl 1. 在 master `~/.codex/` 准备带后缀的上游 profile 文件，例如 `config.toml.` 和 `auth.json.`；禁止覆盖默认 `config.toml` / `auth.json`。 2. 在 `config/platform-infra/sub2api-codex-pool.yaml` 添加 `profiles.entries` 项，指定 `profile`、`accountName`、`configFile`、`authFile`。 -3. 如需要，给该项加 `priority`、`capacity`、`loadFactor`、`trustUpstream`、`openaiResponsesWebSocketsV2Mode` 或 `upstreamUserAgent`；capacity/loadFactor/信任退避的具体数值只写在 YAML。只有显式恢复 Sub2API 内置临时不可调度时才添加 per-account `tempUnschedulable`。 +3. 如需要，给该项加 `priority`、`capacity`、`loadFactor`、`trustUpstream`、`sentinelProtect`、`openaiResponsesWebSocketsV2Mode` 或 `upstreamUserAgent`；capacity/loadFactor/信任退避/保护阈值的具体数值只写在 YAML。只有显式恢复 Sub2API 内置临时不可调度时才添加 per-account `tempUnschedulable`。 4. 如果新增账号会提高声明 capacity 总和，默认让省略的 `pool.minOwnerConcurrency` 继续按 capacity 总和自动解析；只有 YAML 已经显式写了该 override 时，才同步提高到不低于总 capacity，或删除 override 回到自动解析。 5. 跑 `codex-pool plan`，确认 profile 可读、`base_url` 和 API key 来源有效，且 stdout 未泄露完整 key。 6. 跑 `codex-pool sync --confirm`。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 1d21c5e5..e5c6d4e5 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -90,6 +90,12 @@ profiles: configFile: config.toml.only authFile: auth.json.only trustUpstream: true + sentinelProtect: + enabled: true + consecutiveFailures: 3 + initialRetryDelaySeconds: 2 + maxRetryDelaySeconds: 60 + backoffMultiplier: 2 loadFactor: 1 priority: 110 - profile: zakuzaku diff --git a/scripts/src/platform-infra-sub2api-codex-sentinel.ts b/scripts/src/platform-infra-sub2api-codex-sentinel.ts index fcb08174..50657ada 100644 --- a/scripts/src/platform-infra-sub2api-codex-sentinel.ts +++ b/scripts/src/platform-infra-sub2api-codex-sentinel.ts @@ -73,6 +73,15 @@ export interface CodexPoolSentinelProfileSecret { apiKey: string; upstreamUserAgent: string | null; trustUpstream: boolean; + sentinelProtect: CodexPoolSentinelProtectPolicy; +} + +export interface CodexPoolSentinelProtectPolicy { + enabled: boolean; + consecutiveFailures: number; + initialRetryDelaySeconds: number; + maxRetryDelaySeconds: number; + backoffMultiplier: number; } export interface CodexPoolSentinelManifestOptions { @@ -1148,6 +1157,85 @@ def probe_account(profile, config, purpose): "requestShape": resp.get("requestShape"), } +def protect_policy(profile): + policy = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {} + return policy if policy.get("enabled") is True else None + +def protect_failure_threshold(policy): + try: + return max(1, int(policy.get("consecutiveFailures") or 1)) + except Exception: + return 1 + +def protect_retry_delay_seconds(policy, retry_index): + try: + initial = max(1, int(policy.get("initialRetryDelaySeconds") or 2)) + except Exception: + initial = 2 + try: + maximum = max(initial, int(policy.get("maxRetryDelaySeconds") or initial)) + except Exception: + maximum = initial + try: + multiplier = max(1, int(policy.get("backoffMultiplier") or 2)) + except Exception: + multiplier = 2 + if retry_index <= 0: + return 0 + return min(maximum, initial * (multiplier ** (retry_index - 1))) + +def probe_account_with_protection(profile, config, purpose): + policy = protect_policy(profile) + if policy is None: + return probe_account(profile, config, purpose) + threshold = protect_failure_threshold(policy) + attempts = [] + last_result = None + for index in range(threshold): + delay = protect_retry_delay_seconds(policy, index) + if delay > 0: + time.sleep(delay) + attempt_purpose = purpose if index == 0 else purpose + "-protect-retry" + result = probe_account(profile, config, attempt_purpose) + attempt = { + "attempt": index + 1, + "delaySeconds": delay, + "ok": result.get("ok"), + "markerMatched": result.get("markerMatched"), + "httpStatus": result.get("httpStatus"), + "durationMs": result.get("durationMs"), + "failureKind": result.get("failureKind"), + "outputHash": result.get("outputHash"), + "responseBodyHash": result.get("responseBodyHash"), + "errorDetails": result.get("errorDetails"), + } + attempts.append(attempt) + last_result = result + if result.get("markerMatched") is True: + result["sentinelProtect"] = { + "enabled": True, + "threshold": threshold, + "attempts": attempts, + "failureCount": index, + "protected": index > 0, + "decision": "pass", + } + if index > 0: + result["purpose"] = purpose + "-protect-recovered" + return result + if last_result is None: + last_result = probe_account(profile, config, purpose) + last_result["sentinelProtect"] = { + "enabled": True, + "threshold": threshold, + "attempts": attempts, + "failureCount": len(attempts), + "protected": False, + "decision": "fail", + } + last_result["purpose"] = purpose + "-protect-exhausted" + return last_result + def ledger_for(state, now): day = day_key(now) ledger = state.setdefault("ledger", {}).setdefault(day, {"inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "estimatedCostUsd": 0, "requestCount": 0}) @@ -1306,6 +1394,7 @@ def apply_result(result, state, config, now, admin, profile): "ok": result.get("ok"), "purpose": result.get("purpose"), "trustUpstream": result.get("trustUpstream"), + "sentinelProtect": result.get("sentinelProtect"), "successMaxIntervalMinutes": success_max_interval(profile, config), "httpStatus": result.get("httpStatus"), "durationMs": result.get("durationMs"), @@ -1479,6 +1568,68 @@ def next_gateway_freeze_interval(account_state, config): def apply_gateway_failure(account_name, failures, state, config, now, admin, profile): latest = failures[-1] account_state = state.setdefault("accounts", {}).setdefault(account_name, {}) + policy = protect_policy(profile) + protected_probe = None + if policy is not None: + protected_probe = probe_account_with_protection(profile, config, "gateway-failure-confirm") + protected_probe["sourceGatewayFailure"] = { + "requestId": latest.get("requestId"), + "clientRequestId": latest.get("clientRequestId"), + "failureKind": latest.get("failureKind"), + "path": latest.get("path"), + "countInRun": len(failures), + } + account_state["lastProbeAt"] = iso(now) + account_state["lastProbe"] = { + "ok": protected_probe.get("ok"), + "purpose": protected_probe.get("purpose"), + "trustUpstream": protected_probe.get("trustUpstream"), + "sentinelProtect": protected_probe.get("sentinelProtect"), + "successMaxIntervalMinutes": success_max_interval(profile, config), + "httpStatus": protected_probe.get("httpStatus"), + "durationMs": protected_probe.get("durationMs"), + "markerMatched": protected_probe.get("markerMatched"), + "transportOk": protected_probe.get("transportOk"), + "outputHash": protected_probe.get("outputHash"), + "outputPreview": protected_probe.get("outputPreview"), + "responseBodyHash": protected_probe.get("responseBodyHash"), + "responseBodyPreview": protected_probe.get("responseBodyPreview"), + "error": protected_probe.get("error"), + "errorDetails": protected_probe.get("errorDetails"), + "usage": protected_probe.get("usage"), + "failureKind": protected_probe.get("failureKind"), + "sdk": protected_probe.get("sdk"), + "requestShape": protected_probe.get("requestShape"), + "action": {"taken": False, "type": "protect-confirm-pass" if protected_probe.get("markerMatched") is True else "protect-confirm-fail"}, + "sourceGatewayFailure": protected_probe.get("sourceGatewayFailure"), + } + add_usage(state, account_state, now, protected_probe.get("usage") or {}) + account_state["sentinelProtect"] = protected_probe.get("sentinelProtect") + account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["successMaxIntervalMinutes"] = success_max_interval(profile, config) + if protected_probe.get("markerMatched") is True: + interval = next_success_interval(account_state, config, profile) + account_state["successStreak"] = int(account_state.get("successStreak") or 0) + 1 + account_state["successIntervalMinutes"] = interval + account_state["nextProbeAfter"] = iso(add_minutes(now, interval, int(config["cadence"]["jitterPercent"]))) + account_state["lastOkAt"] = iso(now) + account_state["lastStatus"] = "gateway-failure-protect-confirmed-ok" + account_state["lastGatewayFailureAt"] = iso(now) + account_state["lastGatewayFailure"] = { + "accountName": account_name, + "accountId": latest.get("accountId"), + "requestId": latest.get("requestId"), + "clientRequestId": latest.get("clientRequestId"), + "failureKind": latest.get("failureKind"), + "path": latest.get("path"), + "errorPreview": latest.get("errorPreview"), + "countInRun": len(failures), + "firstAt": failures[0].get("at"), + "lastAt": latest.get("at"), + "action": {"taken": False, "type": "protect-confirm-pass"}, + "sentinelProtect": protected_probe.get("sentinelProtect"), + } + return {"taken": False, "type": "protect-confirm-pass", "sentinelProtect": protected_probe.get("sentinelProtect")} interval = next_gateway_freeze_interval(account_state, config) until = add_minutes(now, interval, int(config["freeze"]["jitterPercent"])) actions_enabled = bool(config["actions"]["enabled"]) @@ -1511,6 +1662,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro "countInRun": len(failures), }, "lastBadAt": iso(now), + "sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None, } account_state["nextProbeAfter"] = iso(until) account_state["successStreak"] = 0 @@ -1538,6 +1690,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro "intervalMinutes": interval, "freezeUntil": iso(until), "action": action, + "sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None, } return action @@ -1721,7 +1874,7 @@ def main(): actions = [] if (config["monitor"]["enabled"] or forced_names) and due: with ThreadPoolExecutor(max_workers=max(1, len(due))) as executor: - futures = {executor.submit(probe_account, item["profile"], config, item["purpose"]): item["profile"] for item in due} + futures = {executor.submit(probe_account_with_protection, item["profile"], config, item["purpose"]): item["profile"] for item in due} for future in as_completed(futures): result = future.result() results.append(result) @@ -1760,6 +1913,7 @@ def main(): "accountName": item.get("accountName"), "purpose": item.get("purpose"), "trustUpstream": item.get("trustUpstream"), + "sentinelProtect": item.get("sentinelProtect"), "ok": item.get("ok"), "markerMatched": item.get("markerMatched"), "httpStatus": item.get("httpStatus"), diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index 710a9c39..3c4ca3c8 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -105,6 +105,7 @@ interface CodexProfile { openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null; upstreamUserAgent: string | null; trustUpstream: boolean; + sentinelProtect: CodexSentinelProtectPolicy; priority: number; capacity: number; loadFactor: number; @@ -156,12 +157,21 @@ interface CodexPoolProfileConfig { openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null; upstreamUserAgent: string | null; trustUpstream: boolean; + sentinelProtect: CodexSentinelProtectPolicy; priority: number; capacity: number | null; loadFactor: number | null; tempUnschedulable: CodexTempUnschedulablePolicy; } +export interface CodexSentinelProtectPolicy { + enabled: boolean; + consecutiveFailures: number; + initialRetryDelaySeconds: number; + maxRetryDelaySeconds: number; + backoffMultiplier: number; +} + interface CodexPoolPublicExposureConfig { enabled: boolean; proxyName: string; @@ -714,10 +724,12 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi upstreamUserAgent: profile.upstreamUserAgent, openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, }), openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, priority: profile.priority, capacity: profile.capacity, loadFactor: profile.loadFactor, @@ -1098,6 +1110,7 @@ function collectCodexProfiles(): CodexProfile[] { openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: entry.upstreamUserAgent, trustUpstream: entry.trustUpstream, + sentinelProtect: entry.sentinelProtect, priority: entry.priority, capacity: entry.capacity ?? pool.defaultAccountCapacity, loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor, @@ -1171,6 +1184,7 @@ function discoverCodexProfileConfigs( openaiResponsesWebSocketsV2Mode: null, upstreamUserAgent: null, trustUpstream: false, + sentinelProtect: defaultCodexSentinelProtectPolicy(), priority: defaultPriority, capacity: null, loadFactor: null, @@ -1340,6 +1354,7 @@ function readProfileConfig( const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`); const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`); const trustUpstream = readTrustUpstream(entry.trustUpstream, `profiles.entries[${index}].trustUpstream`); + const sentinelProtect = readSentinelProtectPolicy(entry.sentinelProtect, `profiles.entries[${index}].sentinelProtect`); const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority); const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`); const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`); @@ -1354,6 +1369,7 @@ function readProfileConfig( openaiResponsesWebSocketsV2Mode, upstreamUserAgent, trustUpstream, + sentinelProtect, priority, capacity, loadFactor, @@ -1396,6 +1412,52 @@ function readTrustUpstream(value: unknown, key: string): boolean { return parsed; } +export function defaultCodexSentinelProtectPolicy(): CodexSentinelProtectPolicy { + return { + enabled: false, + consecutiveFailures: 1, + initialRetryDelaySeconds: 2, + maxRetryDelaySeconds: 60, + backoffMultiplier: 2, + }; +} + +function readSentinelProtectPolicy(value: unknown, key: string): CodexSentinelProtectPolicy { + const fallback = defaultCodexSentinelProtectPolicy(); + if (value === undefined || value === null) return fallback; + if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); + const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled); + if (!enabled) return { ...fallback, enabled: false }; + const required = ["consecutiveFailures", "initialRetryDelaySeconds", "maxRetryDelaySeconds", "backoffMultiplier"]; + for (const field of required) { + if (value[field] === undefined || value[field] === null) { + throw new Error(`${codexPoolConfigPath}.${key}.${field} is required when enabled=true`); + } + } + const consecutiveFailures = readBoundedInteger(value.consecutiveFailures, `${key}.consecutiveFailures`, 1, 20); + const initialRetryDelaySeconds = readBoundedInteger(value.initialRetryDelaySeconds, `${key}.initialRetryDelaySeconds`, 1, 3600); + const maxRetryDelaySeconds = readBoundedInteger(value.maxRetryDelaySeconds, `${key}.maxRetryDelaySeconds`, 1, 3600); + const backoffMultiplier = readBoundedInteger(value.backoffMultiplier, `${key}.backoffMultiplier`, 1, 10); + if (maxRetryDelaySeconds < initialRetryDelaySeconds) { + throw new Error(`${codexPoolConfigPath}.${key}.maxRetryDelaySeconds must be >= initialRetryDelaySeconds`); + } + return { + enabled: true, + consecutiveFailures, + initialRetryDelaySeconds, + maxRetryDelaySeconds, + backoffMultiplier, + }; +} + +function readBoundedInteger(value: unknown, key: string, min: number, max: number): number { + const parsed = numberValue(value); + if (parsed === null || !Number.isInteger(parsed) || parsed < min || parsed > max) { + throw new Error(`${codexPoolConfigPath}.${key} must be an integer from ${min} to ${max}`); + } + return parsed; +} + function readAccountPriority(value: unknown, key: string, fallback = defaultAccountPriority): number { if (value === undefined || value === null) return fallback; const priority = numberValue(value); @@ -1723,6 +1785,7 @@ function redactProfile(profile: CodexProfile): Record { openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, priority: profile.priority, capacity: profile.capacity, loadFactor: profile.loadFactor, @@ -1745,6 +1808,8 @@ function compactProfile(profile: CodexProfile): Record { model: profile.model, priority: profile.priority, trustUpstream: profile.trustUpstream, + sentinelProtectEnabled: profile.sentinelProtect.enabled, + sentinelProtectConsecutiveFailures: profile.sentinelProtect.enabled ? profile.sentinelProtect.consecutiveFailures : undefined, capacity: profile.capacity, loadFactor: profile.loadFactor, tempUnschedulableEnabled: profile.tempUnschedulable.enabled, @@ -2061,6 +2126,7 @@ function compactSentinelErrorDetails(value: unknown): Record | function compactSentinelQuarantine(item: unknown): Record { if (!isRecord(item)) return {}; + const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {}; return { accountName: item.accountName, until: item.until, @@ -2068,6 +2134,12 @@ function compactSentinelQuarantine(item: unknown): Record { reason: item.reason, failureKind: item.failureKind, intervalMinutes: item.intervalMinutes, + sentinelProtect: Object.keys(protect).length > 0 ? { + enabled: protect.enabled, + decision: protect.decision, + failureCount: protect.failureCount, + threshold: protect.threshold, + } : undefined, error: compactSentinelErrorDetails(item.errorDetails), }; } @@ -2076,6 +2148,7 @@ function compactSentinelRecentAccount(item: unknown): Record { if (!isRecord(item)) return {}; const action = isRecord(item.action) ? item.action : {}; const error = compactSentinelErrorDetails(item.errorDetails); + const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {}; return { accountName: item.accountName, lastProbeAt: item.lastProbeAt, @@ -2086,6 +2159,13 @@ function compactSentinelRecentAccount(item: unknown): Record { httpStatus: item.httpStatus, durationMs: item.durationMs, markerMatched: item.markerMatched, + sentinelProtect: Object.keys(protect).length > 0 ? { + enabled: protect.enabled, + decision: protect.decision, + protected: protect.protected, + failureCount: protect.failureCount, + threshold: protect.threshold, + } : undefined, failureKind: item.failureKind, requestShape: item.requestShape, action: Object.keys(action).length > 0 ? { @@ -2238,6 +2318,7 @@ function compactSentinelProbeResult(parsed: Record | null): Rec "purpose", "ok", "markerMatched", + "sentinelProtect", "httpStatus", "durationMs", "usage", @@ -2307,12 +2388,14 @@ function renderSentinelReport( lines.push(""); lines.push("ACCOUNTS"); lines.push(renderTable([ - ["ACCOUNT", "STATE", "Q", "T", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"], + ["ACCOUNT", "STATE", "Q", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"], ...accounts.map((account) => [ stringValue(account.account) ?? "-", stringValue(account.status) ?? "-", account.quarantineActive === true ? "Y" : "-", account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-", + account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-", + account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-", textValue(account.freezeIntervalMin), textValue(account.successIntervalMin), textValue(account.successMaxIntervalMin), @@ -2346,7 +2429,7 @@ function renderSentinelReport( ])); } lines.push(""); - lines.push("LEGEND Q=quarantined T=trusted upstream M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions"); + lines.push("LEGEND Q=quarantined T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions"); lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw"); return lines.join("\n"); } @@ -2687,6 +2770,7 @@ function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProf apiKey: profile.apiKey ?? "", upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, })); } @@ -3389,6 +3473,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: { upstreamUserAgent: string | null; openaiResponsesWebSocketsV2Mode: string | null; trustUpstream: boolean; + sentinelProtect: CodexSentinelProtectPolicy; }): string { return fingerprint(JSON.stringify({ accountName: input.accountName, @@ -3398,6 +3483,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: { upstreamUserAgent: input.upstreamUserAgent, openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode, trustUpstream: input.trustUpstream, + sentinelProtect: input.sentinelProtect, })); } @@ -3517,6 +3603,24 @@ def error_code(probe): return openai_error.get("code") or openai_error.get("type") or "" return details.get("kind") or "" +def sentinel_protect_summary(account_state, probe): + probe_protect = probe.get("sentinelProtect") if isinstance(probe.get("sentinelProtect"), dict) else {} + config_protect = account_state.get("sentinelProtectConfig") if isinstance(account_state.get("sentinelProtectConfig"), dict) else {} + source = probe_protect if probe_protect else config_protect + if not isinstance(source, dict) or source.get("enabled") is not True: + return { + "enabled": False, + "threshold": None, + "decision": None, + "failureCount": None, + } + return { + "enabled": True, + "threshold": source.get("threshold") or source.get("consecutiveFailures"), + "decision": probe_protect.get("decision"), + "failureCount": probe_protect.get("failureCount"), + } + def report(): cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME]) state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME]) @@ -3544,6 +3648,7 @@ def report(): if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")): last_action = action_type(gateway_failure) ledger = account_ledger(account_state) + protect = sentinel_protect_summary(account_state, probe) account_rows.append({ "account": name, "status": account_state.get("lastStatus"), @@ -3566,6 +3671,10 @@ def report(): "lastPurpose": probe.get("purpose"), "lastHttp": probe.get("httpStatus"), "lastMarker": probe.get("markerMatched"), + "sentinelProtectEnabled": protect.get("enabled"), + "sentinelProtectDecision": protect.get("decision"), + "sentinelProtectFailureCount": protect.get("failureCount"), + "sentinelProtectThreshold": protect.get("threshold"), "lastFailureKind": last_failure_kind, "lastErrorCode": error_code(probe), "lastAction": last_action, @@ -3987,6 +4096,8 @@ def sentinel_probe_change_reasons(current, profile): runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode")) expected_trust_upstream = profile.get("trustUpstream") is True runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True + expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} + runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False} reasons = [] if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"): reasons.append("profile") @@ -4000,6 +4111,8 @@ def sentinel_probe_change_reasons(current, profile): reasons.append("responses-websockets-v2-mode") if runtime_trust_upstream != expected_trust_upstream: reasons.append("trust-upstream") + if runtime_protect != expected_protect: + reasons.append("sentinel-protect") return reasons def curl_api(method, path, bearer=None, payload=None): @@ -4214,6 +4327,7 @@ def account_payload(profile, group_id): "unidesk_codex_profile": profile["profile"], "unidesk_managed": True, "unidesk_trust_upstream": profile.get("trustUpstream") is True, + "unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, } ws_mode = profile.get("openaiResponsesWebSocketsV2Mode") if ws_mode: @@ -4264,6 +4378,7 @@ def planned_sentinel_account_results(profiles, existing_accounts): "profileConfig": { "accountName": profile["accountName"], "trustUpstream": profile.get("trustUpstream") is True, + "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, }, "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), "sentinelProbeRequired": quality_gate_required, @@ -4308,6 +4423,7 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_fr "profileConfig": { "accountName": profile["accountName"], "trustUpstream": profile.get("trustUpstream") is True, + "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, }, "accountId": data.get("id") if isinstance(data, dict) else None, "action": action, @@ -4467,6 +4583,7 @@ def sentinel_runtime_status(): "failureKind": quarantine.get("failureKind"), "errorDetails": quarantine.get("errorDetails"), "intervalMinutes": quarantine.get("intervalMinutes"), + "sentinelProtect": quarantine.get("sentinelProtect"), }) last_probe = account_state.get("lastProbe") if isinstance(last_probe, dict): @@ -4480,6 +4597,7 @@ def sentinel_runtime_status(): "httpStatus": last_probe.get("httpStatus"), "durationMs": last_probe.get("durationMs"), "markerMatched": last_probe.get("markerMatched"), + "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), "outputHash": last_probe.get("outputHash"), "outputPreview": last_probe.get("outputPreview"), "responseBodyHash": last_probe.get("responseBodyHash"), @@ -4652,6 +4770,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now): quarantine = account_state.get("quarantine") if isinstance(quarantine, dict) and quarantine.get("active") is True: account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile) continue try: @@ -4661,6 +4780,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now): next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter")) max_interval = profile_success_max_interval(profile) account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = max_interval if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60): continue @@ -4751,6 +4871,7 @@ def ensure_sentinel_state_for_sync(account_results, pending_only=False): account_state["successIntervalMinutes"] = 0 profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {} account_state["trustUpstream"] = profile_config.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config) account_state["lastStatus"] = "pending-sentinel-quality-gate" account_state["qualityGate"] = { @@ -4811,6 +4932,7 @@ def sentinel_state_summary(): "failureKind": quarantine.get("failureKind"), "errorDetails": quarantine.get("errorDetails"), "intervalMinutes": quarantine.get("intervalMinutes"), + "sentinelProtect": quarantine.get("sentinelProtect"), }) last_probe = account_state.get("lastProbe") if isinstance(last_probe, dict): @@ -4824,6 +4946,7 @@ def sentinel_state_summary(): "httpStatus": last_probe.get("httpStatus"), "durationMs": last_probe.get("durationMs"), "markerMatched": last_probe.get("markerMatched"), + "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), "usage": last_probe.get("usage"), "responseBodyHash": last_probe.get("responseBodyHash"), "responseBodyPreview": last_probe.get("responseBodyPreview"), From 501455d91854a5c3df4c012bfd1399b9f1b1dfe8 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 05:37:18 +0000 Subject: [PATCH 08/13] docs: document sub2api sentinel protect policy --- docs/reference/platform-infra.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index f23260a3..91109ec9 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -117,13 +117,15 @@ The sentinel must not maintain separate classifiers for "private content", "main `profiles.entries[].trustUpstream` is the durable account-level trust marker for sentinel success cadence, and the absence of the field means untrusted. Trusted and untrusted accounts use separate YAML cadence maximums after marker-matching probes; the values belong only in `config/platform-infra/sub2api-codex-pool.yaml`. This field must not change Sub2API scheduler priority, capacity, load factor, membership, built-in temporary-unschedulable settings, or the marker-only health contract. Its purpose is to keep intermittently unreliable 200-success providers under more frequent direct probes without adding provider-specific content classifiers. -When `codex-pool sync --confirm` creates a YAML-managed account or changes direct-probe-relevant account inputs such as the profile mapping, upstream base URL, API key fingerprint, upstream User-Agent, Responses WebSocket mode, or `trustUpstream`, sync records a pending sentinel probe from the pre-mutation runtime state, updates the account, restores `schedulable=true` unless an active sentinel quarantine already exists, and schedules the account probe immediately. New or changed accounts are not default-frozen; only an actual non-marker probe result or an existing active quarantine may remove an account from the scheduler. This avoids zero-available windows during sync while still ensuring that later marker failures enter the normal freeze/restore state machine. Unchanged accounts must not have their existing success or failure backoff reset by unrelated YAML syncs. +`profiles.entries[].sentinelProtect` is an optional account-level protection policy for sentinel freeze decisions, and the absence of the field means disabled. For protected accounts, the marker-only health contract still applies, but the sentinel must exhaust the configured consecutive marker confirmation attempts before treating the account as failed and entering the freeze state machine. The retry count, initial delay, maximum delay, and backoff multiplier are YAML values; long-term reference prose must not duplicate the current numbers. This policy exists only to absorb occasional marker/probe or gateway-failure confirmation jitter for selected accounts. It must not change Sub2API scheduler priority, capacity, load factor, membership, built-in temporary-unschedulable settings, or the recovery condition. + +When `codex-pool sync --confirm` creates a YAML-managed account or changes direct-probe-relevant account inputs such as the profile mapping, upstream base URL, API key fingerprint, upstream User-Agent, Responses WebSocket mode, `trustUpstream`, or `sentinelProtect`, sync records a pending sentinel probe from the pre-mutation runtime state, updates the account, restores `schedulable=true` unless an active sentinel quarantine already exists, and schedules the account probe immediately. New or changed accounts are not default-frozen; only an actual non-marker probe result or an existing active quarantine may remove an account from the scheduler. This avoids zero-available windows during sync while still ensuring that later marker failures enter the normal freeze/restore state machine. Unchanged accounts must not have their existing success or failure backoff reset by unrelated YAML syncs. If the YAML failure freeze maximum is lowered, `codex-pool sync --confirm` may migrate only currently active sentinel quarantines whose stored interval or next recovery time exceeds the current maximum. The migration keeps the account frozen, marks the next recovery probe due immediately, and lets the next marker result decide restore versus the new shorter failure backoff. It must not clear quarantine or restore schedulability merely because an older TTL has expired. If the YAML success cadence maximum is lowered or an account changes trust class, `codex-pool sync --confirm` may clamp existing successful account state so the next probe is due under the current YAML policy instead of waiting for an older, longer success window to expire. This clamp only affects sentinel state and probe timing; it does not by itself restore a quarantined account or bypass the next marker result. -Operational observation for this sentinel should use the read-only `codex-pool sentinel-report` table or its `--raw` form. It is the canonical low-noise view for per-account probe count, trust class, marker result, HTTP/error diagnostics, freeze TTL, success cadence, success cadence maximum, next probe time, and recent CronJob runs; raw ConfigMap dumps and ad hoc log scraping are fallback diagnostics, not the primary state surface. +Operational observation for this sentinel should use the read-only `codex-pool sentinel-report` table or its `--raw` form. It is the canonical low-noise view for per-account probe count, trust class, protect threshold and latest protect confirmation result, marker result, HTTP/error diagnostics, freeze TTL, success cadence, success cadence maximum, next probe time, and recent CronJob runs; raw ConfigMap dumps and ad hoc log scraping are fallback diagnostics, not the primary state surface. The active Codex-pool request path follows the YAML-selected active target: From 3717becd77e8923b760ab04ff1f3fd1c90961747 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 06:01:27 +0000 Subject: [PATCH 09/13] feat: move AgentRun deploy truth to UniDesk YAML --- .agents/skills/unidesk-cicd/SKILL.md | 22 +- .agents/skills/unidesk-ops/SKILL.md | 17 +- config/agentrun.yaml | 147 ++++ docs/reference/agentrun.md | 19 +- docs/reference/yaml-first-ops.md | 6 + scripts/src/agentrun-lanes.ts | 279 ++++++++ scripts/src/agentrun-manifests.ts | 489 +++++++++++++ scripts/src/agentrun.ts | 986 +++++++++++++++++++++++++-- 8 files changed, 1918 insertions(+), 47 deletions(-) create mode 100644 scripts/src/agentrun-manifests.ts diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 0a454dc2..e7a1548c 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -1,6 +1,6 @@ --- name: unidesk-cicd -description: UniDesk CI/CD 控制面 — `hwlab g14` 和 `agentrun` 子命令，覆盖 PR 监控自动合并、Tekton/Argo 控制面、git-mirror、Secret、observability、CI tools image、PipelineRun 清理和 AgentRun v0.1 部署。用户提到 CI/CD、deploy、rollout、PipelineRun、trigger、git-mirror、control-plane、k3s 部署、agentrun 部署、hwlab g14、monitor-prs、trigger-current 时使用。任何需要把代码变更推送部署到 G14 k3s 的操作都必须走本 skill。 +description: UniDesk CI/CD 控制面 — `hwlab g14` 和 `agentrun` 子命令，覆盖 PR 监控自动合并、Tekton/Argo 控制面、git-mirror、Secret、observability、CI tools image、PipelineRun 清理、AgentRun v0.1 部署和 AgentRun YAML-only lane 部署。用户提到 CI/CD、deploy、rollout、PipelineRun、trigger、git-mirror、control-plane、k3s 部署、agentrun 部署、hwlab g14、monitor-prs、trigger-current 时使用。任何需要把代码变更推送部署到 G14 k3s 的操作都必须走本 skill。 --- # UniDesk HWLAB G14 CI/CD CLI @@ -206,7 +206,25 @@ bun scripts/cli.ts hwlab g14 record-rollout --pr --source-commit --- -## AgentRun v0.1 控制面 +## AgentRun 控制面 + +YAML-only lane 以 `config/agentrun.yaml` 为部署真相；node/lane、source workspace/branch、image build、GitOps branch/path、runtime namespace、Secret、外置数据库、manager env、git-mirror 和 edge 暴露都从 YAML 进入 CLI。AgentRun service repo 的 `deploy/deploy.json` 不能作为 UniDesk deployment truth；新 lane 不再维护该文件。 + +```bash +bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02 +bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 [--dry-run|--confirm] +bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 [--dry-run|--confirm] +bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 [--dry-run|--confirm] +bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--full] +``` + +- `plan`: 只读解析 YAML，输出控制面、source、image build、GitOps、runtime 和 Secret plan，不打印 Secret value +- `apply`: 按 YAML 渲染并 apply Tekton RBAC/Pipeline、Argo AppProject/Application 和 runtime namespace +- `secret-sync`: 按 YAML 的 Secret sourceRef/keyMapping 同步 runtime Secret 和外置 DB Secret，只输出 fingerprint +- `trigger-current`: 确保 source branch/workspace，删除新 lane source branch 的 `deploy/deploy.json`，构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，触发 git-mirror sync 和 provenance PipelineRun +- `status`: 汇总 node/lane 控制面、runtime、Argo、Secret、source workspace 和 GitOps 对齐状态 + +### AgentRun v0.1 兼容入口 ```bash bun scripts/cli.ts agentrun control-plane status \ diff --git a/.agents/skills/unidesk-ops/SKILL.md b/.agents/skills/unidesk-ops/SKILL.md index 99d7649f..096e1342 100644 --- a/.agents/skills/unidesk-ops/SKILL.md +++ b/.agents/skills/unidesk-ops/SKILL.md @@ -76,7 +76,7 @@ bun scripts/cli.ts server cleanup plan [--min-age-hours 24] [--limit N] bun scripts/cli.ts server cleanup run --confirm [--min-age-hours 24] [--limit N] ``` -`plan` 只生成 dry-run 计划；`run --confirm` 只删除同一 classifier 选出的 stale Docker images。保守白名单：保留 running/stopped 容器镜像、deploy.json/CI.json commit-pinned artifact、Compose stable image。禁止 `docker system prune`、`docker image prune`、`docker volume rm`、`docker compose down -v` 和数据库清理。高风险候选必须额外显式 `--include-high-risk` 才会执行。 +`plan` 只生成 dry-run 计划；`run --confirm` 只删除同一 classifier 选出的 stale Docker images。保守白名单：保留 running/stopped 容器镜像、UniDesk YAML/GitOps/image catalog 声明的 commit-pinned artifact、Compose stable image。禁止 `docker system prune`、`docker image prune`、`docker volume rm`、`docker compose down -v` 和数据库清理。高风险候选必须额外显式 `--include-high-risk` 才会执行。 --- @@ -174,6 +174,21 @@ SCRIPT 长期边界见 `docs/reference/pk01.md`；Sub2API 消费侧边界见 `docs/reference/platform-infra.md`。 +## YAML-First 分布式运维边界 + +UniDesk 自有分布式运维以 `config/**/*.yaml` 为 desired-state truth。服务仓库里的 `deploy.json` 不能作为 UniDesk deployment truth；node/lane、runtime namespace、GitOps branch/path、image artifact、public exposure、Secret、外置数据库、probe 和 rollout 等运维选择必须进入所属 UniDesk YAML，并通过受控 CLI 渲染或同步。 + +AgentRun v0.2/D601 这类 YAML-only lane 的控制面、Secret 同步、外置 DB wiring 和状态检查使用： + +```bash +bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02 +bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --confirm +bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --confirm +bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 --full +``` + +部署触发和 GitOps promotion 入口归 `$unidesk-cicd`；本 skill 只记录手动运维边界和长期排障入口。长期架构见 `docs/reference/yaml-first-ops.md`，AgentRun 细则见 `docs/reference/agentrun.md`。 + --- ## Moon Bridge 管理 diff --git a/config/agentrun.yaml b/config/agentrun.yaml index b910350b..769e07c2 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -49,6 +49,7 @@ controlPlane: source: repository: pikasTech/agentrun branch: v0.1 + bootstrapFromBranch: v0.1 remote: git@github.com:pikasTech/agentrun.git workspace: /root/agentrun-v01 runtime: @@ -70,13 +71,83 @@ controlPlane: argoNamespace: argocd argoApplication: agentrun-g14-v01 repoURL: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git + deployment: + format: unidesk-yaml-only + gitopsRoot: deploy/gitops/g14 + runtimeRenderDir: runtime-v01 + artifactCatalogPath: deploy/artifact-catalog.v01.json + argocd: + project: agentrun-v01 + applicationFile: application-v01.yaml + manager: + serviceAccount: agentrun-v01-mgr + apiKeySecretRef: + name: agentrun-v01-api-key + key: HWLAB_API_KEY + unideskSshEndpointEnv: + name: UNIDESK_MAIN_SERVER_IP + value: 74.48.78.17 + bootRepoUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git + imageBuild: + context: . + containerfile: deploy/container/Containerfile + repository: agentrun-mgr-env + network: host + httpProxy: http://127.0.0.1:10808 + httpsProxy: http://127.0.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - hyueapi.com + - .hyueapi.com + envIdentityFiles: + - deploy/container/Containerfile + - deploy/runtime/boot/agentrun-boot.sh + - deploy/runtime/boot/agentrun-mgr.sh + - deploy/runtime/boot/agentrun-runner.sh + - package.json + - bun.lock + - tsconfig.json + timeoutSeconds: 1800 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 800m + memory: 1Gi + runner: + serviceAccount: agentrun-v01-runner + jobNamePrefix: agentrun-v01-runner + apiKeySecretRef: + name: agentrun-v01-api-key + key: HWLAB_API_KEY + localPostgres: + enabled: true + serviceName: agentrun-v01-postgres + image: postgres:16-alpine + storage: 5Gi + port: 5432 gitMirror: namespace: devops-infra readService: git-mirror-http + readDeployment: git-mirror-http writeService: git-mirror-write + writeDeployment: git-mirror-write readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/agentrun.git cachePvc: git-mirror-cache + cacheHostPath: null + sshSecretName: git-mirror-github-ssh + githubProxy: + host: 127.0.0.1 + port: 10808 toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 syncJobPrefix: git-mirror-agentrun-sync-manual flushJobPrefix: git-mirror-agentrun-flush-manual @@ -97,6 +168,7 @@ controlPlane: name: agentrun-v01-mgr-db key: DATABASE_URL localPostgresExpectedAbsent: false + secrets: [] v02: node: D601 @@ -104,6 +176,7 @@ controlPlane: source: repository: pikasTech/agentrun branch: v0.2 + bootstrapFromBranch: v0.1 remote: git@github.com:pikasTech/agentrun.git workspace: /home/ubuntu/workspace/agentrun-v02 runtime: @@ -125,13 +198,79 @@ controlPlane: argoNamespace: argocd argoApplication: agentrun-d601-v02 repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + deployment: + format: unidesk-yaml-only + gitopsRoot: deploy/gitops/node/d601 + runtimeRenderDir: runtime-v02 + artifactCatalogPath: deploy/artifact-catalog.v02.json + argocd: + project: agentrun-v02 + applicationFile: application-v02.yaml + manager: + serviceAccount: agentrun-v02-mgr + apiKeySecretRef: + name: agentrun-v02-api-key + key: HWLAB_API_KEY + unideskSshEndpointEnv: + name: UNIDESK_MAIN_SERVER_IP + value: 74.48.78.17 + bootRepoUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + imageBuild: + context: . + containerfile: deploy/container/Containerfile + repository: agentrun-mgr-env + network: host + httpProxy: http://127.0.0.1:18789 + httpsProxy: http://127.0.0.1:18789 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - hyueapi.com + - .hyueapi.com + envIdentityFiles: + - deploy/container/Containerfile + - deploy/runtime/boot/agentrun-boot.sh + - deploy/runtime/boot/agentrun-mgr.sh + - deploy/runtime/boot/agentrun-runner.sh + - package.json + - bun.lock + - tsconfig.json + timeoutSeconds: 1800 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 800m + memory: 1Gi + runner: + serviceAccount: agentrun-v02-runner + jobNamePrefix: agentrun-v02-runner + apiKeySecretRef: + name: agentrun-v02-api-key + key: HWLAB_API_KEY + localPostgres: + enabled: false gitMirror: namespace: devops-infra readService: git-mirror-http + readDeployment: git-mirror-http writeService: git-mirror-write + writeDeployment: git-mirror-write readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git cachePvc: hwlab-git-mirror-cache + cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d601-v03-git-mirror-cache + sshSecretName: git-mirror-github-ssh + githubProxy: + host: sub2api-egress-proxy.platform-infra.svc.cluster.local + port: 10808 toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 syncJobPrefix: git-mirror-agentrun-d601-v02-sync-manual flushJobPrefix: git-mirror-agentrun-d601-v02-flush-manual @@ -158,3 +297,11 @@ controlPlane: name: agentrun-v02-mgr-db key: DATABASE_URL localPostgresExpectedAbsent: true + secrets: + - id: manager-api-key + sourceRef: /root/.config/hwlab-v02/master-server-admin-api-key.env + sourceKey: HWLAB_API_KEY + targetRef: + namespace: agentrun-v02 + name: agentrun-v02-api-key + key: HWLAB_API_KEY diff --git a/docs/reference/agentrun.md b/docs/reference/agentrun.md index 77119f2a..68e2e48c 100644 --- a/docs/reference/agentrun.md +++ b/docs/reference/agentrun.md @@ -63,7 +63,7 @@ trans G14:k3s kubectl get pods -n agentrun-v01 ## 受控 CI/CD 入口 -AgentRun `v0.1` 的 Tekton/Argo 控制面写操作必须通过 UniDesk 高层 CLI 执行： +AgentRun 控制面写操作必须通过 UniDesk 高层 CLI 执行。历史 `v0.1` G14 lane 仍保留无 `--node/--lane` 的兼容入口；新增或迁移 lane 必须使用 `--node --lane ` 从 `config/agentrun.yaml` 解析目标，不得从 AgentRun service repo 的 `deploy.json` 读取部署真相。 ```bash bun scripts/cli.ts agentrun control-plane status @@ -77,8 +77,23 @@ bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --dry bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --limit 200 --confirm ``` +YAML-only lane 的标准入口是： + +```bash +bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02 +bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run +bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --confirm +bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run +bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --confirm +bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run +bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --confirm +bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 --full +``` + `status` 只读观察 `G14:/root/agentrun-v01` 当前 commit、对应 PipelineRun、GitOps latest、Argo Application、`agentrun-v01` workload、manager source commit 和 git mirror 摘要，并报告 Argo revision 是否对齐 `v0.1-gitops` latest。默认输出是 compact commander 视图，只保留 `summary`、阶段耗时、对齐状态和 drill-down 命令；需要远端 stdout/stderr tail 时显式加 `--full`，需要原始 git mirror cache 输出时显式加 `--raw`。`status` 额外支持 `--pipeline-run ` 与 `--source-commit ` 定点查询，返回 `target`、`targetValidation` 和 `next.*` drill-down，便于直接判断某次 run 是成功、历史成功、运行中、缺失还是 source mismatch。`status` 会向 stderr 输出 `agentrun.control-plane.status.progress` 阶段事件，覆盖 `source`、`runtime` 和 `git-mirror`，避免长时间聚合时无可见进展。`trigger-current` 会先把固定 source worktree 快进到 `origin/v0.1`，再以当前 commit 创建 commit-pinned PipelineRun；同名 PipelineRun 正在运行或已经成功时必须拒绝重复触发，只允许在失败态或不存在时创建。该命令只提交 CI/CD 工作，不等待完整 PipelineRun 或 rollout 完成，后续用 `status` 轮询。`refresh` 只对 `argocd/agentrun-g14-v01` 执行 hard refresh，用于 GitOps promotion 已完成但 Argo 仍停留旧 revision 时的受控同步入口；它不直接 patch runtime workload。 +YAML-only lane 的 `trigger-current` 会先确保目标 source workspace/branch 存在，再从 UniDesk YAML 声明的 image build、GitOps branch/path、runtime namespace、Secret、数据库和 manager env 渲染 artifact catalog 与 GitOps desired state。该路径会删除新 lane source branch 中的 `deploy/deploy.json`，因为部署真相已经迁入 UniDesk YAML；旧 `v0.1` branch 中历史文件只作为迁移前遗留产物存在，不能作为新 lane 的事实来源。 + `cleanup-runs` 是 AgentRun `v0.1` 完成态 CI workspace retention 入口，只清理 `agentrun-ci` namespace 中超过 `--min-age-minutes` 的 `agentrun-v01-ci-*` PipelineRun，通过 Tekton ownerRef 释放临时 workspace PVC。dry-run 必须披露候选 PipelineRun、owned PVC、active mount 保护、local-path 实际估算 bytes 和 confirm 命令。默认保护最新完成的 PipelineRun，保留当前 CI/CD 状态证据。`cleanup-released-pvs` 是二次回收入口，只处理 `agentrun-ci`、`local-path`、`Delete` reclaim policy 的 `Released` PV；它不触碰 `agentrun-v01` runtime namespace、业务 PVC、Secret、registry storage 或 GitOps desired state。磁盘治理和 G14 safe-stop 规则见 `docs/reference/gc.md`。 涉及 AgentRun runner egress、`transientEnv` 或 Secret 不泄露的 closeout，必须用真实 `create/apply/send` 资源原语触发 `agentrun-v01` runner Job，再通过 `describe runnerjob/...`、`events run/...`、`logs session/...` 或必要的兼容 bridge 检查 runner job response、event/trace 和 Kubernetes Pod spec。通过证据应显示 proxy env 是否存在、`NO_PROXY` 是否包含 `hyueapi.com`/`.hyueapi.com`、短期 `HWLAB_API_KEY` 等 `transientEnv` 是否通过 per-job Secret 的 `valueFrom.secretKeyRef` 注入，以及 response/event 只输出 env name、Secret metadata 和 `valuesPrinted=false`。不得在 issue、trace 或 Pod spec 摘要中输出 Secret value。AgentRun 内部 SecretRef 合同以 AgentRun 仓库 `docs/reference/spec-v01-secret-distribution.md` 和 `docs/reference/spec-v01-runtime-assembly.md` 为权威；UniDesk 只记录验证入口和跨仓库归因。 @@ -115,7 +130,7 @@ UniDesk 指挥官新任务入口固定使用 `bun scripts/cli.ts agentrun get|de 资源原语和旧兼容 group 的默认 transport 是直连 AgentRun REST API，配置来源是 UniDesk 自有 YAML `config/agentrun.yaml`。鉴权可以复用 `HWLAB_API_KEY` 的环境变量/固定文件发现风格，但不得依赖 HWLAB runtime、HWLAB backend-core、HWLAB frontend 代理或 SSH official CLI；多一层转发会增加故障面，不能作为正式路径。`--raw` 只披露直连 AgentRun REST envelope 和必要的 `transport=direct-http`、`clientRole=render-only`、`configPath`、`baseUrl`、auth source/redacted metadata，不打印 token value。`agentrun control-plane ...` 和 `git-mirror ...` 仍属于 G14 source/runtime 运维控制路径，可以继续使用 UniDesk SSH capture bridge；这些控制面路径不得反向成为 queue/session 资源原语的默认 transport。 -AgentRun 公网 HTTPS 入口按 Sub2API 的 FRP+Caddy 模式维护：`agentrun-v01` runtime 仍保持 ClusterIP，AgentRun source branch 的 `deploy/deploy.json` 声明 G14 frpc，把 `agentrun-mgr.agentrun-v01.svc.cluster.local:8080` 暴露到 master `127.0.0.1:22880`；UniDesk `config/agentrun.yaml` 声明 `https://agentrun.74-48-78-17.nip.io/`、master frps allow port、master Caddy vhost 和 direct REST 鉴权。`bun scripts/cli.ts agentrun control-plane expose --confirm` 只负责补 master `frps` allow port 与 Caddy site，不在 AgentRun k3s 中创建 Ingress、NodePort、LoadBalancer、hostPort 或 HWLAB 转发层。 +AgentRun 公网 HTTPS 入口、FRP/Caddy edge、direct REST base URL 和鉴权来源都由 UniDesk `config/agentrun.yaml` 声明。YAML-only lane 不允许把这些部署选择写回 AgentRun source branch 的 `deploy/deploy.json`；AgentRun source repo 只保留应用代码、构建输入和 AgentRun 自身契约。`bun scripts/cli.ts agentrun control-plane expose --confirm` 只负责按 UniDesk YAML 补 edge 侧 allow port 与 Caddy site，不在 AgentRun k3s 中创建 Ingress、NodePort、LoadBalancer、hostPort 或 HWLAB 转发层。 AgentRun Queue 任务如果需要调用 UniDesk 维护桥，例如 `trans` / `unidesk-ssh`，长期契约以 AgentRun 仓库 `docs/reference/spec-v01-runtime-assembly.md` 和 `docs/reference/spec-v01-secret-distribution.md` 为准：调用方通过 `executionPolicy.secretScope.toolCredentials[].tool=unidesk-ssh` 请求 `UNIDESK_SSH_CLIENT_TOKEN` SecretRef；非敏感 endpoint 由 runner-job `transientEnv` 显式提供，或由 manager 受控默认值自动补齐。UniDesk bridge 提交 Queue payload 时不得在 prompt、payload 或 `transientEnv` 中携带 token，也不得使用 HWLAB runtime Web 入口冒充 UniDesk frontend。若 dispatcher 已正确请求 `unidesk-ssh` 但 trace 的 `runner-job-created.transientEnv.names` 没有 `UNIDESK_MAIN_SERVER_IP`、`UNIDESK_MAIN_SERVER_HOST` 或 `UNIDESK_FRONTEND_URL`，归为 AgentRun assembly 问题；若 endpoint env 已存在但 route denied/timeout，再按 UniDesk frontend/token scope 或 provider session 排查。 diff --git a/docs/reference/yaml-first-ops.md b/docs/reference/yaml-first-ops.md index b5cf3d42..ee71baee 100644 --- a/docs/reference/yaml-first-ops.md +++ b/docs/reference/yaml-first-ops.md @@ -32,6 +32,12 @@ Code may validate that YAML is present, typed, syntactically valid and renderabl External formats such as JSON, TOML, env files, Kubernetes YAML, Caddyfile, systemd units or app-specific config files may still be generated or consumed at the edge when the external tool requires them. They are inputs or rendered artifacts, not UniDesk desired-state truth. +## Service Deployment Declarations + +UniDesk-managed service deployment declarations must not live in service repository JSON such as `deploy.json`. A service repository may keep application source, build inputs, migrations, API/spec documentation and app-native runtime config required by the process. Node/lane selection, runtime namespace, image artifact selection, GitOps branch/path, public exposure, external database wiring, Secret mapping, service account, probes, rollout and cleanup settings belong in the owning UniDesk YAML and are rendered by UniDesk CLI. + +Generated GitOps YAML, image catalogs, env files, Kubernetes manifests or external tool config may be committed as rendered artifacts when the runtime requires them. They must carry enough provenance to point back to the owning YAML/source commit and must not become a second editable desired-state truth. + ## Architecture Layers YAML-first ops uses five layers. diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index fd2ebfb4..40812b86 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -10,6 +10,19 @@ export interface AgentRunGitMirrorRepositorySpec { readonly gitopsBranch?: string; } +export interface AgentRunSecretRef { + readonly namespace: string; + readonly name: string; + readonly key: string; +} + +export interface AgentRunLaneSecretSpec { + readonly id: string; + readonly sourceRef: string; + readonly sourceKey: string; + readonly targetRef: AgentRunSecretRef; +} + export interface AgentRunLaneSpec { readonly lane: string; readonly nodeId: string; @@ -19,6 +32,7 @@ export interface AgentRunLaneSpec { readonly source: { readonly repository: string; readonly branch: string; + readonly bootstrapFromBranch: string | null; readonly remote: string; readonly workspace: string; }; @@ -44,13 +58,51 @@ export interface AgentRunLaneSpec { readonly argoApplication: string; readonly repoURL: string; }; + readonly deployment: { + readonly format: "unidesk-yaml-only"; + readonly gitopsRoot: string; + readonly runtimeRenderDir: string; + readonly artifactCatalogPath: string; + readonly argocd: { + readonly project: string; + readonly applicationFile: string; + }; + readonly manager: { + readonly serviceAccount: string; + readonly apiKeySecretRef: { readonly name: string; readonly key: string }; + readonly unideskSshEndpointEnv: { readonly name: string; readonly value: string } | null; + readonly bootRepoUrl: string; + readonly imageBuild: AgentRunImageBuildSpec; + readonly resources: AgentRunContainerResources; + }; + readonly runner: { + readonly serviceAccount: string; + readonly jobNamePrefix: string; + readonly apiKeySecretRef: { readonly name: string; readonly key: string }; + }; + readonly localPostgres: { + readonly enabled: boolean; + readonly serviceName: string | null; + readonly image: string | null; + readonly storage: string | null; + readonly port: number | null; + }; + }; readonly gitMirror: { readonly namespace: string; readonly readService: string; + readonly readDeployment: string; readonly writeService: string; + readonly writeDeployment: string; readonly readUrl: string; readonly writeUrl: string; readonly cachePvc: string; + readonly cacheHostPath: string | null; + readonly sshSecretName: string; + readonly githubProxy: { + readonly host: string; + readonly port: number; + }; readonly toolsImage: string; readonly syncJobPrefix: string; readonly flushJobPrefix: string; @@ -67,6 +119,30 @@ export interface AgentRunLaneSpec { readonly secretRef: { readonly name: string; readonly key: string }; readonly localPostgresExpectedAbsent: boolean; }; + readonly secrets: readonly AgentRunLaneSecretSpec[]; +} + +export interface AgentRunContainerResources { + readonly requests: { + readonly cpu: string; + readonly memory: string; + }; + readonly limits: { + readonly cpu: string; + readonly memory: string; + }; +} + +export interface AgentRunImageBuildSpec { + readonly context: string; + readonly containerfile: string; + readonly repository: string; + readonly network: string; + readonly httpProxy: string | null; + readonly httpsProxy: string | null; + readonly noProxy: readonly string[]; + readonly envIdentityFiles: readonly string[]; + readonly timeoutSeconds: number; } export interface AgentRunLaneTarget { @@ -124,6 +200,7 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record ({ key: repo.key, repository: repo.repository, @@ -173,6 +287,13 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record ({ + id: secret.id, + sourceRef: secret.sourceRef.startsWith("/") ? secret.sourceRef : `.state/secrets/${secret.sourceRef}`, + sourceKey: secret.sourceKey, + targetRef: secret.targetRef, + valuesPrinted: false, + })), }; } @@ -233,6 +354,7 @@ function parseLane(lane: string, node: AgentRunNodeSpec, input: Record parseGitMirrorRepository(repo, `${path}.gitMirror.repositories[${index}]`)), }, database: parseDatabase(database, `${path}.database`), + secrets: arrayField(input, "secrets", path).map((secret, index) => parseLaneSecret(secret, `${path}.secrets[${index}]`)), + }; +} + +function parseDeployment(input: Record, path: string): AgentRunLaneSpec["deployment"] { + const argocd = recordField(input, "argocd", path); + const manager = recordField(input, "manager", path); + const runner = recordField(input, "runner", path); + const localPostgres = recordField(input, "localPostgres", path); + return { + format: enumField(input, "format", path, ["unidesk-yaml-only"]), + gitopsRoot: relativePathField(input, "gitopsRoot", path), + runtimeRenderDir: relativePathField(input, "runtimeRenderDir", path), + artifactCatalogPath: relativePathField(input, "artifactCatalogPath", path), + argocd: { + project: stringField(argocd, "project", `${path}.argocd`), + applicationFile: relativePathField(argocd, "applicationFile", `${path}.argocd`), + }, + manager: { + serviceAccount: stringField(manager, "serviceAccount", `${path}.manager`), + apiKeySecretRef: parseSecretRef(recordField(manager, "apiKeySecretRef", `${path}.manager`), `${path}.manager.apiKeySecretRef`), + unideskSshEndpointEnv: optionalEnvPair(manager, "unideskSshEndpointEnv", `${path}.manager`), + bootRepoUrl: urlField(manager, "bootRepoUrl", `${path}.manager`), + imageBuild: parseImageBuild(recordField(manager, "imageBuild", `${path}.manager`), `${path}.manager.imageBuild`), + resources: parseContainerResources(recordField(manager, "resources", `${path}.manager`), `${path}.manager.resources`), + }, + runner: { + serviceAccount: stringField(runner, "serviceAccount", `${path}.runner`), + jobNamePrefix: stringField(runner, "jobNamePrefix", `${path}.runner`), + apiKeySecretRef: parseSecretRef(recordField(runner, "apiKeySecretRef", `${path}.runner`), `${path}.runner.apiKeySecretRef`), + }, + localPostgres: parseLocalPostgres(localPostgres, `${path}.localPostgres`), + }; +} + +function parseLocalPostgres(input: Record, path: string): AgentRunLaneSpec["deployment"]["localPostgres"] { + const enabled = booleanField(input, "enabled", path); + if (!enabled) { + return { + enabled: false, + serviceName: optionalStringField(input, "serviceName", path) ?? null, + image: optionalStringField(input, "image", path) ?? null, + storage: optionalStringField(input, "storage", path) ?? null, + port: optionalIntegerField(input, "port", path) ?? null, + }; + } + return { + enabled: true, + serviceName: stringField(input, "serviceName", path), + image: stringField(input, "image", path), + storage: stringField(input, "storage", path), + port: integerField(input, "port", path), + }; +} + +function parseContainerResources(input: Record, path: string): AgentRunContainerResources { + const requests = recordField(input, "requests", path); + const limits = recordField(input, "limits", path); + return { + requests: { + cpu: stringField(requests, "cpu", `${path}.requests`), + memory: stringField(requests, "memory", `${path}.requests`), + }, + limits: { + cpu: stringField(limits, "cpu", `${path}.limits`), + memory: stringField(limits, "memory", `${path}.limits`), + }, + }; +} + +function parseImageBuild(input: Record, path: string): AgentRunImageBuildSpec { + return { + context: relativePathField(input, "context", path), + containerfile: relativePathField(input, "containerfile", path), + repository: stringField(input, "repository", path), + network: stringField(input, "network", path), + httpProxy: optionalStringField(input, "httpProxy", path) ?? null, + httpsProxy: optionalStringField(input, "httpsProxy", path) ?? null, + noProxy: stringArrayField(input, "noProxy", path), + envIdentityFiles: stringArrayField(input, "envIdentityFiles", path).map((item, index) => { + if (item.startsWith("/") || item.includes("..")) throw new Error(`${path}.envIdentityFiles[${index}] must be a relative path without ..`); + return item; + }), + timeoutSeconds: integerField(input, "timeoutSeconds", path), + }; +} + +function parseLaneSecret(input: Record, path: string): AgentRunLaneSecretSpec { + return { + id: stringField(input, "id", path), + sourceRef: secretSourceRefField(input, "sourceRef", path), + sourceKey: stringField(input, "sourceKey", path), + targetRef: parseNamespacedSecretRef(recordField(input, "targetRef", path), `${path}.targetRef`), }; } @@ -319,6 +541,31 @@ function parseSecretRef(input: Record, path: string): { name: s }; } +function parseNamespacedSecretRef(input: Record, path: string): AgentRunSecretRef { + return { + namespace: stringField(input, "namespace", path), + name: stringField(input, "name", path), + key: stringField(input, "key", path), + }; +} + +function parseGithubProxy(input: Record, path: string): { host: string; port: number } { + return { + host: stringField(input, "host", path), + port: integerField(input, "port", path), + }; +} + +function optionalEnvPair(obj: Record, key: string, path: string): { name: string; value: string } | null { + const value = obj[key]; + if (value === undefined || value === null) return null; + const record = asRecord(value, `${path}.${key}`); + return { + name: stringField(record, "name", `${path}.${key}`), + value: stringField(record, "value", `${path}.${key}`), + }; +} + function asRecord(value: unknown, path: string): Record { if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be a YAML object`); return value as Record; @@ -353,12 +600,28 @@ function integerField(obj: Record, key: string, path: string): return Number(value); } +function optionalIntegerField(obj: Record, key: string, path: string): number | undefined { + const value = obj[key]; + if (value === undefined || value === null) return undefined; + if (!Number.isInteger(value)) throw new Error(`${path}.${key} must be an integer when set`); + return Number(value); +} + function arrayField(obj: Record, key: string, path: string): Record[] { const value = obj[key]; if (!Array.isArray(value)) throw new Error(`${path}.${key} must be a YAML array`); return value.map((item, index) => asRecord(item, `${path}.${key}[${index}]`)); } +function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value)) throw new Error(`${path}.${key} must be a YAML array`); + return value.map((item, index) => { + if (typeof item !== "string" || item.trim().length === 0) throw new Error(`${path}.${key}[${index}] must be a non-empty string`); + return item.trim(); + }); +} + function enumField(obj: Record, key: string, path: string, values: readonly T[]): T { const value = stringField(obj, key, path); if (!values.includes(value as T)) throw new Error(`${path}.${key} must be one of ${values.join(", ")}`); @@ -371,12 +634,28 @@ function absolutePathField(obj: Record, key: string, path: stri return value; } +function optionalAbsolutePathField(obj: Record, key: string, path: string): string | undefined { + const value = obj[key]; + if (value === undefined || value === null) return undefined; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string when set`); + const trimmed = value.trim(); + if (!trimmed.startsWith("/") || trimmed.includes("..")) throw new Error(`${path}.${key} must be an absolute path without ..`); + return trimmed; +} + function relativePathField(obj: Record, key: string, path: string): string { const value = stringField(obj, key, path); if (value.startsWith("/") || value.includes("..")) throw new Error(`${path}.${key} must be a relative path without ..`); return value; } +function secretSourceRefField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (value.includes("..")) throw new Error(`${path}.${key} must not contain ..`); + if (!value.startsWith("/") && value.startsWith(".")) throw new Error(`${path}.${key} must be absolute or relative without a leading dot`); + return value; +} + function urlField(obj: Record, key: string, path: string): string { const value = stringField(obj, key, path); try { diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts new file mode 100644 index 00000000..bdd2d64c --- /dev/null +++ b/scripts/src/agentrun-manifests.ts @@ -0,0 +1,489 @@ +import { createHash } from "node:crypto"; +import type { AgentRunLaneSpec } from "./agentrun-lanes"; + +export interface AgentRunArtifactService { + readonly serviceId: string; + readonly image: string; + readonly digest: string; + readonly repositoryDigest: string; + readonly imageTag: string; + readonly artifactKind: string; + readonly status: string; + readonly envIdentity: string; + readonly envImage: string; + readonly envDigest: string; + readonly envRepositoryDigest: string; + readonly bootCommit: string; + readonly bootScript: string; + readonly provenance: Record; +} + +export interface AgentRunArtifactCatalog { + readonly lane: string; + readonly sourceBranch: string; + readonly gitopsBranch: string; + readonly sourceCommitId: string; + readonly summary: string; + readonly services: readonly AgentRunArtifactService[]; +} + +export interface AgentRunGitopsRenderInput { + readonly sourceCommit: string; + readonly image: AgentRunArtifactService; +} + +export interface AgentRunRenderedFile { + readonly path: string; + readonly content: string; +} + +export function renderAgentRunControlPlaneManifests(spec: AgentRunLaneSpec): readonly Record[] { + return [ + { apiVersion: "v1", kind: "Namespace", metadata: { name: spec.ci.namespace } }, + { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: spec.ci.serviceAccountName, + namespace: spec.ci.namespace, + labels: agentRunLabels(spec), + }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { + name: spec.ci.serviceAccountName, + namespace: spec.ci.namespace, + labels: agentRunLabels(spec), + }, + rules: [ + { apiGroups: ["tekton.dev"], resources: ["pipelineruns", "taskruns"], verbs: ["get", "list", "watch", "create", "patch", "update"] }, + { apiGroups: [""], resources: ["pods", "pods/log", "secrets", "configmaps", "persistentvolumeclaims"], verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { + name: spec.ci.serviceAccountName, + namespace: spec.ci.namespace, + labels: agentRunLabels(spec), + }, + subjects: [{ kind: "ServiceAccount", name: spec.ci.serviceAccountName }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: spec.ci.serviceAccountName }, + }, + agentRunPipelineManifest(spec), + agentRunArgoProjectManifest(spec), + agentRunArgoApplicationManifest(spec), + ]; +} + +export function renderAgentRunGitopsFiles(spec: AgentRunLaneSpec, input: AgentRunGitopsRenderInput): readonly AgentRunRenderedFile[] { + const catalog = agentRunArtifactCatalog(spec, input.sourceCommit, input.image); + const source = { + lane: spec.version, + sourceCommit: input.sourceCommit, + generatedBy: "unidesk config/agentrun.yaml", + configSource: "config/agentrun.yaml", + }; + return [ + { path: "source.json", content: `${JSON.stringify(source, null, 2)}\n` }, + { path: spec.deployment.artifactCatalogPath, content: `${JSON.stringify(catalog, null, 2)}\n` }, + { path: `${spec.deployment.gitopsRoot}/argocd/project.yaml`, content: yaml(agentRunArgoProjectManifest(spec)) }, + { path: `${spec.deployment.gitopsRoot}/argocd/${spec.deployment.argocd.applicationFile}`, content: yaml(agentRunArgoApplicationManifest(spec)) }, + { path: `${spec.deployment.gitopsRoot}/${spec.deployment.runtimeRenderDir}/kustomization.yaml`, content: yaml(agentRunKustomizationManifest(spec)) }, + { path: `${spec.deployment.gitopsRoot}/${spec.deployment.runtimeRenderDir}/namespace.yaml`, content: yaml(agentRunRuntimeNamespaceManifest(spec)) }, + ...(spec.deployment.localPostgres.enabled ? [{ path: `${spec.deployment.gitopsRoot}/${spec.deployment.runtimeRenderDir}/postgres.yaml`, content: yaml(agentRunPostgresManifest(spec)) }] : []), + { path: `${spec.deployment.gitopsRoot}/${spec.deployment.runtimeRenderDir}/mgr.yaml`, content: yamlAll(agentRunManagerManifests(spec, input.sourceCommit, input.image)) }, + { path: `${spec.deployment.gitopsRoot}/${spec.deployment.runtimeRenderDir}/runner-rbac.yaml`, content: yamlAll(agentRunRunnerRbacManifests(spec)) }, + ]; +} + +export function placeholderAgentRunImage(spec: AgentRunLaneSpec, sourceCommit: string): AgentRunArtifactService { + const digest = `sha256:${"0".repeat(64)}`; + const image = `${spec.ci.registryPrefix}/agentrun-mgr-env:${sourceCommit}`; + return { + serviceId: "agentrun-mgr", + artifactKind: "env-reuse", + status: "placeholder", + image, + digest, + repositoryDigest: `${spec.ci.registryPrefix}/agentrun-mgr-env@${digest}`, + imageTag: sourceCommit, + envIdentity: sourceCommit, + envImage: image, + envDigest: digest, + envRepositoryDigest: `${spec.ci.registryPrefix}/agentrun-mgr-env@${digest}`, + bootCommit: sourceCommit, + bootScript: "deploy/runtime/boot/agentrun-boot.sh", + provenance: { + sourceCommitId: sourceCommit, + source: "placeholder", + valuesPrinted: false, + }, + }; +} + +export function agentRunImageArtifact(spec: AgentRunLaneSpec, input: { + sourceCommit: string; + envIdentity: string; + digest: string; + status: string; +}): AgentRunArtifactService { + const image = `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}:${input.envIdentity}`; + return { + serviceId: "agentrun-mgr", + artifactKind: "env-reuse", + status: input.status, + image, + digest: input.digest, + repositoryDigest: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}@${input.digest}`, + imageTag: input.envIdentity, + envIdentity: input.envIdentity, + envImage: image, + envDigest: input.digest, + envRepositoryDigest: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}@${input.digest}`, + bootCommit: input.sourceCommit, + bootScript: "deploy/runtime/boot/agentrun-boot.sh", + provenance: { + sourceCommitId: input.sourceCommit, + source: "unidesk-yaml-only", + configSource: "config/agentrun.yaml", + valuesPrinted: false, + }, + }; +} + +export function renderedFilesDigest(files: readonly AgentRunRenderedFile[]): string { + const hash = createHash("sha256"); + for (const file of [...files].sort((left, right) => left.path.localeCompare(right.path))) { + hash.update(file.path); + hash.update("\0"); + hash.update(file.content); + hash.update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +export function renderedObjectsDigest(objects: readonly Record[]): string { + return `sha256:${createHash("sha256").update(yamlAll(objects)).digest("hex")}`; +} + +function agentRunPipelineManifest(spec: AgentRunLaneSpec): Record { + return { + apiVersion: "tekton.dev/v1", + kind: "Pipeline", + metadata: { + name: spec.ci.pipeline, + namespace: spec.ci.namespace, + labels: agentRunLabels(spec), + }, + spec: { + params: [ + { name: "git-url", type: "string", default: spec.source.remote }, + { name: "git-read-url", type: "string", default: spec.gitMirror.readUrl }, + { name: "git-write-url", type: "string", default: spec.gitMirror.writeUrl }, + { name: "source-branch", type: "string", default: spec.source.branch }, + { name: "gitops-branch", type: "string", default: spec.gitops.branch }, + { name: "revision", type: "string" }, + { name: "registry-prefix", type: "string", default: spec.ci.registryPrefix }, + { name: "tools-image", type: "string", default: spec.ci.toolsImage }, + ], + workspaces: [{ name: "source" }, { name: "git-ssh" }], + tasks: [ + gitopsSmokeTask(spec), + ], + }, + }; +} + +function gitopsSmokeTask(spec: AgentRunLaneSpec): Record { + return { + name: "render-smoke", + workspaces: [{ name: "source", workspace: "source" }], + taskSpec: { + params: [{ name: "revision" }, { name: "tools-image" }], + workspaces: [{ name: "source" }], + steps: [ + { + name: "render-smoke", + image: "$(params.tools-image)", + script: [ + "#!/bin/sh", + "set -eu", + "echo '{\"event\":\"agentrun-ci-render-smoke\",\"status\":\"placeholder\",\"reason\":\"unidesk-yaml-only-control-plane\",\"valuesPrinted\":false}'", + ].join("\n"), + }, + ], + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "tools-image", value: "$(params.tools-image)" }, + ], + when: [{ input: spec.deployment.format, operator: "in", values: ["unidesk-yaml-only"] }], + }; +} + +function agentRunArgoProjectManifest(spec: AgentRunLaneSpec): Record { + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "AppProject", + metadata: { + name: spec.deployment.argocd.project, + namespace: spec.gitops.argoNamespace, + labels: agentRunLabels(spec), + }, + spec: { + description: `AgentRun ${spec.version} GitOps lane`, + sourceRepos: [spec.gitops.repoURL, spec.source.remote], + destinations: [{ server: "https://kubernetes.default.svc", namespace: spec.runtime.namespace }], + clusterResourceWhitelist: [{ group: "", kind: "Namespace" }], + namespaceResourceWhitelist: [{ group: "*", kind: "*" }], + }, + }; +} + +function agentRunArgoApplicationManifest(spec: AgentRunLaneSpec): Record { + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "Application", + metadata: { + name: spec.gitops.argoApplication, + namespace: spec.gitops.argoNamespace, + labels: agentRunLabels(spec), + }, + spec: { + project: spec.deployment.argocd.project, + source: { + repoURL: spec.gitops.repoURL, + targetRevision: spec.gitops.branch, + path: spec.gitops.path, + }, + destination: { + server: "https://kubernetes.default.svc", + namespace: spec.runtime.namespace, + }, + syncPolicy: { + automated: { prune: false, selfHeal: true }, + syncOptions: ["CreateNamespace=true", "ApplyOutOfSyncOnly=true"], + }, + }, + }; +} + +function agentRunKustomizationManifest(spec: AgentRunLaneSpec): Record { + return { + apiVersion: "kustomize.config.k8s.io/v1beta1", + kind: "Kustomization", + resources: [ + "namespace.yaml", + ...(spec.deployment.localPostgres.enabled ? ["postgres.yaml"] : []), + "mgr.yaml", + "runner-rbac.yaml", + ], + }; +} + +function agentRunRuntimeNamespaceManifest(spec: AgentRunLaneSpec): Record { + return { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: spec.runtime.namespace, + labels: agentRunLabels(spec), + }, + }; +} + +function agentRunPostgresManifest(spec: AgentRunLaneSpec): Record { + const localPostgres = spec.deployment.localPostgres; + if (!localPostgres.enabled || localPostgres.serviceName === null || localPostgres.image === null || localPostgres.storage === null || localPostgres.port === null) { + throw new Error(`localPostgres is enabled for ${spec.version} without renderable YAML fields`); + } + const name = localPostgres.serviceName; + return { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "Service", + metadata: { name, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + spec: { selector: { "app.kubernetes.io/name": name }, ports: [{ name: "postgres", port: localPostgres.port, targetPort: "postgres" }] }, + }, + { + apiVersion: "apps/v1", + kind: "StatefulSet", + metadata: { name, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + spec: { + serviceName: name, + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": name } }, + template: { + metadata: { labels: { ...agentRunLabels(spec), "app.kubernetes.io/name": name } }, + spec: { + containers: [ + { + name: "postgres", + image: localPostgres.image, + ports: [{ name: "postgres", containerPort: localPostgres.port }], + }, + ], + }, + }, + volumeClaimTemplates: [ + { + metadata: { name: "data" }, + spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: localPostgres.storage } } }, + }, + ], + }, + }, + ], + }; +} + +function agentRunManagerManifests(spec: AgentRunLaneSpec, sourceCommit: string, image: AgentRunArtifactService): readonly Record[] { + const imageRef = image.envRepositoryDigest || image.repositoryDigest; + return [ + { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: spec.deployment.manager.serviceAccount, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) } }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: spec.runtime.managerService, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + spec: { + selector: { "app.kubernetes.io/name": spec.runtime.managerDeployment }, + ports: [{ name: "http", port: spec.runtime.managerPort, targetPort: "http" }], + }, + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: spec.runtime.managerDeployment, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": spec.runtime.managerDeployment } }, + template: { + metadata: { + labels: { ...agentRunLabels(spec), "app.kubernetes.io/name": spec.runtime.managerDeployment }, + annotations: { + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/source-commit": sourceCommit, + "agentrun.pikastech.local/env-identity": image.envIdentity, + }, + }, + spec: { + serviceAccountName: spec.deployment.manager.serviceAccount, + containers: [ + { + name: "mgr", + image: imageRef, + imagePullPolicy: "IfNotPresent", + ports: [{ name: "http", containerPort: 8080 }], + env: managerEnv(spec, sourceCommit, imageRef, image.envIdentity), + readinessProbe: { httpGet: { path: "/health/readiness", port: "http" } }, + livenessProbe: { httpGet: { path: "/health/live", port: "http" } }, + resources: spec.deployment.manager.resources, + }, + ], + }, + }, + }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: `${spec.deployment.manager.serviceAccount}-runner-job-controller`, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + rules: [ + { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "get", "list", "watch"] }, + { apiGroups: [""], resources: ["pods"], verbs: ["get", "list", "watch"] }, + { apiGroups: [""], resources: ["persistentvolumeclaims"], verbs: ["create", "get", "list", "watch", "delete"] }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: `${spec.deployment.manager.serviceAccount}-runner-job-controller`, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + subjects: [{ kind: "ServiceAccount", name: spec.deployment.manager.serviceAccount }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${spec.deployment.manager.serviceAccount}-runner-job-controller` }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: `${spec.deployment.manager.serviceAccount}-provider-secret-manager`, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + rules: [{ apiGroups: [""], resources: ["secrets"], verbs: ["create", "delete", "get", "list", "patch", "update"] }], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: `${spec.deployment.manager.serviceAccount}-provider-secret-manager`, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + subjects: [{ kind: "ServiceAccount", name: spec.deployment.manager.serviceAccount }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${spec.deployment.manager.serviceAccount}-provider-secret-manager` }, + }, + ]; +} + +function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: string, envIdentity: string): readonly Record[] { + return [ + { name: "AGENTRUN_LANE", value: spec.version }, + { name: "DATABASE_URL", valueFrom: { secretKeyRef: spec.database.secretRef } }, + { name: "AGENTRUN_SOURCE_COMMIT", value: sourceCommit }, + { name: "AGENTRUN_BOOT_COMMIT", value: sourceCommit }, + { name: "AGENTRUN_BOOT_MODE", value: "mgr" }, + { name: "AGENTRUN_BOOT_REPO_URL", value: spec.deployment.manager.bootRepoUrl }, + { name: "AGENTRUN_ENV_IDENTITY", value: envIdentity }, + { name: "AGENTRUN_RUNTIME_NAMESPACE", value: spec.runtime.namespace }, + { name: "AGENTRUN_INTERNAL_MGR_URL", value: spec.runtime.internalBaseUrl }, + { name: "AGENTRUN_RUNNER_IMAGE", value: imageRef }, + { name: "AGENTRUN_RUNNER_SERVICE_ACCOUNT", value: spec.deployment.runner.serviceAccount }, + { name: "AGENTRUN_API_KEY", valueFrom: { secretKeyRef: spec.deployment.manager.apiKeySecretRef } }, + ...(spec.deployment.manager.unideskSshEndpointEnv === null ? [] : [{ name: spec.deployment.manager.unideskSshEndpointEnv.name, value: spec.deployment.manager.unideskSshEndpointEnv.value }]), + ]; +} + +function agentRunRunnerRbacManifests(spec: AgentRunLaneSpec): readonly Record[] { + return [ + { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: spec.deployment.runner.serviceAccount, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) } }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: `${spec.deployment.runner.serviceAccount}-secret-reader`, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + rules: [{ apiGroups: [""], resources: ["secrets"], verbs: ["get"] }], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: `${spec.deployment.runner.serviceAccount}-secret-reader`, namespace: spec.runtime.namespace, labels: agentRunLabels(spec) }, + subjects: [{ kind: "ServiceAccount", name: spec.deployment.runner.serviceAccount }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: `${spec.deployment.runner.serviceAccount}-secret-reader` }, + }, + ]; +} + +function agentRunArtifactCatalog(spec: AgentRunLaneSpec, sourceCommit: string, image: AgentRunArtifactService): AgentRunArtifactCatalog { + return { + lane: spec.version, + sourceBranch: spec.source.branch, + gitopsBranch: spec.gitops.branch, + sourceCommitId: sourceCommit, + summary: image.status === "placeholder" ? "build=0 reuse=0 placeholder=1" : "build=1 reuse=0 placeholder=0", + services: [image], + }; +} + +function agentRunLabels(spec: AgentRunLaneSpec): Record { + return { + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + }; +} + +function yaml(value: unknown): string { + return `${Bun.YAML.stringify(value).trim()}\n`; +} + +function yamlAll(values: readonly unknown[]): string { + return `${values.map((value) => Bun.YAML.stringify(value).trim()).join("\n---\n")}\n`; +} diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index ddad76e7..4ba0fba0 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -14,6 +14,15 @@ import { resolveAgentRunLaneTarget, type AgentRunLaneSpec, } from "./agentrun-lanes"; +import { + agentRunImageArtifact, + placeholderAgentRunImage, + renderedFilesDigest, + renderedObjectsDigest, + renderAgentRunControlPlaneManifests, + renderAgentRunGitopsFiles, + type AgentRunArtifactService, +} from "./agentrun-manifests"; const g14SourceRoute = "G14:/root/agentrun-v01"; const g14K3sRoute = "G14:k3s"; @@ -66,9 +75,12 @@ export function agentRunHelp(): unknown { "bun scripts/cli.ts agentrun send session/ --aipod Artificer --prompt-stdin", "bun scripts/cli.ts agentrun explain task", "bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", + "bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", "bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", "bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --confirm", + "bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run", + "bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --confirm", "bun scripts/cli.ts agentrun control-plane status", "bun scripts/cli.ts agentrun control-plane status --full", "bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", @@ -106,6 +118,7 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str if (config === null) throw new Error("agentrun control-plane and git-mirror commands require UniDesk config"); if (group === "control-plane") { if (action === "plan") return await controlPlanePlan(config, parseStatusOptions(actionArgs)); + if (action === "apply") return await controlPlaneApply(config, parseLaneConfirmOptions(actionArgs)); if (action === "status") return await status(config, parseStatusOptions(actionArgs)); if (action === "secret-sync") return await secretSync(config, parseSecretSyncOptions(actionArgs)); if (action === "expose") return await exposeAgentRun(config, parseConfirmOptions(actionArgs)); @@ -232,11 +245,13 @@ function agentRunHelpText(args: string[]): string { return [ "Usage: bun scripts/cli.ts agentrun control-plane [options]", "", - "Actions: plan, status, secret-sync, expose, trigger-current, refresh, cleanup-runs, cleanup-released-pvs", + "Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runs, cleanup-released-pvs", "Examples:", " bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02", + " bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02", " bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 --dry-run", + " bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 --dry-run", " bun scripts/cli.ts agentrun control-plane status", " bun scripts/cli.ts agentrun control-plane status --pipeline-run agentrun-v01-ci-", " bun scripts/cli.ts agentrun control-plane expose --dry-run", @@ -1634,6 +1649,9 @@ function isRecord(value: unknown): value is Record { interface TriggerOptions { confirm: boolean; dryRun: boolean; + node: string | null; + lane: string | null; + wait: boolean; } interface ConfirmOptions { @@ -1646,6 +1664,11 @@ interface SecretSyncOptions extends ConfirmOptions { lane: string | null; } +interface LaneConfirmOptions extends ConfirmOptions { + node: string | null; + lane: string | null; +} + interface GitMirrorOptions extends ConfirmOptions { timeoutSeconds: number; wait: boolean; @@ -1759,7 +1782,34 @@ function parseStatusOptions(args: string[]): StatusOptions { function parseTriggerOptions(args: string[]): TriggerOptions { - return parseConfirmOptions(args); + const base = parseConfirmOptions(args); + let node: string | null = null; + let lane: string | null = null; + let wait = false; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--confirm" || arg === "--dry-run") continue; + if (arg === "--wait") { + wait = true; + continue; + } + if (arg === "--node") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); + node = value; + index += 1; + continue; + } + if (arg === "--lane") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); + lane = value; + index += 1; + continue; + } + throw new Error(`unsupported trigger-current option: ${arg}`); + } + return { ...base, node, lane, wait }; } function parseSecretSyncOptions(args: string[]): SecretSyncOptions { @@ -1788,6 +1838,33 @@ function parseSecretSyncOptions(args: string[]): SecretSyncOptions { return { ...base, node, lane }; } +function parseLaneConfirmOptions(args: string[]): LaneConfirmOptions { + const base = parseConfirmOptions(args); + let node: string | null = null; + let lane: string | null = null; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--confirm" || arg === "--dry-run") continue; + if (arg === "--node") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--node requires a value"); + node = value; + index += 1; + continue; + } + if (arg === "--lane") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--lane requires a value"); + lane = value; + index += 1; + continue; + } + throw new Error(`unsupported control-plane option: ${arg}`); + } + if (node === null && lane === null) throw new Error("control-plane apply requires --node and --lane"); + return { ...base, node, lane }; +} + function parseConfirmOptions(args: string[]): ConfirmOptions { if (args.includes("--confirm") && args.includes("--dry-run")) throw new Error("accepts only one of --confirm or --dry-run"); return { @@ -1882,6 +1959,60 @@ async function controlPlanePlan(_config: UniDeskConfig, options: StatusOptions): status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, + controlPlaneApply: `bun scripts/cli.ts agentrun control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --dry-run`, + }, + valuesPrinted: false, + }; +} + +async function controlPlaneApply(config: UniDeskConfig, options: LaneConfirmOptions): Promise> { + const { configPath, spec } = resolveAgentRunLaneTarget(options); + const objects = renderAgentRunControlPlaneManifests(spec); + const manifestYaml = `${objects.map((object) => Bun.YAML.stringify(object).trim()).join("\n---\n")}\n`; + const objectRefs = objects.map((object) => manifestObjectRef(object)); + const plan = { + node: spec.nodeId, + kubeRoute: spec.nodeKubeRoute, + lane: spec.lane, + version: spec.version, + objectCount: objects.length, + objects: objectRefs, + manifestBytes: Buffer.byteLength(manifestYaml, "utf8"), + manifestDigest: renderedObjectsDigest(objects), + fieldManager: "unidesk-agentrun-control-plane", + valuesPrinted: false, + }; + if (options.dryRun || !options.confirm) { + return { + ok: true, + command: "agentrun control-plane apply", + mode: "dry-run", + mutation: false, + configPath, + target: agentRunLaneSummary(spec), + plan, + next: { + confirm: `bun scripts/cli.ts agentrun control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; + } + const applied = await capture(config, spec.nodeKubeRoute, ["script", "--", applyYamlScript(manifestYaml, "unidesk-agentrun-control-plane", false)]); + const payload = captureJsonPayload(applied); + return { + ok: applied.exitCode === 0 && payload.ok !== false, + command: "agentrun control-plane apply", + mode: "confirmed-apply", + mutation: true, + configPath, + target: agentRunLaneSummary(spec), + plan, + result: payload, + capture: compactCapture(applied, { full: applied.exitCode !== 0, stdoutTailChars: 5000, stderrTailChars: 5000 }), + next: { + secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, valuesPrinted: false, }; @@ -2058,6 +2189,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar const argo = record(runtimePayload.argo); const manager = record(runtimePayload.manager); const database = record(runtimePayload.database); + const secrets = record(runtimePayload.secrets); const localPostgres = record(runtimePayload.localPostgres); const blockers = [ ...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]), @@ -2074,6 +2206,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar ...(manager.deploymentExists === true ? [] : ["manager-deployment-missing"]), ...(manager.serviceExists === true ? [] : ["manager-service-missing"]), ...(spec.database.mode === "external-postgres" && database.secretPresent !== true ? ["database-secret-missing"] : []), + ...(secrets.ready === true ? [] : ["lane-secret-missing"]), ...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []), ]; return { @@ -2111,6 +2244,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar namespaceExists: runtimePayload.runtimeNamespaceExists ?? false, manager, database, + secrets, localPostgres, }, }, @@ -2132,6 +2266,7 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`, postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, + secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, statusFull: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --full`, }, valuesPrinted: false, @@ -2140,28 +2275,31 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); - if (spec.database.secretSourceRef === null) { + const sources = collectLaneSecretSources(spec); + if (sources.length === 0) { return { ok: false, command: "agentrun control-plane secret-sync", mode: "yaml-declared-node-lane", configPath, target: agentRunLaneSummary(spec), - degradedReason: "database-secret-source-not-declared", + degradedReason: "lane-secret-sources-not-declared", valuesPrinted: false, }; } - const source = readSecretSourceValue(spec, spec.database.secretSourceRef, spec.database.secretRef.key); - const plan = { - namespace: spec.runtime.namespace, - secret: spec.database.secretRef.name, - key: spec.database.secretRef.key, - sourceRef: spec.database.secretSourceRef, - sourcePath: source.redactedPath, - fingerprint: source.fingerprint, - valueBytes: source.valueBytes, + const values = sources.map((source) => ({ spec: source, value: readSecretSourceValue(spec, source.sourceRef, source.sourceKey) })); + const plan = values.map(({ spec: item, value }) => ({ + id: item.id, + namespace: item.targetRef.namespace, + secret: item.targetRef.name, + key: item.targetRef.key, + sourceRef: item.sourceRef, + sourceKey: item.sourceKey, + sourcePath: value.redactedPath, + fingerprint: value.fingerprint, + valueBytes: value.valueBytes, valuesPrinted: false, - }; + })); if (options.dryRun || !options.confirm) { return { ok: true, @@ -2170,7 +2308,7 @@ async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Pr mutation: false, configPath, target: agentRunLaneSummary(spec), - plan, + plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, next: { confirm: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, @@ -2178,7 +2316,7 @@ async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Pr valuesPrinted: false, }; } - const result = await capture(config, spec.nodeKubeRoute, ["script", "--", secretSyncScript(spec, source.value)]); + const result = await capture(config, spec.nodeKubeRoute, ["script", "--", secretSyncScript(spec, values.map(({ spec: item, value }) => ({ targetRef: item.targetRef, value: value.value })))]); const payload = captureJsonPayload(result); return { ok: result.exitCode === 0 && payload.ok !== false, @@ -2187,7 +2325,7 @@ async function secretSync(config: UniDeskConfig, options: SecretSyncOptions): Pr mutation: true, configPath, target: agentRunLaneSummary(spec), - plan, + plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, result: payload, capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { @@ -2401,6 +2539,10 @@ function escapeRegExp(value: string): string { } async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): Promise> { + if (options.node !== null || options.lane !== null) { + const target = resolveAgentRunLaneTarget(options); + return await triggerCurrentYamlLane(config, options, target); + } const source = await capture(config, g14SourceRoute, ["script", "--", [ "set -u", "cd /root/agentrun-v01", @@ -2502,6 +2644,200 @@ async function triggerCurrent(config: UniDeskConfig, options: TriggerOptions): P }; } +async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { + const spec = target.spec; + const probe = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapProbeScript(spec)]); + const source = captureJsonPayload(probe); + const sourceCommit = stringOrNull(source.sourceCommit); + const remoteBranchExists = source.remoteBranchExists === true; + const pipelineRun = sourceCommit !== null && isGitSha(sourceCommit) ? agentRunPipelineRunName(spec, sourceCommit) : null; + const placeholderImage = sourceCommit === null ? null : placeholderAgentRunImage(spec, sourceCommit); + const renderedFiles = placeholderImage === null ? [] : renderAgentRunGitopsFiles(spec, { sourceCommit, image: placeholderImage }); + const plan = { + node: spec.nodeId, + lane: spec.lane, + version: spec.version, + source: { + workspace: spec.source.workspace, + remote: spec.source.remote, + branch: spec.source.branch, + bootstrapFromBranch: spec.source.bootstrapFromBranch, + remoteBranchExists, + sourceCommit, + }, + deploymentFormat: spec.deployment.format, + deploymentTruth: "config/agentrun.yaml", + removedServiceDeployJson: true, + pipelineRun, + imageBuild: { + repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`, + containerfile: spec.deployment.manager.imageBuild.containerfile, + timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds, + proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null, + }, + gitops: { + branch: spec.gitops.branch, + path: spec.gitops.path, + renderedFileCount: renderedFiles.length, + renderedFilesDigest: renderedFiles.length === 0 ? null : renderedFilesDigest(renderedFiles), + }, + valuesPrinted: false, + }; + if (options.dryRun || !options.confirm) { + return { + ok: true, + command: "agentrun control-plane trigger-current", + mode: "dry-run", + mutation: false, + configPath: target.configPath, + target: agentRunLaneSummary(spec), + plan, + sourceProbe: compactCapture(probe, { full: probe.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), + next: { + confirm: `bun scripts/cli.ts agentrun control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`, + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; + } + if (options.wait) { + return await triggerCurrentYamlLaneConfirmed(config, spec, target.configPath, true); + } + return startAsyncAgentRunJob( + `agentrun_${spec.lane}_trigger_current`, + ["bun", "scripts/cli.ts", "agentrun", "control-plane", "trigger-current", "--node", spec.nodeId, "--lane", spec.lane, "--confirm", "--wait"], + `Run AgentRun ${spec.version} YAML-only trigger-current on ${spec.nodeId}`, + ); +} + +async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { + const bootstrap = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapScript(spec)]); + const bootstrapPayload = captureJsonPayload(bootstrap); + const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit); + if (bootstrap.exitCode !== 0 || sourceCommit === null || !isGitSha(sourceCommit)) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-bootstrap", + degradedReason: "yaml-lane-source-bootstrap-failed", + result: bootstrapPayload, + capture: compactCapture(bootstrap, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const buildSubmit = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneBuildImageSubmitScript(spec, sourceCommit)]); + const buildSubmitPayload = captureJsonPayload(buildSubmit); + if (buildSubmit.exitCode !== 0 || buildSubmitPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "image-build-submit", + sourceCommit, + degradedReason: "yaml-lane-image-build-submit-failed", + sourceBootstrap: bootstrapPayload, + result: buildSubmitPayload, + capture: compactCapture(buildSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const build = await waitForYamlLaneBuildImage(config, spec, sourceCommit, stringOrNull(buildSubmitPayload.jobId)); + const buildPayload = build.payload; + const digest = stringOrNull(buildPayload.digest); + const envIdentity = stringOrNull(buildPayload.envIdentity); + if (build.ok !== true || digest === null || envIdentity === null) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "image-build", + sourceCommit, + degradedReason: "yaml-lane-image-build-failed", + sourceBootstrap: bootstrapPayload, + buildSubmit: buildSubmitPayload, + result: buildPayload, + buildStatus: build, + valuesPrinted: false, + }; + } + const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); + const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image }); + const gitops = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneGitopsPublishScript(spec, renderedFiles)]); + const gitopsPayload = captureJsonPayload(gitops); + if (gitops.exitCode !== 0 || gitopsPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "gitops-publish", + sourceCommit, + image, + degradedReason: "yaml-lane-gitops-publish-failed", + sourceBootstrap: bootstrapPayload, + imageBuild: buildPayload, + result: gitopsPayload, + capture: compactCapture(gitops, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const mirror = await runYamlLaneGitMirrorSyncJob(config, spec); + if (mirror.ok !== true) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "git-mirror-sync", + sourceCommit, + image, + gitops: gitopsPayload, + degradedReason: "yaml-lane-git-mirror-sync-failed", + result: mirror, + valuesPrinted: false, + }; + } + const pipelineRun = agentRunPipelineRunName(spec, sourceCommit); + const created = await capture(config, spec.nodeKubeRoute, ["script", "--", yamlLanePipelineRunCreateScript(spec, sourceCommit, pipelineRun)]); + const createPayload = captureJsonPayload(created); + return { + ok: created.exitCode === 0 && createPayload.ok !== false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + mutation: true, + configPath, + target: agentRunLaneSummary(spec), + sourceCommit, + pipelineRun, + image, + sourceBootstrap: bootstrapPayload, + imageBuildSubmit: buildSubmitPayload, + imageBuild: buildPayload, + renderedFiles: { + count: renderedFiles.length, + digest: renderedFilesDigest(renderedFiles), + }, + gitops: gitopsPayload, + gitMirror: mirror, + created: createPayload, + capture: compactCapture(created, { full: created.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }), + next: { + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + statusByPipelineRun: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, + }, + valuesPrinted: false, + }; +} + async function refresh(config: UniDeskConfig, options: ConfirmOptions): Promise> { const source = await capture(config, g14SourceRoute, ["script", "--", [ "cd /root/agentrun-v01", @@ -2738,7 +3074,8 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string `manager_service=${shQuote(spec.runtime.managerService)}`, `database_secret=${shQuote(spec.database.secretRef.name)}`, `database_key=${shQuote(spec.database.secretRef.key)}`, - "export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key", + `secrets_json=${shQuote(JSON.stringify(collectLaneSecretSources(spec).map((item) => item.targetRef)))}`, + "export runtime_namespace ci_namespace pipeline_name pipeline_run service_account argo_namespace argo_application manager_deployment manager_service database_secret database_key secrets_json", "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl get ns \"$runtime_namespace\" -o json > \"$tmp_dir/runtime-ns.json\" 2>/dev/null", @@ -2758,6 +3095,21 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string "manager_svc_exit=$?", "kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null", "db_secret_exit=$?", + "SECRET_REFS_JSON=\"$secrets_json\" NODE_TMP=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/secrets.json\"", + "const fs = require('node:fs');", + "const cp = require('node:child_process');", + "const refs = JSON.parse(process.env.SECRET_REFS_JSON || '[]');", + "const result = [];", + "for (const ref of refs) {", + " const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });", + " let keyPresent = false;", + " if (out.status === 0) {", + " try { keyPresent = Object.prototype.hasOwnProperty.call((JSON.parse(out.stdout).data || {}), ref.key); } catch {}", + " }", + " result.push({ namespace: ref.namespace, name: ref.name, key: ref.key, present: out.status === 0, keyPresent, valuesPrinted: false });", + "}", + "console.log(JSON.stringify({ ready: result.every((item) => item.present && item.keyPresent), count: result.length, items: result, valuesPrinted: false }));", + "NODE", "kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null", "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'", "const fs = require('node:fs');", @@ -2772,6 +3124,7 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string "const managerDeploy = readJson('manager-deploy.json');", "const managerSvc = readJson('manager-svc.json');", "const dbSecret = readJson('db-secret.json');", + "const secrets = readJson('secrets.json') || { ready: true, count: 0, items: [], valuesPrinted: false };", "let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}", "const c = condition(pipelineRun);", "console.log(JSON.stringify({", @@ -2784,6 +3137,7 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string " argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },", " manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },", " database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },", + " secrets,", " localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },", " valuesPrinted: false", "}));", @@ -2791,6 +3145,467 @@ function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: string ].join("\n"); } +function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string { + return [ + "set +e", + `workspace=${shQuote(spec.source.workspace)}`, + `remote=${shQuote(spec.source.remote)}`, + `branch=${shQuote(spec.source.branch)}`, + "workspace_exists=false", + "remote_branch_exists=false", + "source_commit=null", + "if [ -d \"$workspace/.git\" ]; then", + " workspace_exists=true", + " cd \"$workspace\"", + " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", + " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; else source_commit=$(git rev-parse HEAD 2>/dev/null || printf null); fi", + "else", + " remote_branch=$(git ls-remote --heads \"$remote\" \"$branch\" 2>/dev/null | awk '{print $1}' | head -n 1)", + " if [ -n \"$remote_branch\" ]; then remote_branch_exists=true; source_commit=\"$remote_branch\"; fi", + "fi", + "export workspace_exists remote_branch_exists source_commit", + "node <<'NODE'", + "console.log(JSON.stringify({ ok: true, workspaceExists: process.env.workspace_exists === 'true', remoteBranchExists: process.env.remote_branch_exists === 'true', sourceCommit: process.env.source_commit === 'null' ? null : process.env.source_commit, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +function yamlLaneSourceBootstrapScript(spec: AgentRunLaneSpec): string { + const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch; + return [ + "set -eu", + `workspace=${shQuote(spec.source.workspace)}`, + `remote=${shQuote(spec.source.remote)}`, + `branch=${shQuote(spec.source.branch)}`, + `bootstrap_branch=${shQuote(bootstrap)}`, + "mkdir -p \"$(dirname \"$workspace\")\"", + "if [ ! -d \"$workspace/.git\" ]; then", + " git clone --no-checkout \"$remote\" \"$workspace\"", + "fi", + "cd \"$workspace\"", + "git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", + "git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", + "if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", + " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", + "else", + " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", + "fi", + "if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", + "git add -A deploy/deploy.json 2>/dev/null || true", + "if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", + " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", + "fi", + "git push -u origin \"$branch\"", + "source_commit=$(git rev-parse HEAD)", + "status_short=$(git status --short)", + "SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" node <<'NODE'", + "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: string): string { + const build = spec.deployment.manager.imageBuild; + const noProxy = build.noProxy.join(","); + const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; + const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; + const script = [ + "set -eu", + `workspace=${shQuote(spec.source.workspace)}`, + `source_commit=${shQuote(sourceCommit)}`, + `state_dir=${shQuote(stateDir)}`, + `containerfile=${shQuote(build.containerfile)}`, + `context_dir=${shQuote(build.context)}`, + `image_repository=${shQuote(imageRepository)}`, + `network=${shQuote(build.network)}`, + `http_proxy_value=${build.httpProxy === null ? "''" : shQuote(build.httpProxy)}`, + `https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`, + `no_proxy_value=${shQuote(noProxy)}`, + `env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`, + "mkdir -p \"$state_dir\"", + "cd \"$workspace\"", + "git checkout \"$source_commit\"", + "env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" node <<'NODE'", + "const { createHash } = require('node:crypto');", + "const { readFileSync, existsSync } = require('node:fs');", + "const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');", + "const hash = createHash('sha256');", + "for (const file of files) { hash.update(file); hash.update('\\0'); if (existsSync(file)) hash.update(readFileSync(file)); hash.update('\\0'); }", + "process.stdout.write(hash.digest('hex').slice(0, 24));", + "NODE", + ")", + "job_id=\"${source_commit:0:12}-$env_identity\"", + "status_file=\"$state_dir/$job_id.json\"", + "stdout_file=\"$state_dir/$job_id.stdout.log\"", + "stderr_file=\"$state_dir/$job_id.stderr.log\"", + "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-2000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", + "const code = Number(process.env.CODE || 1);", + "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", + "NODE", + " fi; exit \"$code\"; }", + " trap write_failed_status EXIT", + " cd \"$workspace\"", + " image=\"$image_repository:$env_identity\"", + " args=\"--network $network\"", + " if [ -n \"$http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$http_proxy_value --build-arg http_proxy=$http_proxy_value\"; fi", + " if [ -n \"$https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$https_proxy_value --build-arg https_proxy=$https_proxy_value\"; fi", + " if [ -n \"$no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$no_proxy_value --build-arg no_proxy=$no_proxy_value\"; fi", + " if docker image inspect \"$image\" >/dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi", + " docker push \"$image\"", + " digest=$(docker inspect --format='{{index .RepoDigests 0}}' \"$image\" 2>/dev/null | sed 's/^.*@//' || true)", + " if [ -z \"$digest\" ]; then digest=$(curl -fsSI -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \"http://127.0.0.1:5000/v2/${image_repository#127.0.0.1:5000/}/manifests/$env_identity\" 2>/dev/null | awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' | tr -d '\\r' | head -n 1 || true); fi", + " STATUS=\"$build_status\" DIGEST=\"$digest\" JOB_ID=\"$job_id\" SOURCE_COMMIT=\"$source_commit\" ENV_IDENTITY=\"$env_identity\" IMAGE_REPOSITORY=\"$image_repository\" node <<'NODE' > \"$status_file\"", + "const digest = process.env.DIGEST || null;", + "console.log(JSON.stringify({ ok: Boolean(digest), status: process.env.STATUS || 'built', jobId: process.env.JOB_ID, sourceCommit: process.env.SOURCE_COMMIT, envIdentity: process.env.ENV_IDENTITY, image: `${process.env.IMAGE_REPOSITORY}:${process.env.ENV_IDENTITY}`, digest, repositoryDigest: digest ? `${process.env.IMAGE_REPOSITORY}@${digest}` : null, valuesPrinted: false }));", + "NODE", + " trap - EXIT", + ") >\"$stdout_file\" 2>\"$stderr_file\" &", + "pid=$!", + "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", + "NODE", + ].join("\n"); + return script; +} + +async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { + if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "build-job-id-missing", valuesPrinted: false } }; + const startedAt = Date.now(); + const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000; + let lastPayload: Record = {}; + let polls = 0; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probe = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneBuildImageStatusScript(spec, jobId)]); + const payload = captureJsonPayload(probe); + lastPayload = payload; + progressEvent("agentrun.yaml-lane.image-build.progress", { + node: spec.nodeId, + lane: spec.lane, + sourceCommit, + jobId, + polls, + status: stringOrNull(payload.status) ?? "unknown", + elapsedMs: Date.now() - startedAt, + }); + if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; + if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; + await sleep(15_000); + } + return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; +} + +function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): string { + const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; + return [ + "set +e", + `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, + `stdout_file=${shQuote(`${stateDir}/${jobId}.stdout.log`)}`, + `stderr_file=${shQuote(`${stateDir}/${jobId}.stderr.log`)}`, + "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", + "if [ -f \"$stderr_file\" ] && tail -n 20 \"$stderr_file\" | grep -Eq 'ERROR|error|failed|denied'; then :; fi", + ].join("\n"); +} + +function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string { + const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({ + path: file.path, + contentBase64: Buffer.from(file.content, "utf8").toString("base64"), + }))), "utf8").toString("base64"); + return [ + "set -eu", + `workspace=${shQuote(spec.source.workspace)}`, + `remote=${shQuote(spec.source.remote)}`, + `source_branch=${shQuote(spec.source.branch)}`, + `gitops_branch=${shQuote(spec.gitops.branch)}`, + `gitops_root=${shQuote(spec.deployment.gitopsRoot)}`, + `artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`, + `files_b64=${shQuote(filesB64)}`, + "cd \"$workspace\"", + "git fetch origin \"$gitops_branch\" || true", + "if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then", + " git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"", + "else", + " git checkout --orphan \"$gitops_branch\"", + " git rm -rf . >/dev/null 2>&1 || true", + "fi", + "git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true", + "rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json", + "FILES_B64=\"$files_b64\" node <<'NODE'", + "const fs = require('node:fs');", + "const path = require('node:path');", + "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", + "for (const file of files) {", + " const target = path.resolve(process.cwd(), file.path);", + " if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);", + " fs.mkdirSync(path.dirname(target), { recursive: true });", + " fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));", + "}", + "NODE", + "git add source.json \"$artifact_catalog\" \"$gitops_root\"", + "if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi", + "git push -u origin \"$gitops_branch\"", + "gitops_commit=$(git rev-parse HEAD)", + "git checkout \"$source_branch\" >/dev/null 2>&1 || true", + "CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise> { + const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); + const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName); + const created = await capture(config, spec.nodeKubeRoute, ["script", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); + if (created.exitCode !== 0) { + return { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false }; + } + const startedAt = Date.now(); + let polls = 0; + let lastProbe: SshCaptureResult | null = null; + while (Date.now() - startedAt < 300_000) { + polls += 1; + lastProbe = await capture(config, spec.nodeKubeRoute, ["script", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); + const payload = captureJsonPayload(lastProbe); + progressEvent("agentrun.yaml-lane.git-mirror.progress", { + node: spec.nodeId, + lane: spec.lane, + jobName, + polls, + succeeded: payload.succeeded === true, + failed: payload.failed === true, + elapsedMs: Date.now() - startedAt, + }); + if (payload.succeeded === true) return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, valuesPrinted: false }; + if (payload.failed === true) return { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; + await sleep(5_000); + } + return { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; +} + +function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string { + const manifest = { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + name: pipelineRun, + namespace: spec.ci.namespace, + labels: { + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/source-commit": sourceCommit, + "agentrun.pikastech.local/trigger": "unidesk-yaml-only", + }, + }, + spec: { + pipelineRef: { name: spec.ci.pipeline }, + taskRunTemplate: { + serviceAccountName: spec.ci.serviceAccountName, + podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 } }, + }, + params: [ + { name: "git-url", value: spec.source.remote }, + { name: "git-read-url", value: spec.gitMirror.readUrl }, + { name: "git-write-url", value: spec.gitMirror.writeUrl }, + { name: "source-branch", value: spec.source.branch }, + { name: "gitops-branch", value: spec.gitops.branch }, + { name: "revision", value: sourceCommit }, + { name: "registry-prefix", value: spec.ci.registryPrefix }, + { name: "tools-image", value: spec.ci.toolsImage }, + ], + workspaces: [ + { name: "source", emptyDir: {} }, + { name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName } }, + ], + }, + }; + const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + return [ + "set -eu", + `namespace=${shQuote(spec.ci.namespace)}`, + `pipeline_run=${shQuote(pipelineRun)}`, + `manifest_b64=${shQuote(manifestB64)}`, + "tmp=$(mktemp)", + "trap 'rm -f \"$tmp\"' EXIT", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"", + "existing_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", + "if [ \"$existing_status\" = False ]; then kubectl -n \"$namespace\" delete pipelinerun \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true; fi", + "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then created=false; else kubectl create -f \"$tmp\"; created=true; fi", + "status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", + "CREATED=\"$created\" STATUS=\"$status\" PIPELINE_RUN=\"$pipeline_run\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, pipelineRun: process.env.PIPELINE_RUN, created: process.env.CREATED === 'true', status: process.env.STATUS || null, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +function createYamlLaneJobScript(namespace: string, jobName: string, manifest: Record): string { + const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + return [ + "set -eu", + `namespace=${shQuote(namespace)}`, + `job=${shQuote(jobName)}`, + `manifest_b64=${shQuote(manifestB64)}`, + "tmp=$(mktemp)", + "trap 'rm -f \"$tmp\"' EXIT", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"", + "kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + "kubectl create -f \"$tmp\"", + "JOB=\"$job\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, jobName: process.env.JOB, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +function yamlLaneJobProbeScript(namespace: string, jobName: string): string { + return [ + "set +e", + `namespace=${shQuote(namespace)}`, + `job=${shQuote(jobName)}`, + "kubectl -n \"$namespace\" get job \"$job\" -o json > /tmp/agentrun-job.json 2>/dev/null", + "job_exit=$?", + "kubectl -n \"$namespace\" logs \"job/$job\" --tail=120 > /tmp/agentrun-job.log 2>/dev/null", + "JOB_EXIT=\"$job_exit\" JOB=\"$job\" node <<'NODE'", + "const fs = require('node:fs');", + "let job = null; try { job = JSON.parse(fs.readFileSync('/tmp/agentrun-job.json', 'utf8')); } catch {}", + "let log = ''; try { log = fs.readFileSync('/tmp/agentrun-job.log', 'utf8'); } catch {}", + "const succeeded = Number(job?.status?.succeeded || 0) > 0;", + "const failed = Number(job?.status?.failed || 0) > 0;", + "console.log(JSON.stringify({ ok: process.env.JOB_EXIT === '0', jobName: process.env.JOB, succeeded, failed, active: job?.status?.active || 0, logsTail: log.slice(-4000), valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +function yamlLaneGitMirrorCacheVolume(spec: AgentRunLaneSpec): Record { + if (spec.gitMirror.cacheHostPath !== null) { + return { name: "cache", hostPath: { path: spec.gitMirror.cacheHostPath, type: "DirectoryOrCreate" } }; + } + return { name: "cache", persistentVolumeClaim: { claimName: spec.gitMirror.cachePvc } }; +} + +function yamlLaneGitMirrorJobManifest(spec: AgentRunLaneSpec, action: "sync", name: string): Record { + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name, + namespace: spec.gitMirror.namespace, + labels: { + "app.kubernetes.io/name": "git-mirror", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/component": action, + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 600, + ttlSecondsAfterFinished: 3600, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "git-mirror", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/component": action, + }, + }, + spec: { + restartPolicy: "Never", + volumes: [ + yamlLaneGitMirrorCacheVolume(spec), + { name: "git-ssh", secret: { secretName: spec.gitMirror.sshSecretName, defaultMode: 0o400 } }, + ], + containers: [{ + name: action, + image: spec.gitMirror.toolsImage, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec", yamlLaneGitMirrorSyncShell(spec)], + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }], + }, + }, + }, + }; +} + +function yamlLaneGitMirrorSyncShell(spec: AgentRunLaneSpec): string { + return [ + "set -eu", + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "cat > /tmp/agentrun-github-proxy-connect.cjs <<'NODE_PROXY'", + "const net = require('node:net');", + "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", + "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", + "const targetPort = Number.parseInt(targetPortRaw || '', 10);", + "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);", + "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", + "let buffer = Buffer.alloc(0);", + "socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });", + "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", + "socket.on('error', () => process.exit(66));", + "function onData(chunk) {", + " buffer = Buffer.concat([buffer, chunk]);", + " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", + " if (headerEnd === -1 && buffer.length < 8192) return;", + " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", + " const statusLine = head.split('\\r\\n', 1)[0] || '';", + " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", + " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }", + " socket.off('data', onData);", + " socket.setTimeout(0);", + " const rest = buffer.slice(headerEnd + 4);", + " if (rest.length) process.stdout.write(rest);", + " process.stdin.pipe(socket);", + " socket.pipe(process.stdout);", + "}", + "socket.on('data', onData);", + "socket.on('close', () => process.exit(0));", + "NODE_PROXY", + "chmod 0700 /tmp/agentrun-github-proxy-connect.cjs", + `export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/agentrun-github-proxy-connect.cjs ${spec.gitMirror.githubProxy.host} ${spec.gitMirror.githubProxy.port} %h %p'`)}`, + `repository=${shQuote(spec.source.repository)}`, + `source_branch=${shQuote(spec.source.branch)}`, + `gitops_branch=${shQuote(spec.gitops.branch)}`, + "repo=\"/cache/${repository}.git\"", + "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", + "mkdir -p \"$(dirname \"$repo\")\"", + "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", + " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", + "else", + " rm -rf \"$repo\"", + " git init --bare \"$repo\"", + " git --git-dir=\"$repo\" remote add origin \"$remote\"", + "fi", + "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", + "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", + "git --git-dir=\"$repo\" config http.uploadpack true", + "git --git-dir=\"$repo\" config http.receivepack true", + "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", + "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", + "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", + "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", + "gitops_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\")", + "git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$gitops_sha\"", + "git --git-dir=\"$repo\" update-server-info", + "SOURCE_SHA=\"$source_sha\" GITOPS_SHA=\"$gitops_sha\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, localSource: process.env.SOURCE_SHA, localGitops: process.env.GITOPS_SHA, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { return [ "set +e", @@ -2828,21 +3643,27 @@ function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { } function readSecretSourceValue(spec: AgentRunLaneSpec, sourceRef: string, key: string): { redactedPath: string; value: string; valueBytes: number; fingerprint: string } { - if (sourceRef.startsWith("/") || sourceRef.includes("..")) throw new Error(`secret sourceRef must be relative without ..: ${sourceRef}`); - const secretRoot = resolveSecretSourceRoot(spec); - const sourcePath = join(secretRoot, ...sourceRef.split("/")); - if (!existsSync(sourcePath)) throw new Error(`secret source ${sourceRef} is missing; run platform-db postgres apply --config config/platform-db/postgres-pk01.yaml --confirm first`); + if (sourceRef.includes("..")) throw new Error(`secret sourceRef must not contain ..: ${sourceRef}`); + const sourcePath = sourceRef.startsWith("/") + ? sourceRef + : join(resolveSecretSourceRoot(spec), ...sourceRef.split("/")); + if (!existsSync(sourcePath)) throw new Error(`secret source ${sourceRef} is missing`); const values = parseEnvFile(readFileSync(sourcePath, "utf8")); const value = values.get(key); if (value === undefined || value.length === 0) throw new Error(`secret source ${sourceRef} is missing required key ${key}`); return { - redactedPath: `.state/secrets/${sourceRef}`, + redactedPath: sourceRef.startsWith("/") ? redactAbsoluteSecretPath(sourceRef) : `.state/secrets/${sourceRef}`, value, valueBytes: Buffer.byteLength(value, "utf8"), fingerprint: `sha256:${createHash("sha256").update(value).digest("hex")}`, }; } +function redactAbsoluteSecretPath(sourceRef: string): string { + const parts = sourceRef.split("/").filter(Boolean); + return parts.length === 0 ? "/" : `/${parts.slice(0, -1).join("/")}/`; +} + function resolveSecretSourceRoot(spec: AgentRunLaneSpec): string { if (spec.database.configRef === null) return rootPath(".state", "secrets"); const configPath = spec.database.configRef.startsWith("/") ? spec.database.configRef : rootPath(spec.database.configRef); @@ -2870,35 +3691,116 @@ function parseEnvFile(text: string): Map { return result; } -function secretSyncScript(spec: AgentRunLaneSpec, value: string): string { - const encoded = Buffer.from(value, "utf8").toString("base64"); +function collectLaneSecretSources(spec: AgentRunLaneSpec): Array<{ id: string; sourceRef: string; sourceKey: string; targetRef: { namespace: string; name: string; key: string } }> { + const result: Array<{ id: string; sourceRef: string; sourceKey: string; targetRef: { namespace: string; name: string; key: string } }> = []; + if (spec.database.secretSourceRef !== null) { + result.push({ + id: "database", + sourceRef: spec.database.secretSourceRef, + sourceKey: spec.database.secretRef.key, + targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: spec.database.secretRef.key }, + }); + } + for (const secret of spec.secrets) { + result.push({ + id: secret.id, + sourceRef: secret.sourceRef, + sourceKey: secret.sourceKey, + targetRef: secret.targetRef, + }); + } + const seen = new Set(); + return result.filter((item) => { + const key = `${item.targetRef.namespace}/${item.targetRef.name}/${item.targetRef.key}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ targetRef: { namespace: string; name: string; key: string }; value: string }>): string { + const encoded = Buffer.from(JSON.stringify(values.map((item) => ({ + targetRef: item.targetRef, + valueBase64: Buffer.from(item.value, "utf8").toString("base64"), + }))), "utf8").toString("base64"); return [ "set -eu", - `namespace=${shQuote(spec.runtime.namespace)}`, - `secret_name=${shQuote(spec.database.secretRef.name)}`, - `secret_key=${shQuote(spec.database.secretRef.key)}`, - `secret_value_b64=${shQuote(encoded)}`, + `payload_b64=${shQuote(encoded)}`, "tmp_dir=$(mktemp -d)", "trap 'rm -rf \"$tmp_dir\"' EXIT", - "secret_file=\"$tmp_dir/secret-value\"", - "printf '%s' \"$secret_value_b64\" | base64 -d > \"$secret_file\"", - "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply --server-side --field-manager=unidesk-agentrun-secret-sync -f - >/dev/null", - "kubectl -n \"$namespace\" create secret generic \"$secret_name\" --from-file=\"$secret_key=$secret_file\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-agentrun-secret-sync -f - >/dev/null", - "rm -f \"$secret_file\"", - "kubectl -n \"$namespace\" get secret \"$secret_name\" -o json > \"$tmp_dir/secret.json\"", - "NAMESPACE=\"$namespace\" SECRET_NAME=\"$secret_name\" SECRET_KEY=\"$secret_key\" SECRET_JSON=\"$tmp_dir/secret.json\" node <<'NODE'", + "payload_json=\"$tmp_dir/payload.json\"", + "printf '%s' \"$payload_b64\" | base64 -d > \"$payload_json\"", + "PAYLOAD_JSON=\"$payload_json\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", "const fs = require('node:fs');", + "const cp = require('node:child_process');", "const crypto = require('node:crypto');", - "const secret = JSON.parse(fs.readFileSync(process.env.SECRET_JSON, 'utf8'));", - "const data = secret.data || {};", - "const raw = data[process.env.SECRET_KEY] || '';", - "const bytes = raw ? Buffer.from(raw, 'base64').length : 0;", - "const fingerprint = raw ? 'sha256:' + crypto.createHash('sha256').update(Buffer.from(raw, 'base64')).digest('hex') : null;", - "console.log(JSON.stringify({ ok: Boolean(raw), namespace: process.env.NAMESPACE, secret: process.env.SECRET_NAME, key: process.env.SECRET_KEY, valueBytes: bytes, fingerprint, valuesPrinted: false }));", + "const items = JSON.parse(fs.readFileSync(process.env.PAYLOAD_JSON, 'utf8'));", + "const results = [];", + "function run(argv, input) {", + " const out = cp.spawnSync(argv[0], argv.slice(1), { input, encoding: 'utf8' });", + " if (out.status !== 0) throw new Error(`${argv.join(' ')} failed: ${out.stderr || out.stdout}`);", + " return out;", + "}", + "for (let index = 0; index < items.length; index += 1) {", + " const item = items[index];", + " const ref = item.targetRef;", + " const value = Buffer.from(item.valueBase64, 'base64');", + " const file = `${process.env.TMP_DIR}/secret-${index}`;", + " fs.writeFileSync(file, value);", + " const ns = run(['kubectl', 'create', 'namespace', ref.namespace, '--dry-run=client', '-o', 'yaml']).stdout;", + " run(['kubectl', 'apply', '--server-side', '--field-manager=unidesk-agentrun-secret-sync', '-f', '-'], ns);", + " const secret = run(['kubectl', '-n', ref.namespace, 'create', 'secret', 'generic', ref.name, `--from-file=${ref.key}=${file}`, '--dry-run=client', '-o', 'yaml']).stdout;", + " run(['kubectl', 'apply', '--server-side', '--force-conflicts', '--field-manager=unidesk-agentrun-secret-sync', '-f', '-'], secret);", + " const fetched = JSON.parse(run(['kubectl', '-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json']).stdout);", + " const raw = fetched.data?.[ref.key] || '';", + " const decoded = raw ? Buffer.from(raw, 'base64') : Buffer.alloc(0);", + " results.push({ namespace: ref.namespace, secret: ref.name, key: ref.key, ok: raw.length > 0, valueBytes: decoded.length, fingerprint: raw ? 'sha256:' + crypto.createHash('sha256').update(decoded).digest('hex') : null, valuesPrinted: false });", + "}", + "console.log(JSON.stringify({ ok: results.every((item) => item.ok), secretCount: results.length, items: results, valuesPrinted: false }));", "NODE", ].join("\n"); } +function applyYamlScript(yaml: string, fieldManager: string, dryRun: boolean): string { + const encoded = Buffer.from(yaml, "utf8").toString("base64"); + return [ + "set -eu", + `manifest_b64=${shQuote(encoded)}`, + `field_manager=${shQuote(fieldManager)}`, + `dry_run=${dryRun ? "true" : "false"}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "manifest=\"$tmp_dir/manifest.yaml\"", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest\"", + "args=\"--server-side --force-conflicts --field-manager=$field_manager\"", + "if [ \"$dry_run\" = true ]; then args=\"$args --dry-run=server\"; fi", + "set +e", + "kubectl apply $args -f \"$manifest\" > \"$tmp_dir/apply.out\" 2> \"$tmp_dir/apply.err\"", + "apply_exit=$?", + "set -e", + "APPLY_EXIT=\"$apply_exit\" APPLY_OUT=\"$tmp_dir/apply.out\" APPLY_ERR=\"$tmp_dir/apply.err\" MANIFEST=\"$manifest\" node <<'NODE'", + "const fs = require('node:fs');", + "const crypto = require('node:crypto');", + "const out = fs.readFileSync(process.env.APPLY_OUT, 'utf8');", + "const err = fs.readFileSync(process.env.APPLY_ERR, 'utf8');", + "const manifest = fs.readFileSync(process.env.MANIFEST, 'utf8');", + "const resources = out.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean).slice(0, 80);", + "console.log(JSON.stringify({ ok: process.env.APPLY_EXIT === '0', exitCode: Number(process.env.APPLY_EXIT), resourceCount: resources.length, resources, manifestBytes: Buffer.byteLength(manifest, 'utf8'), manifestDigest: 'sha256:' + crypto.createHash('sha256').update(manifest).digest('hex'), stderrTail: err.slice(-3000), valuesPrinted: false }));", + "NODE", + "exit \"$apply_exit\"", + ].join("\n"); +} + +function manifestObjectRef(object: Record): Record { + const metadata = record(object.metadata); + return { + apiVersion: stringOrNull(object.apiVersion), + kind: stringOrNull(object.kind), + namespace: stringOrNull(metadata.namespace), + name: stringOrNull(metadata.name), + }; +} + function cleanupRunsPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); From 1a6b6e3d1f2824d6d944b7d240998c7ab7c9cf54 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 06:15:37 +0000 Subject: [PATCH 10/13] fix: poll AgentRun source bootstrap --- config/agentrun.yaml | 6 ++ scripts/src/agentrun-lanes.ts | 9 +++ scripts/src/agentrun.ts | 143 +++++++++++++++++++++++++++------- 3 files changed, 130 insertions(+), 28 deletions(-) diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 769e07c2..7efbe8fd 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -50,6 +50,8 @@ controlPlane: repository: pikasTech/agentrun branch: v0.1 bootstrapFromBranch: v0.1 + bootstrapTimeoutSeconds: 900 + bootstrapPollSeconds: 15 remote: git@github.com:pikasTech/agentrun.git workspace: /root/agentrun-v01 runtime: @@ -115,6 +117,7 @@ controlPlane: - bun.lock - tsconfig.json timeoutSeconds: 1800 + pollSeconds: 15 resources: requests: cpu: 100m @@ -177,6 +180,8 @@ controlPlane: repository: pikasTech/agentrun branch: v0.2 bootstrapFromBranch: v0.1 + bootstrapTimeoutSeconds: 900 + bootstrapPollSeconds: 15 remote: git@github.com:pikasTech/agentrun.git workspace: /home/ubuntu/workspace/agentrun-v02 runtime: @@ -242,6 +247,7 @@ controlPlane: - bun.lock - tsconfig.json timeoutSeconds: 1800 + pollSeconds: 15 resources: requests: cpu: 100m diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 40812b86..f4014c74 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -33,6 +33,8 @@ export interface AgentRunLaneSpec { readonly repository: string; readonly branch: string; readonly bootstrapFromBranch: string | null; + readonly bootstrapTimeoutSeconds: number; + readonly bootstrapPollSeconds: number; readonly remote: string; readonly workspace: string; }; @@ -143,6 +145,7 @@ export interface AgentRunImageBuildSpec { readonly noProxy: readonly string[]; readonly envIdentityFiles: readonly string[]; readonly timeoutSeconds: number; + readonly pollSeconds: number; } export interface AgentRunLaneTarget { @@ -201,6 +204,8 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record, path: string): AgentRun return item; }), timeoutSeconds: integerField(input, "timeoutSeconds", path), + pollSeconds: integerField(input, "pollSeconds", path), }; } diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 4ba0fba0..27747841 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2662,6 +2662,8 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt remote: spec.source.remote, branch: spec.source.branch, bootstrapFromBranch: spec.source.bootstrapFromBranch, + bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds, + bootstrapPollSeconds: spec.source.bootstrapPollSeconds, remoteBranchExists, sourceCommit, }, @@ -2673,6 +2675,7 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`, containerfile: spec.deployment.manager.imageBuild.containerfile, timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds, + pollSeconds: spec.deployment.manager.imageBuild.pollSeconds, proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null, }, gitops: { @@ -2711,10 +2714,31 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt } async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { - const bootstrap = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapScript(spec)]); - const bootstrapPayload = captureJsonPayload(bootstrap); + progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { + node: spec.nodeId, + lane: spec.lane, + status: "submitting", + }); + const bootstrapSubmit = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapSubmitScript(spec)]); + const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit); + if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-bootstrap-submit", + degradedReason: "yaml-lane-source-bootstrap-submit-failed", + result: bootstrapSubmitPayload, + capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId)); + const bootstrapPayload = bootstrap.payload; const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit); - if (bootstrap.exitCode !== 0 || sourceCommit === null || !isGitSha(sourceCommit)) { + if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) { return { ok: false, command: "agentrun control-plane trigger-current", @@ -2724,7 +2748,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen phase: "source-bootstrap", degradedReason: "yaml-lane-source-bootstrap-failed", result: bootstrapPayload, - capture: compactCapture(bootstrap, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + bootstrapStatus: bootstrap, valuesPrinted: false, }; } @@ -3170,37 +3194,100 @@ function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string { ].join("\n"); } -function yamlLaneSourceBootstrapScript(spec: AgentRunLaneSpec): string { +function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string { const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch; + const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, `bootstrap_branch=${shQuote(bootstrap)}`, - "mkdir -p \"$(dirname \"$workspace\")\"", - "if [ ! -d \"$workspace/.git\" ]; then", - " git clone --no-checkout \"$remote\" \"$workspace\"", - "fi", - "cd \"$workspace\"", - "git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", - "git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", - "if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", - " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", - "else", - " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", - "fi", - "if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", - "git add -A deploy/deploy.json 2>/dev/null || true", - "if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", - " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", - "fi", - "git push -u origin \"$branch\"", - "source_commit=$(git rev-parse HEAD)", - "status_short=$(git status --short)", - "SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" node <<'NODE'", - "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", + `state_dir=${shQuote(stateDir)}`, + "mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"", + "job_id=\"source-bootstrap-$(date +%s)-$$\"", + "status_file=\"$state_dir/$job_id.json\"", + "stdout_file=\"$state_dir/$job_id.stdout.log\"", + "stderr_file=\"$state_dir/$job_id.stderr.log\"", + "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", + "const code = Number(process.env.CODE || 1);", + "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", + " fi; exit \"$code\"; }", + " trap write_failed_status EXIT", + " if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", + " :", + " else", + " rm -rf \"$workspace\"", + " git clone --no-checkout \"$remote\" \"$workspace\"", + " fi", + " cd \"$workspace\"", + " git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", + " git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", + " if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", + " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", + " else", + " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", + " fi", + " if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", + " git add -A deploy/deploy.json 2>/dev/null || true", + " if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", + " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", + " fi", + " git push -u origin \"$branch\"", + " source_commit=$(git rev-parse HEAD)", + " status_short=$(git status --short)", + " SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", + "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", + "NODE", + " trap - EXIT", + ") >\"$stdout_file\" 2>\"$stderr_file\" &", + "pid=$!", + "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise & { ok: boolean; payload: Record }> { + if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } }; + const startedAt = Date.now(); + const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000; + const pollMs = spec.source.bootstrapPollSeconds * 1000; + let lastPayload: Record = {}; + let polls = 0; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probe = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]); + const payload = captureJsonPayload(probe); + lastPayload = payload; + progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { + node: spec.nodeId, + lane: spec.lane, + jobId, + polls, + status: stringOrNull(payload.status) ?? "unknown", + sourceCommit: stringOrNull(payload.sourceCommit), + elapsedMs: Date.now() - startedAt, + }); + if (payload.ok === true && stringOrNull(payload.sourceCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; + if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; + await sleep(pollMs); + } + return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; +} + +function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string { + const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; + return [ + "set +e", + `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, + "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); } @@ -3295,7 +3382,7 @@ async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLa }); if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; - await sleep(15_000); + await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; } From f77abd67800711fd9800cdbbcd6a2fcf10fe3a4a Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 06:21:12 +0000 Subject: [PATCH 11/13] fix: use POSIX source commit slicing --- scripts/src/agentrun.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 27747841..7ccfdae2 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -3321,7 +3321,8 @@ function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: st "process.stdout.write(hash.digest('hex').slice(0, 24));", "NODE", ")", - "job_id=\"${source_commit:0:12}-$env_identity\"", + "source_short=$(printf '%s' \"$source_commit\" | cut -c1-12)", + "job_id=\"$source_short-$env_identity\"", "status_file=\"$state_dir/$job_id.json\"", "stdout_file=\"$state_dir/$job_id.stdout.log\"", "stderr_file=\"$state_dir/$job_id.stderr.log\"", From 5727fe275c934555f0b51bf50ba82ba5bb572191 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:25:33 +0800 Subject: [PATCH 12/13] fix: support D601 v03 PK01 postgres secrets (#324) Co-authored-by: Codex --- config/platform-db/postgres-pk01.yaml | 91 ++++++++++++++++++++++ scripts/src/hwlab-node.ts | 104 ++++++++++++++++++++++---- 2 files changed, 179 insertions(+), 16 deletions(-) diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index 5434c07f..031c5788 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -230,6 +230,36 @@ postgres: user: agentrun_v02 address: 74.48.78.17/32 method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: postgres + user: hwlab_d601_v03_app + address: 10.0.8.0/22 + method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 36.49.29.73/32 + method: scram-sha-256 + - type: hostssl + database: postgres + user: hwlab_d601_v03_app + address: 36.49.29.73/32 + method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 74.48.78.17/32 + method: scram-sha-256 + - type: hostssl + database: postgres + user: hwlab_d601_v03_app + address: 74.48.78.17/32 + method: scram-sha-256 secrets: source: master-local @@ -291,6 +321,20 @@ secrets: AGENTRUN_V02_DB_NAME: agentrun_v02 randomHex: AGENTRUN_V02_DB_PASSWORD: 32 + - name: hwlab-d601-v03-db-credentials + sourceRef: platform-db/hwlab-d601-v03-db.env + type: env + requiredKeys: + - HWLAB_D601_V03_DB_USER + - HWLAB_D601_V03_DB_PASSWORD + - HWLAB_D601_V03_DB_NAME + createIfMissing: + enabled: true + values: + HWLAB_D601_V03_DB_USER: hwlab_d601_v03_app + HWLAB_D601_V03_DB_NAME: hwlab_d601_v03 + randomHex: + HWLAB_D601_V03_DB_PASSWORD: 32 objects: roles: @@ -330,6 +374,15 @@ objects: createdb: false createrole: false superuser: false + - name: hwlab_d601_v03_app + passwordRef: + sourceRef: platform-db/hwlab-d601-v03-db.env + key: HWLAB_D601_V03_DB_PASSWORD + login: true + attributes: + createdb: false + createrole: false + superuser: false databases: - name: sub2api owner: sub2api @@ -351,6 +404,11 @@ objects: encoding: UTF8 locale: C.UTF-8 extensions: [] + - name: hwlab_d601_v03 + owner: hwlab_d601_v03_app + encoding: UTF8 + locale: C.UTF-8 + extensions: [] exports: connectionStrings: @@ -414,6 +472,36 @@ exports: - scope: agentrun-v02 secret: agentrun-v02-mgr-db key: DATABASE_URL + - name: hwlab-d601-v03-cloud-api-database-url + sourceSecretRef: platform-db/hwlab-d601-v03-db.env + render: + envKey: DATABASE_URL + format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: hwlab/d601-v03-cloud-api-db.env + key: DATABASE_URL + mode: update-or-insert + consumers: + - scope: hwlab-v03 + secret: hwlab-cloud-api-v03-db + key: DATABASE_URL + - name: hwlab-d601-v03-openfga-datastore-uri + sourceSecretRef: platform-db/hwlab-d601-v03-db.env + render: + envKey: DATASTORE_URI + format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require&uselibpqcompat=true + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: hwlab/d601-v03-openfga-db.env + key: DATASTORE_URI + mode: update-or-insert + consumers: + - scope: hwlab-v03 + secret: hwlab-v03-openfga + key: DATASTORE_URI backup: phase: minimum-restoreable @@ -453,6 +541,9 @@ observability: - kind: psql-app-role database: agentrun_v02 user: agentrun_v02 + - kind: psql-app-role + database: hwlab_d601_v03 + user: hwlab_d601_v03_app - kind: disk-free path: /var/lib/postgresql/16/main minFreeGiB: 10 diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 332a9e55..2e6bf429 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -47,7 +47,11 @@ interface RuntimeSecretSpec { lane: string; namespace: string; platformDb: boolean; + runtimeLaneSpec?: HwlabRuntimeLaneSpec; + externalPostgres?: NonNullable; platformPostgresService: string; + platformPostgresEndpointAddress?: string; + platformPostgresEndpointSlice: string; postgresSecret: string; postgresStatefulSet: string; postgresAdminUser: string; @@ -2959,7 +2963,7 @@ function parseSecretOptions(args: string[]): NodeSecretOptions { } if (name === spec.cloudApiDbSecret) { if (key !== undefined && key !== spec.cloudApiDbKey) throw new Error(`secret ${name} supports only key ${spec.cloudApiDbKey}`); - if (actionRaw === "ensure" && spec.platformDb) { + if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { @@ -2980,7 +2984,7 @@ function parseSecretOptions(args: string[]): NodeSecretOptions { if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) { throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`); } - if (actionRaw === "ensure" && spec.platformDb) { + if (actionRaw === "ensure" && spec.platformDb && spec.externalPostgres === undefined) { throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`); } return { @@ -2998,32 +3002,43 @@ function parseSecretOptions(args: string[]): NodeSecretOptions { function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecretSpec { const namespace = `hwlab-${input.lane}`; - const platformDb = /^v0*[3-9]\d*$/.test(input.lane); - const platformPostgresService = "g14-platform-postgres"; + const runtimeLaneSpec = isHwlabRuntimeLane(input.lane) ? hwlabRuntimeLaneSpecForNode(input.lane, input.node) : undefined; + const externalPostgres = runtimeLaneSpec?.externalPostgres; + const platformDb = externalPostgres !== undefined || /^v0*[3-9]\d*$/.test(input.lane); + const platformPostgresService = externalPostgres?.serviceName ?? "g14-platform-postgres"; const legacyPostgresHost = `${namespace}-postgres.${namespace}.svc.cluster.local`; const platformPostgresHost = `${platformPostgresService}.${namespace}.svc.cluster.local`; + const platformPostgresEndpointSlice = `${platformPostgresService}-host`; + const openFgaDbName = externalPostgres?.database ?? (platformDb ? `openfga_${input.lane}` : "hwlab_openfga"); + const openFgaDbUser = externalPostgres?.openfga.role ?? (platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga"); + const cloudApiDbName = externalPostgres?.database ?? `hwlab_${input.lane}`; + const cloudApiDbUser = externalPostgres?.cloudApi.role ?? (platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`); return { node: input.node, lane: input.lane, namespace, platformDb, + ...(runtimeLaneSpec === undefined ? {} : { runtimeLaneSpec }), + ...(externalPostgres === undefined ? {} : { externalPostgres }), platformPostgresService, + platformPostgresEndpointAddress: externalPostgres?.endpointAddress, + platformPostgresEndpointSlice, postgresSecret: `${namespace}-postgres`, postgresStatefulSet: `${namespace}-postgres`, postgresAdminUser: `hwlab_${input.lane}`, - openFgaSecret: `${namespace}-openfga`, - openFgaDbName: platformDb ? `openfga_${input.lane}` : "hwlab_openfga", - openFgaDbUser: platformDb ? `openfga_${input.lane}_app` : "hwlab_openfga", + openFgaSecret: externalPostgres?.openfga.secretName ?? `${namespace}-openfga`, + openFgaDbName, + openFgaDbUser, openFgaDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, masterAdminApiKeySecret: `${namespace}-master-server-admin-api-key`, bootstrapAdminSecret: `${namespace}-bootstrap-admin`, bootstrapAdminPasswordHashKey: BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY, bootstrapAdminSourceNamespace: BOOTSTRAP_ADMIN_SOURCE_NAMESPACE, bootstrapAdminSourceSecret: BOOTSTRAP_ADMIN_SOURCE_SECRET, - cloudApiDbSecret: `hwlab-cloud-api-${input.lane}-db`, - cloudApiDbKey: CLOUD_API_DB_KEY, - cloudApiDbName: `hwlab_${input.lane}`, - cloudApiDbUser: platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`, + cloudApiDbSecret: externalPostgres?.cloudApi.secretName ?? `hwlab-cloud-api-${input.lane}-db`, + cloudApiDbKey: externalPostgres?.cloudApi.secretKey ?? CLOUD_API_DB_KEY, + cloudApiDbName, + cloudApiDbUser, cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost, cloudApiDeployment: "hwlab-cloud-api", obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`, @@ -3039,6 +3054,9 @@ function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecret function runNodeSecret(options: NodeSecretOptions): Record { const spec = runtimeSecretSpec(options); if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec); + if (spec.externalPostgres !== undefined && options.action === "ensure" && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return runExternalPostgresSecretEnsure(options, spec); + } const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun ? readMasterAdminApiKey().key : ""; @@ -3084,6 +3102,9 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): if (options.action === "cleanup-obsolete") { return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` }; } + if (spec.externalPostgres !== undefined && (options.preset === "cloud-api-db" || options.preset === "openfga")) { + return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; + } if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return { status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`, @@ -3093,6 +3114,50 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): return { ensure: `bun scripts/cli.ts hwlab nodes secret ensure --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }; } +function runExternalPostgresSecretEnsure(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { + const runtimeLaneSpec = spec.runtimeLaneSpec; + if (runtimeLaneSpec === undefined) { + return { + ok: false, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-ensure", + mutation: false, + degradedReason: "external-postgres-runtime-lane-spec-missing", + valuesRedacted: true, + }; + } + const sync = syncNodeExternalPostgresSecrets(runtimeLaneSpec, options.dryRun, options.timeoutSeconds); + const shouldReadStatus = sync !== null && sync.ok === true; + const statusResult = shouldReadStatus ? runTransScript(options.node, platformDbSecretStatusScript({ ...options, action: "status", dryRun: true }, spec), "", options.timeoutSeconds) : null; + const status = statusResult === null + ? null + : secretStatusFromText(statusText(statusResult), statusResult.exitCode === 0, statusResult.exitCode, statusResult.stderr, spec); + const ok = sync !== null && sync.ok === true && (options.dryRun || status?.ok === true); + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: options.key ?? null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-ensure", + status: status ?? sync, + externalPostgresSecretSync: sync, + mutation: sync?.mutation === true, + result: statusResult === null ? null : compactCommandResult(statusResult), + valuesRedacted: true, + next: ok ? undefined : nextSecretCommand(options, spec), + }; +} + function publicExposureSummary(exposure: HwlabRuntimePublicExposureSpec): Record { return { mode: exposure.mode, @@ -3921,7 +3986,7 @@ function endpointBridgeScript(options: { lane: HwlabRuntimeLane; dryRun: boolean function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const pvc = `data-${spec.postgresSecret}-0`; - const platformService = "g14-platform-postgres"; + const platformService = spec.platformPostgresService; const postgresService = spec.postgresSecret; const postgresConfigMap = `${spec.postgresSecret}-init`; return [ @@ -4176,12 +4241,14 @@ function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: Runti function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const isOpenFga = options.preset === "openfga"; - const platformEndpointSlice = `${spec.platformPostgresService}-host`; + const platformEndpointSlice = spec.platformPostgresEndpointSlice; + const expectedUriHost = spec.platformPostgresEndpointAddress ?? (isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost); + const databaseUrlKey = isOpenFga ? spec.externalPostgres?.openfga.secretKey ?? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`, - `database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`, + `database_url_key=${shellQuote(databaseUrlKey)}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`, @@ -4189,9 +4256,10 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS `platform_endpointslice=${shellQuote(platformEndpointSlice)}`, `platform_host=${shellQuote(spec.platformPostgresService)}`, `platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`, + `platform_endpoint_address=${shellQuote(spec.platformPostgresEndpointAddress ?? "")}`, `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`, `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`, - `db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`, + `db_host=${shellQuote(expectedUriHost)}`, `selected_key=${shellQuote(options.key ?? "")}`, `preset=${shellQuote(options.preset)}`, "dry_run=true", @@ -4209,7 +4277,10 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS " uri_has_platform_host=no", " uri_has_db_name=no", " uri_has_db_user=no", - " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*) uri_has_platform_host=yes ;; esac", + " case \"$uri\" in *\"@$platform_host:\"*|*\"@$platform_host/\"*|*\"@$platform_host_fqdn:\"*|*\"@$platform_host_fqdn/\"*|*\"@$db_host:\"*|*\"@$db_host/\"*) uri_has_platform_host=yes ;; esac", + " if [ -n \"$platform_endpoint_address\" ]; then", + " case \"$uri\" in *\"@$platform_endpoint_address:\"*|*\"@$platform_endpoint_address/\"*) uri_has_platform_host=yes ;; esac", + " fi", " case \"$uri\" in *\"/$db_name\"|*\"/$db_name?\"*|*\"/$db_name?\"*) uri_has_db_name=yes ;; esac", " case \"$uri\" in postgres://$db_user:*|postgresql://$db_user:*) uri_has_db_user=yes ;; esac", "}", @@ -4254,6 +4325,7 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS "printf 'platformEndpointsExists\\t%s\\n' \"$platform_endpoints_exists\"", "printf 'platformEndpointSlice\\t%s\\n' \"$platform_endpointslice\"", "printf 'platformEndpointSliceExists\\t%s\\n' \"$platform_endpointslice_exists\"", + "printf 'platformEndpointAddress\\t%s\\n' \"$platform_endpoint_address\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbHost\\t%s\\n' \"$db_host\"", From a9c8f4cff24428f42ccec104d981ae680044bba4 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 06:29:13 +0000 Subject: [PATCH 13/13] fix: parameterize AgentRun base image --- config/agentrun.yaml | 4 ++++ scripts/src/agentrun-lanes.ts | 13 +++++++++++++ scripts/src/agentrun.ts | 17 ++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 7efbe8fd..f8a5417a 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -95,6 +95,8 @@ controlPlane: containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host + buildArgs: + BUN_IMAGE: oven/bun:1.2.15-alpine httpProxy: http://127.0.0.1:10808 httpsProxy: http://127.0.0.1:10808 noProxy: @@ -225,6 +227,8 @@ controlPlane: containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host + buildArgs: + BUN_IMAGE: oven/bun:1-alpine httpProxy: http://127.0.0.1:18789 httpsProxy: http://127.0.0.1:18789 noProxy: diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index f4014c74..b97fa38c 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -140,6 +140,7 @@ export interface AgentRunImageBuildSpec { readonly containerfile: string; readonly repository: string; readonly network: string; + readonly buildArgs: Readonly>; readonly httpProxy: string | null; readonly httpsProxy: string | null; readonly noProxy: readonly string[]; @@ -247,6 +248,7 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record, path: string): AgentRun containerfile: relativePathField(input, "containerfile", path), repository: stringField(input, "repository", path), network: stringField(input, "network", path), + buildArgs: stringRecordField(recordField(input, "buildArgs", path), `${path}.buildArgs`), httpProxy: optionalStringField(input, "httpProxy", path) ?? null, httpsProxy: optionalStringField(input, "httpsProxy", path) ?? null, noProxy: stringArrayField(input, "noProxy", path), @@ -631,6 +634,16 @@ function stringArrayField(obj: Record, key: string, path: strin }); } +function stringRecordField(obj: Record, path: string): Readonly> { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`${path}.${key} must be a valid build arg name`); + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + result[key] = value.trim(); + } + return result; +} + function enumField(obj: Record, key: string, path: string, values: readonly T[]): T { const value = stringField(obj, key, path); if (!values.includes(value as T)) throw new Error(`${path}.${key} must be one of ${values.join(", ")}`); diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 7ccfdae2..20ff17df 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2674,6 +2674,7 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt imageBuild: { repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`, containerfile: spec.deployment.manager.imageBuild.containerfile, + buildArgNames: Object.keys(spec.deployment.manager.imageBuild.buildArgs).sort(), timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds, pollSeconds: spec.deployment.manager.imageBuild.pollSeconds, proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null, @@ -3296,6 +3297,9 @@ function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: st const noProxy = build.noProxy.join(","); const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; const stateDir = `/tmp/unidesk-agentrun-build-${spec.nodeId}-${spec.lane}`; + const buildArgs = Object.entries(build.buildArgs) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => `${key}=${value}`); const script = [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, @@ -3309,14 +3313,17 @@ function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: st `https_proxy_value=${build.httpsProxy === null ? "''" : shQuote(build.httpsProxy)}`, `no_proxy_value=${shQuote(noProxy)}`, `env_identity_files=${shQuote(JSON.stringify(build.envIdentityFiles))}`, + `build_args_json=${shQuote(JSON.stringify(buildArgs))}`, "mkdir -p \"$state_dir\"", "cd \"$workspace\"", "git checkout \"$source_commit\"", - "env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" node <<'NODE'", + "env_identity=$(ENV_IDENTITY_FILES=\"$env_identity_files\" BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", "const { createHash } = require('node:crypto');", "const { readFileSync, existsSync } = require('node:fs');", "const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');", + "const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", "const hash = createHash('sha256');", + "for (const item of buildArgs) { hash.update('build-arg'); hash.update('\\0'); hash.update(item); hash.update('\\0'); }", "for (const file of files) { hash.update(file); hash.update('\\0'); if (existsSync(file)) hash.update(readFileSync(file)); hash.update('\\0'); }", "process.stdout.write(hash.digest('hex').slice(0, 24));", "NODE", @@ -3343,6 +3350,14 @@ function yamlLaneBuildImageSubmitScript(spec: AgentRunLaneSpec, sourceCommit: st " if [ -n \"$http_proxy_value\" ]; then args=\"$args --build-arg HTTP_PROXY=$http_proxy_value --build-arg http_proxy=$http_proxy_value\"; fi", " if [ -n \"$https_proxy_value\" ]; then args=\"$args --build-arg HTTPS_PROXY=$https_proxy_value --build-arg https_proxy=$https_proxy_value\"; fi", " if [ -n \"$no_proxy_value\" ]; then args=\"$args --build-arg NO_PROXY=$no_proxy_value --build-arg no_proxy=$no_proxy_value\"; fi", + " build_arg_values=$(BUILD_ARGS_JSON=\"$build_args_json\" node <<'NODE'", + "const values = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');", + "for (const value of values) console.log(value);", + "NODE", + " )", + " while IFS= read -r build_arg_value; do [ -n \"$build_arg_value\" ] && args=\"$args --build-arg $build_arg_value\"; done </dev/null 2>&1; then build_status=reused; else docker build $args -f \"$containerfile\" -t \"$image\" \"$context_dir\"; build_status=built; fi", " docker push \"$image\"", " digest=$(docker inspect --format='{{index .RepoDigests 0}}' \"$image\" 2>/dev/null | sed 's/^.*@//' || true)",