From 3aebd2561c0b87053ae84841e1785c5f63f997dd Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 07:01:43 +0200 Subject: [PATCH] =?UTF-8?q?feat(pikaoa):=20=E6=B7=BB=E5=8A=A0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=8E=AF=E5=A2=83=E8=87=AA=E5=8A=A8=E4=BA=A4=E4=BB=98?= =?UTF-8?q?=E6=8E=A7=E5=88=B6=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/fixtures/pikaoa-test-target.yaml | 43 +- config/pikaoa.yaml | 49 ++- config/platform-infra/gitea.yaml | 50 ++- config/platform-infra/pipelines-as-code.yaml | 78 +++- docs/reference/pikaoa.md | 16 +- .../src/pikaoa-test-delivery-renderer.test.ts | 93 ++++ scripts/src/pikaoa-test-delivery-renderer.ts | 360 ++++++++++++++++ scripts/src/pikaoa-test-runtime-manifest.ts | 254 +++++++++++ scripts/src/pikaoa-test-target-async.test.ts | 54 +-- scripts/src/pikaoa-test-target.ts | 407 ++++++++---------- ...-infra-pipelines-as-code-bootstrap.test.ts | 8 +- ...-pipelines-as-code-source-artifact.test.ts | 43 ++ ...infra-pipelines-as-code-source-artifact.ts | 55 ++- .../src/platform-infra-pipelines-as-code.ts | 9 +- 14 files changed, 1202 insertions(+), 317 deletions(-) create mode 100644 scripts/src/pikaoa-test-delivery-renderer.test.ts create mode 100644 scripts/src/pikaoa-test-delivery-renderer.ts create mode 100644 scripts/src/pikaoa-test-runtime-manifest.ts diff --git a/config/fixtures/pikaoa-test-target.yaml b/config/fixtures/pikaoa-test-target.yaml index 37b81d12..30bfa952 100644 --- a/config/fixtures/pikaoa-test-target.yaml +++ b/config/fixtures/pikaoa-test-target.yaml @@ -30,12 +30,48 @@ testRuntime: mode: prebuilt-images repository: pikainc/pikaoa commitPolicy: cli-required + branch: master + worktreeRemote: https://github.com/pikainc/pikaoa.git + readUrl: https://gitea.example.invalid/mirrors/pikainc-pikaoa.git + snapshotPrefix: refs/unidesk/snapshots/gitea-actions/pikaoa-master-test01 images: api: registry.invalid/pikaoa/api:${commit} worker: registry.invalid/pikaoa/worker:${commit} web: registry.invalid/pikaoa/web:${commit} - migration: registry.invalid/pikaoa/migration:${commit} + initializer: registry.invalid/pikaoa/api:${commit} pullPolicy: IfNotPresent + ci: + lane: pikaoa-test + namespace: pikaoa-ci + pipeline: pikaoa-test-test01-pac + pipelineRunPrefix: pikaoa-test-test01 + serviceAccountName: pikaoa-test-test01-tekton-runner + workspaceSize: 1Gi + pipelineTimeout: 20m0s + toolsImage: registry.invalid/tools:latest + buildkitImage: registry.invalid/buildkit:rootless + build: + dockerfiles: + api: deploy/api.Dockerfile + worker: deploy/worker.Dockerfile + web: deploy/web.Dockerfile + images: + api: registry.invalid/pikaoa/api + worker: registry.invalid/pikaoa/worker + web: registry.invalid/pikaoa/web + networkMode: host + gitops: + readUrl: https://gitea.example.invalid/mirrors/pikainc-pikaoa.git + writeUrl: https://gitea.example.invalid/mirrors/pikainc-pikaoa.git + branch: test01-pikaoa-test-gitops + manifestPath: deploy/gitops/test01-test/resources.yaml + credential: + secretName: pac-gitea-pikaoa-test-test01 + tokenKey: token + username: unidesk-admin + author: + name: PikaOA Test Fixture + email: pikaoa-test-fixture@unidesk.local database: configRef: config/platform-db/postgres-pk01.yaml#exports.connectionStrings.pikaoa-test-database-url name: pikaoa_test @@ -82,8 +118,9 @@ testRuntime: sourceRef: ./secrets/pikaoa-test.env sourceKey: PIKAOA_SESSION_SECRET targetKey: PIKAOA_SESSION_SECRET - migration: - jobName: pikaoa-migrate + initializer: + mode: fresh-database-only + jobName: pikaoa-init backoffLimit: 2 command: - /usr/local/bin/pikaoa diff --git a/config/pikaoa.yaml b/config/pikaoa.yaml index 78698329..c4f4f167 100644 --- a/config/pikaoa.yaml +++ b/config/pikaoa.yaml @@ -21,7 +21,7 @@ modules: - audit extensionContract: registration: module-descriptor - persistence: schema-owned-migrations + persistence: schema-owned-initialization api: versioned-http-routes events: transactional-outbox workers: named-consumer @@ -49,12 +49,48 @@ testRuntime: mode: prebuilt-images repository: pikainc/pikaoa commitPolicy: cli-required + branch: master + worktreeRemote: https://github.com/pikainc/pikaoa.git + readUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git + snapshotPrefix: refs/unidesk/snapshots/gitea-actions/pikaoa-master-nc01 images: api: 127.0.0.1:5000/pikaoa/api:${commit} worker: 127.0.0.1:5000/pikaoa/worker:${commit} web: 127.0.0.1:5000/pikaoa/web:${commit} - migration: 127.0.0.1:5000/pikaoa/api:${commit} + initializer: 127.0.0.1:5000/pikaoa/api:${commit} pullPolicy: IfNotPresent + ci: + lane: pikaoa-test + namespace: pikaoa-ci + pipeline: pikaoa-test-nc01-pac + pipelineRunPrefix: pikaoa-test-nc01 + serviceAccountName: pikaoa-test-nc01-tekton-runner + workspaceSize: 8Gi + pipelineTimeout: 20m0s + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless + build: + dockerfiles: + api: deploy/api.Dockerfile + worker: deploy/worker.Dockerfile + web: deploy/web.Dockerfile + images: + api: 127.0.0.1:5000/pikaoa/api + worker: 127.0.0.1:5000/pikaoa/worker + web: 127.0.0.1:5000/pikaoa/web + networkMode: host + gitops: + readUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git + writeUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git + branch: nc01-pikaoa-test-gitops + manifestPath: deploy/gitops/nc01-test/resources.yaml + credential: + secretName: pac-gitea-pikaoa-test-nc01 + tokenKey: token + username: unidesk-admin + author: + name: PikaOA Test NC01 CI + email: pikaoa-test-nc01-ci@unidesk.local database: configRef: config/platform-db/postgres-pk01.yaml#exports.connectionStrings.pikaoa-test-database-url name: pikaoa_test @@ -101,8 +137,9 @@ testRuntime: sourceRef: /root/.unidesk/.state/secrets/platform-infra/pikaoa-test.env sourceKey: PIKAOA_SESSION_SECRET targetKey: PIKAOA_SESSION_SECRET - migration: - jobName: pikaoa-migrate + initializer: + mode: fresh-database-only + jobName: pikaoa-init backoffLimit: 2 command: - /usr/local/bin/pikaoa @@ -158,9 +195,9 @@ delivery: source: repository: pikainc/pikaoa worktreeRemote: https://github.com/pikainc/pikaoa.git - branch: master + branch: release readUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git - snapshotPrefix: refs/unidesk/snapshots/gitea-actions/pikaoa-master-nc01 + snapshotPrefix: refs/unidesk/snapshots/gitea-actions/pikaoa-release-nc01 build: dockerfiles: api: deploy/api.Dockerfile diff --git a/config/platform-infra/gitea.yaml b/config/platform-infra/gitea.yaml index 1e72523e..25e15eba 100644 --- a/config/platform-infra/gitea.yaml +++ b/config/platform-infra/gitea.yaml @@ -446,7 +446,7 @@ sourceAuthority: naming: gitea-actions-immutable-source prefix: refs/unidesk/snapshots/gitea-actions/selfmedia-release-nc01 legacyGitMirror: null - - key: pikaoa-nc01 + - key: pikaoa-test-nc01 targetId: NC01 credentialOverride: github: @@ -486,12 +486,58 @@ sourceAuthority: publicRead: false readUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git gitops: - branch: nc01-pikaoa-gitops + branch: nc01-pikaoa-test-gitops flushDisposition: gitea-writeback snapshot: naming: gitea-actions-immutable-source prefix: refs/unidesk/snapshots/gitea-actions/pikaoa-master-nc01 legacyGitMirror: null + - key: pikaoa-nc01 + targetId: NC01 + credentialOverride: + github: + transport: https-token + sourceRef: pikainc-selfmedia-gh-token.txt + sourceKey: GH_TOKEN + format: raw-token + requiredFor: + - upstream-mirror + - mirror-sync + - managed-repository-fetch + - github-head-observe + - github-hooks-list + - github-hooks-reconcile + permissions: + contents: read + metadata: read + webhooks: read-write + gitFetchCredential: + apiVersion: unidesk.ai/v1 + kind: GitFetchCredential + authMode: github-https-token + host: github.com + secretRef: + namespace: devops-infra + name: gitea-github-sync-secrets + key: github-token-pikainc-pikaoa + upstream: + repository: pikainc/pikaoa + cloneUrl: https://github.com/pikainc/pikaoa.git + branch: release + visibility: private + gitea: + owner: mirrors + name: pikainc-pikaoa + mirrorMode: controlled-push + publicRead: false + readUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git + gitops: + branch: nc01-pikaoa-gitops + flushDisposition: gitea-writeback + snapshot: + naming: gitea-actions-immutable-source + prefix: refs/unidesk/snapshots/gitea-actions/pikaoa-release-nc01 + legacyGitMirror: null targets: - id: JD01 diff --git a/config/platform-infra/pipelines-as-code.yaml b/config/platform-infra/pipelines-as-code.yaml index cea135da..8436c436 100644 --- a/config/platform-infra/pipelines-as-code.yaml +++ b/config/platform-infra/pipelines-as-code.yaml @@ -213,15 +213,15 @@ repositories: runtime_service_port: "4317" health_path: /healthz health_url: http://selfmedia.selfmedia.svc.cluster.local:4317/healthz - - id: pikaoa-nc01 - name: pikaoa-nc01 + - id: pikaoa-test-nc01 + name: pikaoa-test-nc01 namespace: pikaoa-ci providerType: gitea url: https://gitea.pikapython.com/mirrors/pikainc-pikaoa cloneUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git owner: mirrors repo: pikainc-pikaoa - secretName: pac-gitea-pikaoa-nc01 + secretName: pac-gitea-pikaoa-test-nc01 tokenKey: token webhookSecretKey: webhook.secret concurrencyLimit: 1 @@ -230,9 +230,9 @@ repositories: source_branch: master source_snapshot_prefix: refs/unidesk/snapshots/gitea-actions/pikaoa-master-nc01 node: NC01 - pipeline_name: pikaoa-nc01-pac - pipeline_run_prefix: pikaoa-nc01 - service_account: pikaoa-nc01-tekton-runner + pipeline_name: pikaoa-test-nc01-pac + pipeline_run_prefix: pikaoa-test-nc01 + service_account: pikaoa-test-nc01-tekton-runner pipeline_timeout: 20m0s workspace_pvc_size: 8Gi image_repository: 127.0.0.1:5000/pikaoa/api @@ -242,16 +242,16 @@ repositories: registry_probe_base: http://127.0.0.1:5000 gitops_read_url: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git gitops_write_url: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git - gitops_branch: nc01-pikaoa-gitops + gitops_branch: nc01-pikaoa-test-gitops gitops_username: unidesk-admin - gitops_secret_name: pac-gitea-pikaoa-nc01 - gitops_manifest_path: deploy/gitops/nc01/resources.yaml - runtime_namespace: pikaoa + gitops_secret_name: pac-gitea-pikaoa-test-nc01 + gitops_manifest_path: deploy/gitops/nc01-test/resources.yaml + runtime_namespace: pikaoa-test runtime_deployment: pikaoa-api runtime_service: pikaoa-web runtime_service_port: "8080" health_path: /healthz - health_url: http://pikaoa-web.pikaoa.svc.cluster.local:8080/healthz + health_url: http://pikaoa-web.pikaoa-test.svc.cluster.local:8080/healthz consumers: - extends: templates.consumers.agentrunV02 variables: @@ -467,8 +467,49 @@ consumers: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet fsGroup: 1000 + - id: pikaoa-test-nc01 + repositoryRef: pikaoa-test-nc01 + node: NC01 + lane: pikaoa-test + namespace: pikaoa-ci + pipeline: pikaoa-test-nc01-pac + pipelineRunPrefix: pikaoa-test-nc01 + argoNamespace: argocd + argoApplication: pikaoa-test-nc01 + closeoutGitOpsMirrorFlush: false + argoBootstrap: + project: default + repoUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git + targetRevision: nc01-pikaoa-test-gitops + path: deploy/gitops/nc01-test + destinationNamespace: pikaoa-test + automated: true + repositoryCredential: + secretName: argocd-repo-pikaoa-test-nc01 + username: unidesk-admin + deliveryProvenance: + required: true + markerValue: admission-pac-v2:pikaoa-test-nc01 + executionServiceAccountName: pikaoa-test-nc01-tekton-runner + gitOps: + repoUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git + targetRevision: nc01-pikaoa-test-gitops + runnerServiceAccount: + name: pikaoa-test-nc01-tekton-runner + automountServiceAccountToken: false + roleBindingName: pikaoa-test-nc01-tekton-runner + sourceArtifact: + mode: embedded-pipeline-spec + renderer: pikaoa-test-runtime + configRef: config/pikaoa.yaml#testRuntime.targets.NC01 + pipelineRunPath: .tekton/pikaoa-test-nc01-pac.yaml + maxKeepRuns: 8 + taskRunTemplate: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + fsGroup: 1000 - id: pikaoa-nc01 - repositoryRef: pikaoa-nc01 + repositoryRef: pikaoa-test-nc01 node: NC01 lane: pikaoa namespace: pikaoa-ci @@ -477,6 +518,19 @@ consumers: argoNamespace: argocd argoApplication: pikaoa-nc01 closeoutGitOpsMirrorFlush: false + params: + source_branch: release + source_snapshot_prefix: refs/unidesk/snapshots/gitea-actions/pikaoa-release-nc01 + pipeline_name: pikaoa-nc01-pac + pipeline_run_prefix: pikaoa-nc01 + service_account: pikaoa-nc01-tekton-runner + gitops_branch: nc01-pikaoa-gitops + gitops_secret_name: pac-gitea-pikaoa-nc01 + gitops_manifest_path: deploy/gitops/nc01/resources.yaml + runtime_namespace: pikaoa + runtime_deployment: pikaoa-api + runtime_service: pikaoa-web + health_url: http://pikaoa-web.pikaoa.svc.cluster.local:8080/healthz argoBootstrap: project: default repoUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikainc-pikaoa.git diff --git a/docs/reference/pikaoa.md b/docs/reference/pikaoa.md index c2f71fa0..a9888f20 100644 --- a/docs/reference/pikaoa.md +++ b/docs/reference/pikaoa.md @@ -78,7 +78,7 @@ 5. `secrets sync` 按 YAML 声明补齐允许生成的本地来源，并下发 `pikaoa-test-runtime`。 6. 使用同一 commit 依次执行： - - `--step migration`； + - `--step init`，仅初始化全新数据库； - `--step api`； - `--step worker`； - `--step web`。 @@ -87,7 +87,7 @@ ## 单步运行 - foundation 不需要 `--commit`，也不检查业务 Secret。 -- migration、api、worker、web 和 all 必须显式提供 `--commit`。 +- init、api、worker、web 和 all 必须显式提供 `--commit`。 - 每个 `start` 立即返回任务 ID；使用同一 target、instance 和 step 的 `status` 查询终态。 - request identity 包含： - target； @@ -108,19 +108,21 @@ - 提供新的 `--commit` 修订标识； - 修改 foundation 的稳定 manifest 结构。 - 不为重跑引入第二状态库、控制器、租约或长期历史存储。 -- 迁移、rollout 或运行面查询失败返回具名失败码，不静默进入下一阶段。 +- initializer、rollout 或运行面查询失败返回具名失败码，不静默进入下一阶段。 - OTel exporter、配置版本和 commit 对齐漂移只产生 `blocking=false` warning，不恢复 dedicated 门禁。 ## 交付阶段 -- 当前阶段先稳定 YAML-first CLI 原入口和固定 NC01 测试运行面。 -- CLI 闭环通过后，再建设独立 test CI/CD： - - 复用同一 owning YAML； - - 按 foundation、Secret sync、migration、api、worker、web 顺序编排； +- `pikaoa-test-nc01` 独立 test CI/CD 复用同一 owning YAML： + - 产品 `master` 只生成 `.tekton/pikaoa-test-nc01-pac.yaml`； + - API、Worker、Web 并行构建并以 digest 发布 GitOps； + - initializer 通过 Argo `PreSync` 在 API、Worker、Web 前初始化全新数据库； - 不创建专用测试集群； - 不生成动态 namespace； - 不创建 namespace-local PostgreSQL； - 不把测试 CI/CD 变成生产交付或用户业务的阻塞门禁。 +- GitOps 接管后，手动 `test-target start|stop` mutation 仅用于暂停 Argo 自动同步后的有界调试；禁止 CLI direct-apply 与 Argo 同时写入。 +- 生产 `pikaoa-nc01` 保留 `release` source authority，但当前不 bootstrap、不生成 `.tekton`、不创建 PipelineRun。 - PR、构建或单测不能替代固定 NodePort 原入口验收。 ## 本地验证 diff --git a/scripts/src/pikaoa-test-delivery-renderer.test.ts b/scripts/src/pikaoa-test-delivery-renderer.test.ts new file mode 100644 index 00000000..2585747e --- /dev/null +++ b/scripts/src/pikaoa-test-delivery-renderer.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { renderPikaoaTestDesiredPipeline } from "./pikaoa-test-delivery-renderer"; + +function pacDocument(): Record { + return Bun.YAML.parse(readFileSync(resolve(import.meta.dir, "../../config/platform-infra/pipelines-as-code.yaml"), "utf8")) as Record; +} + +function rendererBinding(): Parameters[0] { + const document = pacDocument(); + const repositories = document.repositories as Array>; + const consumers = document.consumers as Array>; + const repository = repositories.find((item) => item.id === "pikaoa-test-nc01"); + const consumer = consumers.find((item) => item.id === "pikaoa-test-nc01"); + if (repository === undefined || consumer === undefined) throw new Error("PikaOA test PaC fixture is required"); + return { consumer, repository } as Parameters[0]; +} + +function sourceFixture(): string { + const root = mkdtempSync(resolve(tmpdir(), "pikaoa-test-renderer-")); + mkdirSync(resolve(root, "deploy"), { recursive: true }); + for (const file of ["api.Dockerfile", "worker.Dockerfile", "web.Dockerfile"]) writeFileSync(resolve(root, "deploy", file), "FROM scratch\n"); + return root; +} + +describe("PikaOA test PaC renderer", () => { + test("keeps master test and release production consumers isolated", () => { + const document = pacDocument(); + const repositories = document.repositories as Array>; + const consumers = document.consumers as Array>; + const repository = repositories.find((item) => item.id === "pikaoa-test-nc01"); + const testConsumer = consumers.find((item) => item.id === "pikaoa-test-nc01"); + const productionConsumer = consumers.find((item) => item.id === "pikaoa-nc01"); + expect((repository?.params as Record).source_branch).toBe("master"); + expect((testConsumer?.sourceArtifact as Record).pipelineRunPath).toBe(".tekton/pikaoa-test-nc01-pac.yaml"); + expect((testConsumer?.sourceArtifact as Record).renderer).toBe("pikaoa-test-runtime"); + expect((productionConsumer?.params as Record).source_branch).toBe("release"); + expect((productionConsumer?.params as Record).source_snapshot_prefix).toBe("refs/unidesk/snapshots/gitea-actions/pikaoa-release-nc01"); + expect(productionConsumer?.sourceArtifact).toBeUndefined(); + }); + + test("renders parallel digest builds and a PreSync fresh database initializer", () => { + const source = sourceFixture(); + try { + const rendered = renderPikaoaTestDesiredPipeline(rendererBinding(), source); + const spec = rendered.pipeline.spec as Record; + const tasks = spec.tasks as Array>; + for (const name of ["build-api", "build-worker", "build-web"]) { + const task = tasks.find((item) => item.name === name); + expect(task?.runAfter).toEqual(["source"]); + expect(JSON.stringify(task)).toContain("buildctl-daemonless.sh build"); + expect(JSON.stringify(task)).toContain("$(params.revision)"); + expect(JSON.stringify(task)).not.toContain("source-token"); + } + const sourceTask = tasks.find((item) => item.name === "source"); + expect(JSON.stringify(sourceTask)).toContain("source-token"); + expect(JSON.stringify(sourceTask)).toContain("GIT_ASKPASS"); + expect(JSON.stringify(sourceTask)).toContain("$(params.gitops-secret-name)"); + const publish = tasks.find((item) => item.name === "gitops-publish"); + expect(publish?.runAfter).toEqual(["build-api", "build-worker", "build-web"]); + const script = ((((publish?.taskSpec as Record).steps as Array>)[0]?.script) as string); + for (const evidence of [ + '"status":"published"', + '"registryVerified":true', + '"publishedCount":3', + '"requiredServiceCount":3', + '"gitopsCommit"', + '"service":"api"', + '"service":"worker"', + '"service":"web"', + ]) expect(script).toContain(evidence); + const encoded = /printf '%s' '([A-Za-z0-9+/=]+)' \| base64 -d/u.exec(script)?.[1]; + if (encoded === undefined) throw new Error("embedded GitOps manifest is required"); + const manifest = Buffer.from(encoded, "base64").toString("utf8"); + expect(manifest).toContain("argocd.argoproj.io/hook: PreSync"); + expect(manifest).toContain("name: pikaoa-init"); + expect(manifest).toContain("name: initializer"); + expect(manifest).not.toContain("kind: Secret"); + expect(manifest).toContain("secretName: pikaoa-test-runtime"); + expect(manifest).toContain("image: __PIKAOA_API_IMAGE__"); + expect(manifest).toContain("pikaoa.unidesk.io/source-commit: __PIKAOA_SOURCE_COMMIT__"); + expect(script).toContain('s|__PIKAOA_SOURCE_COMMIT__|$SOURCE_COMMIT|g'); + expect(script).toContain("GitOps manifest contains unresolved provenance or image placeholders"); + expect(script).toContain("git ls-remote --exit-code --heads"); + expect(script).toContain("git -C /workspace/gitops checkout --orphan"); + } finally { + rmSync(source, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/src/pikaoa-test-delivery-renderer.ts b/scripts/src/pikaoa-test-delivery-renderer.ts new file mode 100644 index 00000000..9ff83392 --- /dev/null +++ b/scripts/src/pikaoa-test-delivery-renderer.ts @@ -0,0 +1,360 @@ +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; + +import { + pikaoaTestRuntimeConfigRef, + resolvePikaoaTestRuntimeTargetByConfigRef, + type TestTargetSpec, +} from "./pikaoa-test-target"; +import { renderPikaoaTestRuntimeManifest, type PikaoaTestRenderContext } from "./pikaoa-test-runtime-manifest"; +import { stableJsonSha256 } from "./stable-json"; + +export interface PikaoaTestRendererBinding { + readonly consumer: { + readonly id: string; + readonly node: string; + readonly lane: string; + readonly namespace: string; + readonly pipeline: string; + readonly pipelineRunPrefix: string; + readonly sourceArtifact: { + readonly configRef: string; + readonly mode: string; + readonly renderer: string; + }; + }; + readonly repository: { + readonly id: string; + readonly params: Readonly>; + }; +} + +export interface PikaoaTestRenderedPipeline { + readonly pipeline: Record; + readonly configRef: string; + readonly effectiveConfigSha256: string; + readonly sourceWorktreeRemote: string; +} + +export function renderPikaoaTestDesiredPipeline(binding: PikaoaTestRendererBinding, sourceWorktree: string): PikaoaTestRenderedPipeline { + const target = resolvePikaoaTestRuntimeTargetByConfigRef(binding.consumer.sourceArtifact.configRef); + const configRef = pikaoaTestRuntimeConfigRef(target.id); + if (binding.consumer.sourceArtifact.renderer !== "pikaoa-test-runtime" || binding.consumer.sourceArtifact.mode !== "embedded-pipeline-spec") { + throw new Error("pikaoa-test-runtime requires embedded-pipeline-spec"); + } + if (binding.consumer.sourceArtifact.configRef !== configRef) { + throw new Error(`sourceArtifact.configRef must equal resolved owning selector ${configRef}`); + } + assertBinding(target, binding); + assertSourceRepositoryContract(target, sourceWorktree); + return { + pipeline: pipeline(target), + configRef, + effectiveConfigSha256: stableJsonSha256(target), + sourceWorktreeRemote: target.source.worktreeRemote, + }; +} + +export function pikaoaTestSourceWorktreeRemote(targetId: string): string { + return resolvePikaoaTestRuntimeTargetByConfigRef(pikaoaTestRuntimeConfigRef(targetId)).source.worktreeRemote; +} + +function pipeline(target: TestTargetSpec): Record { + const runtimeManifest = renderGitOpsManifest(target); + return { + apiVersion: "tekton.dev/v1", + kind: "Pipeline", + metadata: { name: target.ci.pipeline, namespace: target.ci.namespace }, + spec: { + params: [ + { name: "revision", type: "string" }, + { name: "source-snapshot-prefix", type: "string", default: target.source.snapshotPrefix }, + { name: "git-read-url", type: "string", default: target.source.readUrl }, + { name: "gitops-read-url", type: "string", default: target.gitops.readUrl }, + { name: "gitops-write-url", type: "string", default: target.gitops.writeUrl }, + { name: "gitops-username", type: "string", default: target.gitops.credentialUsername }, + { name: "gitops-secret-name", type: "string", default: target.gitops.credentialSecretName }, + ], + workspaces: [{ name: "workspace" }], + tasks: [ + sourceTask(target), + imageBuildTask(target, "api"), + imageBuildTask(target, "worker"), + imageBuildTask(target, "web"), + gitOpsTask(target, runtimeManifest), + ], + }, + }; +} + +function sourceTask(target: TestTargetSpec): Record { + return { + name: "source", + taskSpec: { + params: [ + { name: "revision", type: "string" }, + { name: "source-snapshot-prefix", type: "string" }, + { name: "git-read-url", type: "string" }, + { name: "gitops-username", type: "string" }, + { name: "gitops-secret-name", type: "string" }, + ], + workspaces: [{ name: "workspace" }], + steps: [{ + name: "checkout", + image: target.ci.toolsImage, + imagePullPolicy: "IfNotPresent", + script: `#!/bin/sh +set -eu +revision="$(params.revision)" +case "$revision" in *[!0-9a-f]*|'') echo 'revision must be a full lowercase source commit' >&2; exit 2;; esac +test "\${#revision}" -eq 40 +askpass=/tmp/pikaoa-source-git-askpass +cat >"$askpass" <<'ASKPASS' +#!/bin/sh +case "$1" in + *Username*) printf '%s' "$PIKAOA_GIT_USERNAME" ;; + *Password*) cat /var/run/pikaoa-source/token ;; + *) exit 1 ;; +esac +ASKPASS +chmod 700 "$askpass" +export PIKAOA_GIT_USERNAME="$(params.gitops-username)" +export GIT_ASKPASS="$askpass" +export GIT_TERMINAL_PROMPT=0 +rm -rf /workspace/source +git init /workspace/source +git -C /workspace/source remote add origin "$(params.git-read-url)" +git -C /workspace/source fetch --depth=1 origin "$(params.source-snapshot-prefix)/$revision" +git -C /workspace/source checkout --detach FETCH_HEAD +test "$(git -C /workspace/source rev-parse HEAD)" = "$revision" +rm -f "$askpass" +printf '{"ok":true,"phase":"source","sourceCommit":"%s","valuesPrinted":false}\n' "$revision" +`, + securityContext: nonRootSecurity(), + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "source-token", mountPath: "/var/run/pikaoa-source", readOnly: true }, + ], + }], + volumes: [{ name: "source-token", secret: { secretName: "$(params.gitops-secret-name)", items: [{ key: target.gitops.credentialTokenKey, path: "token" }] } }], + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "source-snapshot-prefix", value: "$(params.source-snapshot-prefix)" }, + { name: "git-read-url", value: "$(params.git-read-url)" }, + { name: "gitops-username", value: "$(params.gitops-username)" }, + { name: "gitops-secret-name", value: "$(params.gitops-secret-name)" }, + ], + workspaces: [{ name: "workspace", workspace: "workspace" }], + }; +} + +function imageBuildTask(target: TestTargetSpec, component: "api" | "worker" | "web"): Record { + const taskName = `build-${component}`; + const imageRepository = target.build.images[component]; + const dockerfile = target.build.dockerfiles[component]; + return { + name: taskName, + runAfter: ["source"], + taskSpec: { + params: [{ name: "revision", type: "string" }], + workspaces: [{ name: "workspace" }], + results: [{ name: "digest", type: "string" }, { name: "image", type: "string" }], + steps: [{ + name: "build", + image: target.ci.buildkitImage, + imagePullPolicy: "IfNotPresent", + workingDir: "/workspace/source", + env: [{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }], + script: `#!/bin/sh +set -eu +revision="$(params.revision)" +tag=${shellSingleQuote(imageRepository)}:"$revision" +metadata=/tmp/${component}-metadata.json +buildctl-daemonless.sh build \\ + --frontend dockerfile.v0 \\ + --local context=. \\ + --local dockerfile=. \\ + --opt filename=${shellSingleQuote(dockerfile)} \\ + --opt network=${shellSingleQuote(target.build.networkMode)} \\ + --output "type=image,name=$tag,push=true" \\ + --metadata-file "$metadata" +digest=$(grep -o '"containerimage.digest":"sha256:[0-9a-f]*"' "$metadata" | head -n1 | cut -d'"' -f4) +case "$digest" in sha256:[0-9a-f]*) ;; *) echo 'build digest missing' >&2; exit 3;; esac +printf '%s' "$digest" > "$(results.digest.path)" +printf '%s@%s' ${shellSingleQuote(imageRepository)} "$digest" > "$(results.image.path)" +printf '{"ok":true,"phase":"${taskName}","sourceCommit":"%s","imageTag":"%s","digest":"%s","valuesPrinted":false}\n' "$revision" "$tag" "$digest" +`, + securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, + volumeMounts: [{ name: "workspace", mountPath: "/workspace" }], + }], + }, + params: [{ name: "revision", value: "$(params.revision)" }], + workspaces: [{ name: "workspace", workspace: "workspace" }], + }; +} + +function gitOpsTask(target: TestTargetSpec, runtimeManifest: string): Record { + const manifestB64 = Buffer.from(runtimeManifest, "utf8").toString("base64"); + return { + name: "gitops-publish", + runAfter: ["build-api", "build-worker", "build-web"], + taskSpec: { + params: [ + { name: "revision", type: "string" }, + { name: "gitops-read-url", type: "string" }, + { name: "gitops-write-url", type: "string" }, + { name: "gitops-username", type: "string" }, + { name: "gitops-secret-name", type: "string" }, + { name: "api-image", type: "string" }, + { name: "worker-image", type: "string" }, + { name: "web-image", type: "string" }, + ], + workspaces: [{ name: "workspace" }], + steps: [{ + name: "publish", + image: target.ci.toolsImage, + imagePullPolicy: "IfNotPresent", + env: [{ name: "SOURCE_COMMIT", value: "$(params.revision)" }], + script: `#!/bin/sh +set -eu +askpass=/tmp/pikaoa-git-askpass +cat >"$askpass" <<'ASKPASS' +#!/bin/sh +case "$1" in + *Username*) printf '%s' "$PIKAOA_GIT_USERNAME" ;; + *Password*) cat /var/run/pikaoa-gitops/token ;; + *) exit 1 ;; +esac +ASKPASS +chmod 700 "$askpass" +export PIKAOA_GIT_USERNAME="$(params.gitops-username)" +export GIT_ASKPASS="$askpass" +export GIT_TERMINAL_PROMPT=0 +case "$SOURCE_COMMIT" in *[!0-9a-f]*|'') echo 'source commit must be a full lowercase commit' >&2; exit 2;; esac +test "\${#SOURCE_COMMIT}" -eq 40 +api_image="$(params.api-image)" +worker_image="$(params.worker-image)" +web_image="$(params.web-image)" +for image in "$api_image" "$worker_image" "$web_image"; do + case "$image" in *@sha256:[0-9a-f]*) ;; *) echo 'required digestRef missing' >&2; exit 4;; esac +done +rm -rf /workspace/gitops +mkdir -p /workspace/gitops +git -C /workspace/gitops init +git -C /workspace/gitops remote add origin "$(params.gitops-read-url)" +if git ls-remote --exit-code --heads "$(params.gitops-read-url)" ${shellSingleQuote(target.gitops.branch)} >/dev/null 2>&1; then + git -C /workspace/gitops fetch --depth=1 origin ${shellSingleQuote(target.gitops.branch)} + git -C /workspace/gitops checkout -B ${shellSingleQuote(target.gitops.branch)} FETCH_HEAD +else + git -C /workspace/gitops checkout --orphan ${shellSingleQuote(target.gitops.branch)} + git -C /workspace/gitops rm -rf . >/dev/null 2>&1 || true +fi +manifest=/workspace/gitops/${shellSingleQuote(target.gitops.manifestPath)} +mkdir -p "$(dirname "$manifest")" +printf '%s' ${shellSingleQuote(manifestB64)} | base64 -d > "$manifest" +sed -i \\ + -e "s|__PIKAOA_SOURCE_COMMIT__|$SOURCE_COMMIT|g" \\ + -e "s|__PIKAOA_API_IMAGE__|$(params.api-image)|g" \\ + -e "s|__PIKAOA_WORKER_IMAGE__|$(params.worker-image)|g" \\ + -e "s|__PIKAOA_WEB_IMAGE__|$(params.web-image)|g" \\ + "$manifest" +if grep -Eq '\$\(params\.revision\)|__PIKAOA_SOURCE_COMMIT__|__PIKAOA_(API|WORKER|WEB)_IMAGE__' "$manifest"; then + echo 'GitOps manifest contains unresolved provenance or image placeholders' >&2 + exit 6 +fi +git -C /workspace/gitops config user.name ${shellSingleQuote(target.gitops.authorName)} +git -C /workspace/gitops config user.email ${shellSingleQuote(target.gitops.authorEmail)} +git -C /workspace/gitops add -- ${shellSingleQuote(target.gitops.manifestPath)} +if git -C /workspace/gitops diff --cached --quiet; then + changed=false +else + git -C /workspace/gitops commit -m "chore(gitops): deploy pikaoa test $SOURCE_COMMIT" + git -C /workspace/gitops push "$(params.gitops-write-url)" HEAD:${shellSingleQuote(target.gitops.branch)} + changed=true +fi +gitops_commit=$(git -C /workspace/gitops rev-parse HEAD) +case "$gitops_commit" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) ;; *) echo 'gitops commit missing' >&2; exit 5;; esac +rm -f "$askpass" +printf '{"ok":true,"status":"published","phase":"gitops-publish","sourceCommitId":"%s","registryVerified":true,"publishedCount":3,"reusedCount":0,"requiredServiceCount":3,"services":[{"service":"api","digestRef":"%s","digest":"%s"},{"service":"worker","digestRef":"%s","digest":"%s"},{"service":"web","digestRef":"%s","digest":"%s"}],"gitopsCommit":"%s","changed":%s,"valuesPrinted":false}\n' \ + "$SOURCE_COMMIT" "$api_image" "\${api_image##*@}" "$worker_image" "\${worker_image##*@}" "$web_image" "\${web_image##*@}" "$gitops_commit" "$changed" +`, + securityContext: nonRootSecurity(), + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "gitops-token", mountPath: "/var/run/pikaoa-gitops", readOnly: true }, + ], + }], + volumes: [{ name: "gitops-token", secret: { secretName: "$(params.gitops-secret-name)", items: [{ key: target.gitops.credentialTokenKey, path: "token" }] } }], + }, + params: [ + { name: "revision", value: "$(params.revision)" }, + { name: "gitops-read-url", value: "$(params.gitops-read-url)" }, + { name: "gitops-write-url", value: "$(params.gitops-write-url)" }, + { name: "gitops-username", value: "$(params.gitops-username)" }, + { name: "gitops-secret-name", value: "$(params.gitops-secret-name)" }, + { name: "api-image", value: "$(tasks.build-api.results.image)" }, + { name: "worker-image", value: "$(tasks.build-worker.results.image)" }, + { name: "web-image", value: "$(tasks.build-web.results.image)" }, + ], + workspaces: [{ name: "workspace", workspace: "workspace" }], + }; +} + +function renderGitOpsManifest(target: TestTargetSpec): string { + const context: PikaoaTestRenderContext = { + instanceId: "default", + commit: "__PIKAOA_SOURCE_COMMIT__", + step: "all", + namespace: target.namespace, + expiresAt: "gitops-managed", + apiImage: "__PIKAOA_API_IMAGE__", + workerImage: "__PIKAOA_WORKER_IMAGE__", + webImage: "__PIKAOA_WEB_IMAGE__", + initializerImage: "__PIKAOA_API_IMAGE__", + }; + const manifest = renderPikaoaTestRuntimeManifest(target, context, { secretMode: "existing", initializerHook: "argo-presync" }); + return manifest.map((object) => Bun.YAML.stringify(object, null, 2).trim()).join("\n---\n") + "\n"; +} + +function assertBinding(target: TestTargetSpec, binding: PikaoaTestRendererBinding): void { + const expected: Readonly> = { + id: "pikaoa-test-nc01", + node: target.node, + lane: target.ci.lane, + namespace: target.ci.namespace, + pipeline: target.ci.pipeline, + pipelineRunPrefix: target.ci.pipelineRunPrefix, + sourceBranch: target.source.branch, + gitopsBranch: target.gitops.branch, + serviceAccount: target.ci.serviceAccountName, + }; + const actual: Readonly> = { + id: binding.consumer.id, + node: binding.consumer.node, + lane: binding.consumer.lane, + namespace: binding.consumer.namespace, + pipeline: binding.consumer.pipeline, + pipelineRunPrefix: binding.consumer.pipelineRunPrefix, + sourceBranch: binding.repository.params.source_branch, + gitopsBranch: binding.repository.params.gitops_branch, + serviceAccount: binding.repository.params.service_account, + }; + for (const [key, value] of Object.entries(expected)) { + if (actual[key] !== value) throw new Error(`pikaoa-test-runtime binding ${key} must equal owning YAML value ${value}`); + } +} + +function assertSourceRepositoryContract(target: TestTargetSpec, sourceWorktree: string): void { + for (const path of Object.values(target.build.dockerfiles)) { + if (!existsSync(resolve(sourceWorktree, path))) throw new Error(`PikaOA source worktree is missing ${path}`); + } +} + +function nonRootSecurity(): Record { + return { allowPrivilegeEscalation: false, runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000 }; +} + +function shellSingleQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} diff --git a/scripts/src/pikaoa-test-runtime-manifest.ts b/scripts/src/pikaoa-test-runtime-manifest.ts new file mode 100644 index 00000000..a688348e --- /dev/null +++ b/scripts/src/pikaoa-test-runtime-manifest.ts @@ -0,0 +1,254 @@ +import type { TestTargetSpec } from "./pikaoa-test-target"; + +export type PikaoaTestTargetStep = "all" | "foundation" | "init" | "api" | "worker" | "web"; + +export interface PikaoaTestRenderContext { + instanceId: string; + commit: string; + step: PikaoaTestTargetStep; + namespace: string; + expiresAt: string; + apiImage: string; + workerImage: string; + webImage: string; + initializerImage: string; +} + +export interface PikaoaTestSecretMaterial { + databaseURL: string; + sessionSecret: string; + administratorPassword: string; + employeePassword: string; +} + +export interface PikaoaTestManifestOptions { + secretMode: "materialized" | "existing"; + secrets?: PikaoaTestSecretMaterial; + initializerHook?: "argo-presync"; +} + +const MANAGED_BY = "unidesk-pikaoa-test-target"; +const TEST_RUNTIME_LABEL = "pikaoa.unidesk.io/test-runtime"; +const TARGET_LABEL = "pikaoa.unidesk.io/target"; +const INSTANCE_LABEL = "pikaoa.unidesk.io/instance"; + +export function renderPikaoaTestRuntimeManifest( + target: TestTargetSpec, + context: PikaoaTestRenderContext, + options: PikaoaTestManifestOptions, +): Record[] { + if (options.secretMode === "materialized" && options.secrets === undefined) { + throw new Error("materialized PikaOA test manifest requires secret material"); + } + const labels = ownershipLabels(target, context); + const apiSelector = componentSelector("api", context.instanceId); + const workerSelector = componentSelector("worker", context.instanceId); + const webSelector = componentSelector("web", context.instanceId); + const initializerSelector = componentSelector("initializer", context.instanceId); + const prometheusPodLabels = target.observability.prometheusScrape ? { + [target.observability.prometheusSelector.key]: target.observability.prometheusSelector.value, + } : {}; + const commonPodAnnotations = (path: string, port: number): Record => target.observability.prometheusScrape ? { + "prometheus.io/scrape": "true", + "prometheus.io/path": path, + "prometheus.io/port": String(port), + } : {}; + const initializerAnnotations = options.initializerHook === "argo-presync" ? { + "argocd.argoproj.io/hook": "PreSync", + "argocd.argoproj.io/hook-delete-policy": "BeforeHookCreation,HookSucceeded", + } : {}; + return [ + { + apiVersion: "v1", kind: "Namespace", metadata: { name: context.namespace, labels, annotations: namespaceAnnotations(target, context) }, + }, + ...(options.secretMode === "materialized" ? [{ + apiVersion: "v1", kind: "Secret", metadata: { name: target.runtime.secretName, namespace: context.namespace, labels }, type: "Opaque", + stringData: runtimeSecretData(target, context, options.secrets!), + }] : []), + { + apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: target.runtime.attachment.claimName, namespace: context.namespace, labels }, + spec: { + accessModes: target.runtime.attachment.accessModes, + resources: { requests: { storage: target.runtime.attachment.storageRequest } }, + ...(target.runtime.attachment.storageClassName === null ? {} : { storageClassName: target.runtime.attachment.storageClassName }), + }, + }, + { + apiVersion: "v1", kind: "Service", metadata: { name: target.runtime.apiServiceName, namespace: context.namespace, labels: { ...labels, ...apiSelector } }, + spec: { selector: apiSelector, ports: [{ name: "http", port: target.runtime.apiPort, targetPort: target.runtime.apiPort }] }, + }, + { + apiVersion: "v1", kind: "Service", metadata: { name: target.exposure.serviceName, namespace: context.namespace, labels: { ...labels, ...webSelector } }, + spec: { + type: target.exposure.serviceType, + selector: webSelector, + ports: [{ name: "http", port: target.runtime.webPort, targetPort: target.runtime.webPort, nodePort: target.exposure.port }], + }, + }, + { + apiVersion: "batch/v1", kind: "Job", + metadata: { name: target.initializer.jobName, namespace: context.namespace, labels: { ...labels, ...initializerSelector }, annotations: initializerAnnotations }, + spec: { + backoffLimit: target.initializer.backoffLimit, + template: { + metadata: { labels: { ...labels, ...initializerSelector } }, + spec: { + restartPolicy: "Never", + containers: [{ + name: "initializer", image: context.initializerImage, imagePullPolicy: target.source.pullPolicy, + command: target.initializer.command, + env: [{ name: "PIKAOA_CONFIG", value: "/etc/pikaoa/pikaoa.yaml" }], + volumeMounts: [{ name: "runtime", mountPath: "/etc/pikaoa", readOnly: true }], + }], + volumes: [{ name: "runtime", secret: { secretName: target.runtime.secretName, items: [{ key: "pikaoa.yaml", path: "pikaoa.yaml" }] } }], + }, + }, + }, + }, + deployment(target, context, "api", context.apiImage, apiSelector, target.runtime.apiPort, prometheusPodLabels, commonPodAnnotations(target.observability.apiMetricsPath, target.runtime.apiPort), [ + { name: "PIKAOA_CONFIG", value: "/etc/pikaoa/pikaoa.yaml" }, + { name: "PIKAOA_SESSION_SECRET", valueFrom: { secretKeyRef: { name: target.runtime.secretName, key: target.runtime.sessionSecret.targetKey } } }, + { name: "PIKAOA_BOOTSTRAP_ADMIN_USERNAME", value: target.runtime.administrator.username }, + { name: "PIKAOA_BOOTSTRAP_ADMIN_PASSWORD", valueFrom: { secretKeyRef: { name: target.runtime.secretName, key: target.runtime.administrator.password.targetKey } } }, + { name: "PIKAOA_BOOTSTRAP_EMPLOYEE_USERNAME", value: target.runtime.employee.username }, + { name: "PIKAOA_BOOTSTRAP_EMPLOYEE_PASSWORD", valueFrom: { secretKeyRef: { name: target.runtime.secretName, key: target.runtime.employee.password.targetKey } } }, + ], target.probes.apiLivenessPath, target.probes.apiReadinessPath), + deployment(target, context, "worker", context.workerImage, workerSelector, target.runtime.workerMetricsPort, prometheusPodLabels, commonPodAnnotations(target.observability.workerMetricsPath, target.runtime.workerMetricsPort), [ + { name: "PIKAOA_CONFIG", value: "/etc/pikaoa/pikaoa.yaml" }, + ], target.probes.workerLivenessPath, target.probes.workerReadinessPath), + deployment(target, context, "web", context.webImage, webSelector, target.runtime.webPort, {}, {}, [ + { name: target.runtime.webApiUpstreamEnv, value: `http://${target.runtime.apiServiceName}.${context.namespace}.svc.cluster.local:${target.runtime.apiPort}` }, + ], target.probes.webLivenessPath, target.probes.webReadinessPath), + ]; +} + +export function pikaoaManifestForStep(manifest: Record[], step: PikaoaTestTargetStep): Record[] { + if (step === "all") return manifest; + return manifest.filter((object) => { + if (object.kind === "Namespace" || object.kind === "PersistentVolumeClaim") return true; + if (step === "foundation") return false; + if (object.kind === "Secret") return true; + if (step === "init") return object.kind === "Job"; + const metadata = optionalRecord(object.metadata); + const labels = optionalRecord(metadata?.labels); + return labels?.["app.kubernetes.io/component"] === step; + }); +} + +export function ownershipLabels(target: TestTargetSpec, context: PikaoaTestRenderContext): Record { + return { + "app.kubernetes.io/name": "pikaoa", + "app.kubernetes.io/part-of": "pikaoa-test-runtime", + "app.kubernetes.io/managed-by": MANAGED_BY, + [TEST_RUNTIME_LABEL]: "true", + [TARGET_LABEL]: target.id, + [INSTANCE_LABEL]: context.instanceId, + }; +} + +export function componentSelector(component: string, instanceId: string): Record { + return { "app.kubernetes.io/name": "pikaoa", "app.kubernetes.io/component": component, [INSTANCE_LABEL]: instanceId }; +} + +export function namespaceAnnotations(target: TestTargetSpec, context: PikaoaTestRenderContext): Record { + return { + "pikaoa.unidesk.io/source-repository": target.source.repository, + "pikaoa.unidesk.io/source-commit": context.commit, + "pikaoa.unidesk.io/expires-at": context.expiresAt, + "pikaoa.unidesk.io/ttl-seconds": String(target.ttlSeconds), + }; +} + +export function renderDatabaseURL(target: TestTargetSpec, sourceURL: string): string { + const databaseURL = new URL(sourceURL); + databaseURL.searchParams.set("search_path", target.database.schema); + return databaseURL.toString(); +} + +export function placeholderPikaoaSecrets(): PikaoaTestSecretMaterial { + return { + databaseURL: "postgresql://redacted@db.invalid/pikaoa_test?sslmode=require", + sessionSecret: "", + administratorPassword: "", + employeePassword: "", + }; +} + +function deployment( + target: TestTargetSpec, + context: PikaoaTestRenderContext, + component: "api" | "worker" | "web", + image: string, + selector: Record, + port: number, + podLabels: Record, + annotations: Record, + env: Array>, + livenessPath: string, + readinessPath: string, +): Record { + const labels = ownershipLabels(target, context); + return { + apiVersion: "apps/v1", kind: "Deployment", metadata: { name: `pikaoa-${component}`, namespace: context.namespace, labels: { ...labels, ...selector } }, + spec: { + replicas: 1, selector: { matchLabels: selector }, + template: { + metadata: { labels: { ...labels, ...selector, ...podLabels }, annotations }, + spec: { + ...(component === "api" ? { securityContext: { fsGroup: target.runtime.attachment.fsGroup } } : {}), + containers: [{ + name: component, image, imagePullPolicy: target.source.pullPolicy, env, + ports: [{ name: component === "worker" ? "metrics" : "http", containerPort: port }], + volumeMounts: [ + ...(component === "web" ? [] : [{ name: "runtime", mountPath: "/etc/pikaoa", readOnly: true }]), + ...(component === "api" ? [{ name: target.runtime.attachment.claimName, mountPath: target.runtime.attachment.storageRoot }] : []), + ], + livenessProbe: { httpGet: { path: livenessPath, port }, periodSeconds: 5, failureThreshold: 12 }, + readinessProbe: { httpGet: { path: readinessPath, port }, periodSeconds: 3, failureThreshold: 20 }, + }], + volumes: [ + ...(component === "web" ? [] : [{ name: "runtime", secret: { secretName: target.runtime.secretName, items: [{ key: "pikaoa.yaml", path: "pikaoa.yaml" }] } }]), + ...(component === "api" ? [{ name: target.runtime.attachment.claimName, persistentVolumeClaim: { claimName: target.runtime.attachment.claimName } }] : []), + ], + }, + }, + }, + }; +} + +function runtimeSecretData(target: TestTargetSpec, context: PikaoaTestRenderContext, secrets: PikaoaTestSecretMaterial): Record { + const databaseURL = renderDatabaseURL(target, secrets.databaseURL); + return { + [target.database.connection.targetKey]: databaseURL, + [target.runtime.sessionSecret.targetKey]: secrets.sessionSecret, + [target.runtime.administrator.password.targetKey]: secrets.administratorPassword, + [target.runtime.employee.password.targetKey]: secrets.employeePassword, + "pikaoa.yaml": [ + "api:", + ` listen: 0.0.0.0:${target.runtime.apiPort}`, + "worker:", + ` interval: ${target.runtime.workerInterval}`, + ` metricsListen: 0.0.0.0:${target.runtime.workerMetricsPort}`, + " outbox:", + ` batchSize: ${target.runtime.outboxBatchSize}`, + ` maxAttempts: ${target.runtime.outboxMaxAttempts}`, + ` retryDelay: ${target.runtime.outboxRetryDelay}`, + "database:", + ` url: ${JSON.stringify(databaseURL)}`, + "identity:", + ` sessionTTL: ${target.runtime.sessionTTL}`, + "attachment:", + ` storageRoot: ${JSON.stringify(target.runtime.attachment.storageRoot)}`, + ` maxBytes: ${target.runtime.attachment.maxBytes}`, + "observability:", + ` serviceName: pikaoa-test-${context.instanceId}`, + ` otlpEndpoint: ${JSON.stringify(target.observability.otlpEndpoint)}`, + `shutdownTimeout: ${target.runtime.shutdownTimeout}`, + "", + ].join("\n"), + }; +} + +function optionalRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} diff --git a/scripts/src/pikaoa-test-target-async.test.ts b/scripts/src/pikaoa-test-target-async.test.ts index 61ad054c..79bc6dcd 100644 --- a/scripts/src/pikaoa-test-target-async.test.ts +++ b/scripts/src/pikaoa-test-target-async.test.ts @@ -74,11 +74,11 @@ if printf '%s' "$args" | grep -q ' rollout status '; then [ ! -f "$root/slow" ] || sleep 0.4 exit 0 fi -if printf '%s' "$args" | grep -q ' get job/pikaoa-migrate '; then - if [ -f "$root/fail-migration-query" ]; then printf '%s\n' 'job query failed' >&2; exit 1; fi - if [ -f "$root/failure-target-migration" ]; then +if printf '%s' "$args" | grep -q ' get job/pikaoa-init '; then + if [ -f "$root/fail-initializer-query" ]; then printf '%s\n' 'job query failed' >&2; exit 1; fi + if [ -f "$root/failure-target-initializer" ]; then printf '%s\n' '{"status":{"conditions":[{"type":"FailureTarget","status":"True","reason":"BackoffLimitExceeded"}]}}' - elif [ -f "$root/fail-migration" ]; then + elif [ -f "$root/fail-initializer" ]; then printf '%s\n' '{"status":{"conditions":[{"type":"Failed","status":"True","reason":"BackoffLimitExceeded"}]}}' else printf '%s\n' '{"status":{"conditions":[{"type":"Complete","status":"True"}]}}' @@ -273,33 +273,33 @@ exit 64 rmSync(join(fakeRoot, "slow")); const workerMissingSucceeded = await waitFor("runtime", "start-succeeded", "api"); - writeFileSync(join(fakeRoot, "fail-migration"), "1\n"); - await projection("start", "runtime", true, "migration"); - const failed = await waitFor("runtime", "start-failed", "migration"); + writeFileSync(join(fakeRoot, "fail-initializer"), "1\n"); + await projection("start", "runtime", true, "init"); + const failed = await waitFor("runtime", "start-failed", "init"); const failedTask = ((failed.status as Record).task as Record); - assert.equal(failedTask.code, "migration-job-failed"); + assert.equal(failedTask.code, "initializer-job-failed"); assert.equal(failedTask.exitCode, 46); - rmSync(join(fakeRoot, "fail-migration")); + rmSync(join(fakeRoot, "fail-initializer")); - writeFileSync(join(fakeRoot, "failure-target-migration"), "1\n"); - await projectionWithCommit("start", "runtime", concurrentCommit, "migration"); - const failureTarget = await waitFor("runtime", "start-failed", "migration"); + writeFileSync(join(fakeRoot, "failure-target-initializer"), "1\n"); + await projectionWithCommit("start", "runtime", concurrentCommit, "init"); + const failureTarget = await waitFor("runtime", "start-failed", "init"); const failureTargetTask = ((failureTarget.status as Record).task as Record); - assert.equal(failureTargetTask.code, "migration-job-failure-target"); + assert.equal(failureTargetTask.code, "initializer-job-failure-target"); assert.equal(failureTargetTask.exitCode, 46); - rmSync(join(fakeRoot, "failure-target-migration")); + rmSync(join(fakeRoot, "failure-target-initializer")); - writeFileSync(join(fakeRoot, "fail-migration-query"), "1\n"); - await projectionWithCommit("start", "runtime", "abcdef0123456789", "migration"); - const migrationQueryFailed = await waitFor("runtime", "start-failed", "migration"); - const migrationQueryFailedTask = ((migrationQueryFailed.status as Record).task as Record); - assert.equal(migrationQueryFailedTask.code, "migration-job-query-failed"); - assert.equal(migrationQueryFailedTask.exitCode, 46); - rmSync(join(fakeRoot, "fail-migration-query")); + writeFileSync(join(fakeRoot, "fail-initializer-query"), "1\n"); + await projectionWithCommit("start", "runtime", "abcdef0123456789", "init"); + const initializerQueryFailed = await waitFor("runtime", "start-failed", "init"); + const initializerQueryFailedTask = ((initializerQueryFailed.status as Record).task as Record); + assert.equal(initializerQueryFailedTask.code, "initializer-job-query-failed"); + assert.equal(initializerQueryFailedTask.exitCode, 46); + rmSync(join(fakeRoot, "fail-initializer-query")); - await projectionWithCommit("start", "runtime", "1234567890abcdef", "migration"); - const migrationSucceeded = await waitFor("runtime", "start-succeeded", "migration"); - assert.equal((((migrationSucceeded.status as Record).task as Record).exitCode), 0); + await projectionWithCommit("start", "runtime", "1234567890abcdef", "init"); + const initializerSucceeded = await waitFor("runtime", "start-succeeded", "init"); + assert.equal((((initializerSucceeded.status as Record).task as Record).exitCode), 0); writeFileSync(join(fakeRoot, "slow"), "1\n"); await projection("start", "runtime", true, "web"); @@ -344,8 +344,8 @@ exit 64 workerMissingSucceeded, failed, failureTarget, - migrationQueryFailed, - migrationSucceeded, + initializerQueryFailed, + initializerSucceeded, stopped, foundationSubmitted, foundationSucceeded, @@ -397,7 +397,7 @@ test("validation-only fixture never invokes remote capture", async () => { secretName: "pikaoa-test-runtime", key: "pikaoa.yaml", mountPath: "/etc/pikaoa/pikaoa.yaml", - consumers: ["migration", "api", "worker"], + consumers: ["initializer", "api", "worker"], valuesPrinted: false, }); assert.equal(planPayload.valuesPrinted, false); diff --git a/scripts/src/pikaoa-test-target.ts b/scripts/src/pikaoa-test-target.ts index 1ea5c84d..89fe6184 100644 --- a/scripts/src/pikaoa-test-target.ts +++ b/scripts/src/pikaoa-test-target.ts @@ -10,9 +10,18 @@ import type { RenderedCliResult } from "./output"; import { CliInputError } from "./output"; import { capture, compactCapture, parseJsonOutput, readYamlRecord, sha256Fingerprint, shQuote } from "./platform-infra-ops-library"; import { resolveConfigRef } from "./ops/config-refs"; +import { + pikaoaManifestForStep, + placeholderPikaoaSecrets, + renderDatabaseURL, + renderPikaoaTestRuntimeManifest, + type PikaoaTestRenderContext, + type PikaoaTestSecretMaterial, + type PikaoaTestTargetStep, +} from "./pikaoa-test-runtime-manifest"; type TestTargetAction = "plan" | "status" | "start" | "stop"; -type TestTargetStep = "all" | "foundation" | "migration" | "api" | "worker" | "web"; +type TestTargetStep = PikaoaTestTargetStep; interface TestTargetOptions { action: TestTargetAction; @@ -41,7 +50,7 @@ interface PrometheusSelectorSpec { valuesPrinted: false; } -interface TestTargetSpec { +export interface TestTargetSpec { id: string; enabled: boolean; validationOnly: boolean; @@ -64,12 +73,43 @@ interface TestTargetSpec { mode: string; repository: string; commitPolicy: string; + branch: string; + worktreeRemote: string; + readUrl: string; + snapshotPrefix: string; apiImage: string; workerImage: string; webImage: string; - migrationImage: string; + initializerImage: string; pullPolicy: string; }; + ci: { + lane: string; + namespace: string; + pipeline: string; + pipelineRunPrefix: string; + serviceAccountName: string; + workspaceSize: string; + pipelineTimeout: string; + toolsImage: string; + buildkitImage: string; + }; + build: { + dockerfiles: { api: string; worker: string; web: string }; + images: { api: string; worker: string; web: string }; + networkMode: string; + }; + gitops: { + readUrl: string; + writeUrl: string; + branch: string; + manifestPath: string; + credentialSecretName: string; + credentialTokenKey: string; + credentialUsername: string; + authorName: string; + authorEmail: string; + }; database: { configRef: string; name: string; @@ -103,7 +143,8 @@ interface TestTargetSpec { employee: { username: string; password: SecretSourceSpec }; sessionSecret: SecretSourceSpec; }; - migration: { + initializer: { + mode: "fresh-database-only"; jobName: string; backoffLimit: number; command: string[]; @@ -155,24 +196,8 @@ interface Blocker { message: string; } -interface RenderContext { - instanceId: string; - commit: string; - step: TestTargetStep; - namespace: string; - expiresAt: string; - apiImage: string; - workerImage: string; - webImage: string; - migrationImage: string; -} - -interface SecretMaterial { - databaseURL: string; - sessionSecret: string; - administratorPassword: string; - employeePassword: string; -} +type RenderContext = PikaoaTestRenderContext; +type SecretMaterial = PikaoaTestSecretMaterial; type RemoteCapture = typeof capture; @@ -194,14 +219,14 @@ export function pikaoaTestTargetHelp(): Record { usage: [ "bun scripts/cli.ts pikaoa test-target plan [--config path] [--target id] [--instance id] [--commit sha] [--output json]", "bun scripts/cli.ts pikaoa test-target status [--config path] [--target id] [--instance id] [--output json]", - "bun scripts/cli.ts pikaoa test-target start --target id --instance id [--commit sha] [--step foundation|migration|api|worker|web|all] --confirm", + "bun scripts/cli.ts pikaoa test-target start --target id --instance id [--commit sha] [--step foundation|init|api|worker|web|all] --confirm", "bun scripts/cli.ts pikaoa test-target stop --target id --instance id --confirm", ], source: "config/pikaoa.yaml#testRuntime", guarantees: [ "未声明测试 target 时 plan/status 返回 configured=false，start/stop 在远端调用前失败。", "validationOnly target 只用于本地渲染，不连接 route。", - "start 可按 foundation、migration、api、worker、web 或 all 单步提交并立即返回；foundation 只创建固定 Namespace/PVC，不要求业务 Secret 已存在。", + "start 可按 foundation、init、api、worker、web 或 all 单步提交并立即返回；foundation 只创建固定 Namespace/PVC，不要求业务 Secret 已存在。", "正式 delivery namespace、PaC、GitOps、公网暴露和生产数据库不属于该入口的写入范围。", "Secret 只披露 sourceRef、targetKey、presence 和 fingerprint，不输出值。", ], @@ -240,7 +265,7 @@ function parseOptions(args: string[]): TestTargetOptions { else if (arg === "--target") targetId = simpleId(value, arg); else if (arg === "--instance") instanceId = kubernetesName(value, arg, 30); else if (arg === "--commit") commit = sourceCommit(value); - else if (arg === "--step") step = enumString(value, arg, ["all", "foundation", "migration", "api", "worker", "web"]); + else if (arg === "--step") step = enumString(value, arg, ["all", "foundation", "init", "api", "worker", "web"]); else { if (value !== "json" && value !== "text") throw inputError(`${arg} 只支持 text 或 json`, "invalid-output", value, ["text", "json"]); output = value; @@ -298,6 +323,13 @@ function parseTarget(id: string, root: Record, configLabel: str simpleId(id, "target id"); const source = record(root.source, `${path}.source`); const images = record(source.images, `${path}.source.images`); + const ci = record(root.ci, `${path}.ci`); + const build = record(root.build, `${path}.build`); + const dockerfiles = record(build.dockerfiles, `${path}.build.dockerfiles`); + const buildImages = record(build.images, `${path}.build.images`); + const gitops = record(root.gitops, `${path}.gitops`); + const gitopsCredential = record(gitops.credential, `${path}.gitops.credential`); + const gitopsAuthor = record(gitops.author, `${path}.gitops.author`); const database = record(root.database, `${path}.database`); const runtime = record(root.runtime, `${path}.runtime`); const attachment = record(runtime.attachment, `${path}.runtime.attachment`); @@ -313,7 +345,7 @@ function parseTarget(id: string, root: Record, configLabel: str const workerProbes = record(probes.worker, `${path}.probes.worker`); const webProbes = record(probes.web, `${path}.probes.web`); const cleanup = record(root.cleanup, `${path}.cleanup`); - const migration = record(root.migration, `${path}.migration`); + const initializer = record(root.initializer, `${path}.initializer`); const taskState = record(root.taskState, `${path}.taskState`); if (exporterFailure.blocking !== false) throw new Error(`${path}.observability.exporterFailure.blocking 必须为 false`); if (exporterFailure.warning !== true) throw new Error(`${path}.observability.exporterFailure.warning 必须为 true`); @@ -341,12 +373,51 @@ function parseTarget(id: string, root: Record, configLabel: str mode: exactString(source.mode, `${path}.source.mode`, "prebuilt-images"), repository: nonEmpty(source.repository, `${path}.source.repository`), commitPolicy: exactString(source.commitPolicy, `${path}.source.commitPolicy`, "cli-required"), + branch: nonEmpty(source.branch, `${path}.source.branch`), + worktreeRemote: nonEmpty(source.worktreeRemote, `${path}.source.worktreeRemote`), + readUrl: nonEmpty(source.readUrl, `${path}.source.readUrl`), + snapshotPrefix: nonEmpty(source.snapshotPrefix, `${path}.source.snapshotPrefix`), apiImage: imageReference(images.api, `${path}.source.images.api`), workerImage: imageReference(images.worker, `${path}.source.images.worker`), webImage: imageReference(images.web, `${path}.source.images.web`), - migrationImage: imageReference(images.migration, `${path}.source.images.migration`), + initializerImage: imageReference(images.initializer, `${path}.source.images.initializer`), pullPolicy: enumString(source.pullPolicy, `${path}.source.pullPolicy`, ["Always", "IfNotPresent", "Never"]), }, + ci: { + lane: simpleId(nonEmpty(ci.lane, `${path}.ci.lane`), `${path}.ci.lane`), + namespace: kubernetesName(nonEmpty(ci.namespace, `${path}.ci.namespace`), `${path}.ci.namespace`, 63), + pipeline: kubernetesName(nonEmpty(ci.pipeline, `${path}.ci.pipeline`), `${path}.ci.pipeline`, 63), + pipelineRunPrefix: kubernetesName(nonEmpty(ci.pipelineRunPrefix, `${path}.ci.pipelineRunPrefix`), `${path}.ci.pipelineRunPrefix`, 63), + serviceAccountName: kubernetesName(nonEmpty(ci.serviceAccountName, `${path}.ci.serviceAccountName`), `${path}.ci.serviceAccountName`, 63), + workspaceSize: nonEmpty(ci.workspaceSize, `${path}.ci.workspaceSize`), + pipelineTimeout: nonEmpty(ci.pipelineTimeout, `${path}.ci.pipelineTimeout`), + toolsImage: imageReference(ci.toolsImage, `${path}.ci.toolsImage`), + buildkitImage: imageReference(ci.buildkitImage, `${path}.ci.buildkitImage`), + }, + build: { + dockerfiles: { + api: relativeSourcePath(dockerfiles.api, `${path}.build.dockerfiles.api`), + worker: relativeSourcePath(dockerfiles.worker, `${path}.build.dockerfiles.worker`), + web: relativeSourcePath(dockerfiles.web, `${path}.build.dockerfiles.web`), + }, + images: { + api: imageRepository(buildImages.api, `${path}.build.images.api`), + worker: imageRepository(buildImages.worker, `${path}.build.images.worker`), + web: imageRepository(buildImages.web, `${path}.build.images.web`), + }, + networkMode: exactString(build.networkMode, `${path}.build.networkMode`, "host"), + }, + gitops: { + readUrl: nonEmpty(gitops.readUrl, `${path}.gitops.readUrl`), + writeUrl: nonEmpty(gitops.writeUrl, `${path}.gitops.writeUrl`), + branch: nonEmpty(gitops.branch, `${path}.gitops.branch`), + manifestPath: relativeSourcePath(gitops.manifestPath, `${path}.gitops.manifestPath`), + credentialSecretName: kubernetesName(nonEmpty(gitopsCredential.secretName, `${path}.gitops.credential.secretName`), `${path}.gitops.credential.secretName`, 63), + credentialTokenKey: nonEmpty(gitopsCredential.tokenKey, `${path}.gitops.credential.tokenKey`), + credentialUsername: nonEmpty(gitopsCredential.username, `${path}.gitops.credential.username`), + authorName: nonEmpty(gitopsAuthor.name, `${path}.gitops.author.name`), + authorEmail: nonEmpty(gitopsAuthor.email, `${path}.gitops.author.email`), + }, database: { configRef: nonEmpty(database.configRef, `${path}.database.configRef`), name: postgresIdentifier(database.name, `${path}.database.name`), @@ -386,10 +457,11 @@ function parseTarget(id: string, root: Record, configLabel: str }, sessionSecret: secretSource(runtime.sessionSecret, "runtime.sessionSecret", `${path}.runtime.sessionSecret`), }, - migration: { - jobName: kubernetesName(nonEmpty(migration.jobName, `${path}.migration.jobName`), `${path}.migration.jobName`, 63), - backoffLimit: nonNegativeInteger(migration.backoffLimit, `${path}.migration.backoffLimit`), - command: nonEmptyStringList(migration.command, `${path}.migration.command`), + initializer: { + mode: exactString(initializer.mode, `${path}.initializer.mode`, "fresh-database-only") as "fresh-database-only", + jobName: kubernetesName(nonEmpty(initializer.jobName, `${path}.initializer.jobName`), `${path}.initializer.jobName`, 63), + backoffLimit: nonNegativeInteger(initializer.backoffLimit, `${path}.initializer.backoffLimit`), + command: nonEmptyStringList(initializer.command, `${path}.initializer.command`), }, exposure: { serviceName: kubernetesName(nonEmpty(exposure.serviceName, `${path}.exposure.serviceName`), `${path}.exposure.serviceName`, 63), @@ -422,6 +494,20 @@ function parseTarget(id: string, root: Record, configLabel: str return target; } +export function pikaoaTestRuntimeConfigRef(targetId: string): string { + simpleId(targetId, "PikaOA test runtime target id"); + return `config/pikaoa.yaml#testRuntime.targets.${targetId}`; +} + +export function resolvePikaoaTestRuntimeTargetByConfigRef(configRef: string): TestTargetSpec { + const resolved = resolveConfigRef(configRef, "PikaOA test runtime configRef"); + const match = /^testRuntime\.targets\.([A-Za-z0-9._-]+)$/u.exec(resolved.fragment); + if (resolved.file !== "config/pikaoa.yaml" || match === null) { + throw new Error("PikaOA test runtime configRef must select config/pikaoa.yaml#testRuntime.targets."); + } + return parseTarget(match[1]!, record(resolved.value, configRef), resolved.file); +} + function planPayload(options: TestTargetOptions, selection: Selection): Record { const base = basePayload(options, selection); if (selection.target === null) return base; @@ -486,7 +572,10 @@ function renderedPlan(selection: Selection, context: RenderContext): Record secretSourceSummary(selection.configPath, source)), blockers: [] as Blocker[], } : readSecretMaterial(selection, selection.target); if (!material.ok) { return renderResult({ ...planned, ok: false, mutation: false, remoteQueried: false, blockers: material.blockers, secret: { sources: material.sources, valuesPrinted: false }, failure: "secret-preflight-blocked" }, options); } - const manifest = manifestForStep(buildManifest(selection.target, context, material.values), options.step); + const manifest = pikaoaManifestForStep(renderPikaoaTestRuntimeManifest(selection.target, context, { + secretMode: "materialized", + secrets: material.values, + }), options.step); const remote = await remoteCapture(config, selection.target.route, ["sh"], startScript(selection.target, context, manifest)); const parsed = parseJsonOutput(remote.stdout); const mutation = parsed?.mutation === true; @@ -647,7 +740,7 @@ function renderContext(options: TestTargetOptions, target: TestTargetSpec, requi apiImage: renderImage(target.source.apiImage, commit), workerImage: renderImage(target.source.workerImage, commit), webImage: renderImage(target.source.webImage, commit), - migrationImage: renderImage(target.source.migrationImage, commit), + initializerImage: renderImage(target.source.initializerImage, commit), }; } @@ -674,9 +767,9 @@ function safetyBlockers(options: TestTargetOptions, selection: Selection, contex function consistencyWarnings(target: TestTargetSpec, context: RenderContext | null): Warning[] { const warnings: Warning[] = []; - if (!target.migration.command.includes("migrate") || !target.migration.command.includes("up")) warnings.push(warning("migration-command-unrecognized", "迁移命令未显示产品 migrate up；配置一致性只报告 warning。")); - if (![target.source.apiImage, target.source.workerImage, target.source.webImage, target.source.migrationImage].every((image) => image.includes("${commit}"))) warnings.push(warning("image-template-not-commit-scoped", "API、Worker、Web 或迁移镜像模板未包含 ${commit}；版本一致性只报告 warning。")); - if (context !== null && ![context.apiImage, context.workerImage, context.webImage, context.migrationImage].every((image) => image.includes(context.commit))) warnings.push(warning("image-commit-not-visible", "渲染镜像引用未显示选中 commit；该漂移不阻塞当前 MVP。")); + if (!target.initializer.command.includes("migrate") || !target.initializer.command.includes("up")) warnings.push(warning("initializer-command-unrecognized", "空库 initializer 的当前产品命令未显示 migrate up；配置一致性只报告 warning。")); + if (![target.source.apiImage, target.source.workerImage, target.source.webImage, target.source.initializerImage].every((image) => image.includes("${commit}"))) warnings.push(warning("image-template-not-commit-scoped", "API、Worker、Web 或 initializer 镜像模板未包含 ${commit}；版本一致性只报告 warning。")); + if (context !== null && ![context.apiImage, context.workerImage, context.webImage, context.initializerImage].every((image) => image.includes(context.commit))) warnings.push(warning("image-commit-not-visible", "渲染镜像引用未显示选中 commit；该漂移不阻塞当前 MVP。")); if (target.observability.exporterWarning) warnings.push(warning("otel-exporter-warning-only", "OTel exporter 失败按 owning YAML 仅报告 warning，不阻塞业务运行。")); if (!target.observability.prometheusScrape) warnings.push(warning("prometheus-scrape-disabled", "Prometheus scrape 已在 target YAML 中关闭。")); return warnings; @@ -717,178 +810,6 @@ function prometheusRuntimeStatus(items: Record[], selector: Pro }; } -function buildManifest(target: TestTargetSpec, context: RenderContext, secrets: SecretMaterial): Record[] { - const labels = ownershipLabels(target, context); - const secretData = runtimeSecretData(target, context, secrets); - const apiSelector = componentSelector("api", context.instanceId); - const workerSelector = componentSelector("worker", context.instanceId); - const webSelector = componentSelector("web", context.instanceId); - const migrationSelector = componentSelector("migration", context.instanceId); - const prometheusPodLabels = target.observability.prometheusScrape ? { - [target.observability.prometheusSelector.key]: target.observability.prometheusSelector.value, - } : {}; - const commonPodAnnotations = (path: string, port: number): Record => target.observability.prometheusScrape ? { - "prometheus.io/scrape": "true", - "prometheus.io/path": path, - "prometheus.io/port": String(port), - } : {}; - return [ - { - apiVersion: "v1", kind: "Namespace", metadata: { name: context.namespace, labels, annotations: namespaceAnnotations(target, context) }, - }, - { - apiVersion: "v1", kind: "Secret", metadata: { name: target.runtime.secretName, namespace: context.namespace, labels }, type: "Opaque", stringData: secretData, - }, - { - apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: target.runtime.attachment.claimName, namespace: context.namespace, labels }, - spec: { - accessModes: target.runtime.attachment.accessModes, - resources: { requests: { storage: target.runtime.attachment.storageRequest } }, - ...(target.runtime.attachment.storageClassName === null ? {} : { storageClassName: target.runtime.attachment.storageClassName }), - }, - }, - { - apiVersion: "v1", kind: "Service", metadata: { name: target.runtime.apiServiceName, namespace: context.namespace, labels: { ...labels, ...apiSelector } }, - spec: { selector: apiSelector, ports: [{ name: "http", port: target.runtime.apiPort, targetPort: target.runtime.apiPort }] }, - }, - { - apiVersion: "v1", kind: "Service", metadata: { name: target.exposure.serviceName, namespace: context.namespace, labels: { ...labels, ...webSelector } }, - spec: { - type: target.exposure.serviceType, - selector: webSelector, - ports: [{ name: "http", port: target.runtime.webPort, targetPort: target.runtime.webPort, nodePort: target.exposure.port }], - }, - }, - { - apiVersion: "batch/v1", kind: "Job", metadata: { name: target.migration.jobName, namespace: context.namespace, labels: { ...labels, ...migrationSelector } }, - spec: { - backoffLimit: target.migration.backoffLimit, - template: { - metadata: { labels: { ...labels, ...migrationSelector } }, - spec: { - restartPolicy: "Never", - containers: [{ - name: "migration", image: context.migrationImage, imagePullPolicy: target.source.pullPolicy, - command: target.migration.command, - env: [{ name: "PIKAOA_CONFIG", value: "/etc/pikaoa/pikaoa.yaml" }], - volumeMounts: [{ name: "runtime", mountPath: "/etc/pikaoa", readOnly: true }], - }], - volumes: [{ name: "runtime", secret: { secretName: target.runtime.secretName, items: [{ key: "pikaoa.yaml", path: "pikaoa.yaml" }] } }], - }, - }, - }, - }, - deployment(target, context, "api", context.apiImage, apiSelector, target.runtime.apiPort, prometheusPodLabels, commonPodAnnotations(target.observability.apiMetricsPath, target.runtime.apiPort), [ - { name: "PIKAOA_CONFIG", value: "/etc/pikaoa/pikaoa.yaml" }, - { name: "PIKAOA_SESSION_SECRET", valueFrom: { secretKeyRef: { name: target.runtime.secretName, key: target.runtime.sessionSecret.targetKey } } }, - { name: "PIKAOA_BOOTSTRAP_ADMIN_USERNAME", value: target.runtime.administrator.username }, - { name: "PIKAOA_BOOTSTRAP_ADMIN_PASSWORD", valueFrom: { secretKeyRef: { name: target.runtime.secretName, key: target.runtime.administrator.password.targetKey } } }, - { name: "PIKAOA_BOOTSTRAP_EMPLOYEE_USERNAME", value: target.runtime.employee.username }, - { name: "PIKAOA_BOOTSTRAP_EMPLOYEE_PASSWORD", valueFrom: { secretKeyRef: { name: target.runtime.secretName, key: target.runtime.employee.password.targetKey } } }, - ], target.probes.apiLivenessPath, target.probes.apiReadinessPath), - deployment(target, context, "worker", context.workerImage, workerSelector, target.runtime.workerMetricsPort, prometheusPodLabels, commonPodAnnotations(target.observability.workerMetricsPath, target.runtime.workerMetricsPort), [ - { name: "PIKAOA_CONFIG", value: "/etc/pikaoa/pikaoa.yaml" }, - ], target.probes.workerLivenessPath, target.probes.workerReadinessPath), - deployment(target, context, "web", context.webImage, webSelector, target.runtime.webPort, {}, {}, [ - { name: target.runtime.webApiUpstreamEnv, value: `http://${target.runtime.apiServiceName}:${target.runtime.apiPort}` }, - ], target.probes.webLivenessPath, target.probes.webReadinessPath), - ]; -} - -function manifestForStep(manifest: Record[], step: TestTargetStep): Record[] { - if (step === "all") return manifest; - return manifest.filter((object) => { - if (object.kind === "Namespace" || object.kind === "PersistentVolumeClaim") return true; - if (step === "foundation") return false; - if (object.kind === "Secret") return true; - if (step === "migration") return object.kind === "Job"; - const metadata = optionalRecord(object.metadata, "metadata") ?? {}; - const labels = optionalRecord(metadata.labels, "metadata.labels") ?? {}; - return labels["app.kubernetes.io/component"] === step; - }); -} - -function deployment( - target: TestTargetSpec, - context: RenderContext, - component: "api" | "worker" | "web", - image: string, - selector: Record, - port: number, - podLabels: Record, - annotations: Record, - env: Array>, - livenessPath: string, - readinessPath: string, -): Record { - const labels = ownershipLabels(target, context); - return { - apiVersion: "apps/v1", kind: "Deployment", metadata: { name: `pikaoa-${component}`, namespace: context.namespace, labels: { ...labels, ...selector } }, - spec: { - replicas: 1, selector: { matchLabels: selector }, - template: { - metadata: { labels: { ...labels, ...selector, ...podLabels }, annotations }, - spec: { - ...(component === "api" ? { securityContext: { fsGroup: target.runtime.attachment.fsGroup } } : {}), - containers: [{ - name: component, image, imagePullPolicy: target.source.pullPolicy, env, - ports: [{ name: component === "worker" ? "metrics" : "http", containerPort: port }], - volumeMounts: [ - ...(component === "web" ? [] : [{ name: "runtime", mountPath: "/etc/pikaoa", readOnly: true }]), - ...(component === "api" ? [{ name: target.runtime.attachment.claimName, mountPath: target.runtime.attachment.storageRoot }] : []), - ], - livenessProbe: { httpGet: { path: livenessPath, port }, periodSeconds: 5, failureThreshold: 12 }, - readinessProbe: { httpGet: { path: readinessPath, port }, periodSeconds: 3, failureThreshold: 20 }, - }], - volumes: [ - ...(component === "web" ? [] : [{ name: "runtime", secret: { secretName: target.runtime.secretName, items: [{ key: "pikaoa.yaml", path: "pikaoa.yaml" }] } }]), - ...(component === "api" ? [{ name: target.runtime.attachment.claimName, persistentVolumeClaim: { claimName: target.runtime.attachment.claimName } }] : []), - ], - }, - }, - }, - }; -} - -function runtimeSecretData(target: TestTargetSpec, context: RenderContext, secrets: SecretMaterial): Record { - const databaseURL = renderDatabaseURL(target, secrets.databaseURL); - return { - [target.database.connection.targetKey]: databaseURL, - [target.runtime.sessionSecret.targetKey]: secrets.sessionSecret, - [target.runtime.administrator.password.targetKey]: secrets.administratorPassword, - [target.runtime.employee.password.targetKey]: secrets.employeePassword, - "pikaoa.yaml": [ - "api:", - ` listen: 0.0.0.0:${target.runtime.apiPort}`, - "worker:", - ` interval: ${target.runtime.workerInterval}`, - ` metricsListen: 0.0.0.0:${target.runtime.workerMetricsPort}`, - " outbox:", - ` batchSize: ${target.runtime.outboxBatchSize}`, - ` maxAttempts: ${target.runtime.outboxMaxAttempts}`, - ` retryDelay: ${target.runtime.outboxRetryDelay}`, - "database:", - ` url: ${JSON.stringify(databaseURL)}`, - "identity:", - ` sessionTTL: ${target.runtime.sessionTTL}`, - "attachment:", - ` storageRoot: ${JSON.stringify(target.runtime.attachment.storageRoot)}`, - ` maxBytes: ${target.runtime.attachment.maxBytes}`, - "observability:", - ` serviceName: pikaoa-test-${context.instanceId}`, - ` otlpEndpoint: ${JSON.stringify(target.observability.otlpEndpoint)}`, - `shutdownTimeout: ${target.runtime.shutdownTimeout}`, - "", - ].join("\n"), - }; -} - -function renderDatabaseURL(target: TestTargetSpec, sourceURL: string): string { - const databaseURL = new URL(sourceURL); - databaseURL.searchParams.set("search_path", target.database.schema); - return databaseURL.toString(); -} - function databasePlanSummary(target: TestTargetSpec): Record { const databaseURL = new URL(renderDatabaseURL(target, "postgresql://redacted@db.invalid/pikaoa_test?sslmode=require")); return { @@ -1134,7 +1055,7 @@ printf '{"ok":true,"mutation":true,"code":"start-submitted","submitted":true,"pr } function startRunnerScript(target: TestTargetSpec, context: RenderContext, manifest: Record[], job: AsyncJobIdentity): string { - const migration = manifest.filter((object) => object.kind === "Job"); + const initializer = manifest.filter((object) => object.kind === "Job"); const workload = (name: string): Record[] => manifest.filter((object) => { if (object.kind !== "Deployment" && object.kind !== "Service") return false; const metadata = optionalRecord(object.metadata, "metadata") ?? {}; @@ -1146,7 +1067,7 @@ function startRunnerScript(target: TestTargetSpec, context: RenderContext, manif const encode = (objects: Record[]): string => Buffer.from(objects.map((object) => JSON.stringify(object)).join("\n---\n"), "utf8").toString("base64"); const foundationEncoded = encode(foundation); const runtimeEncoded = encode(runtime); - const migrationEncoded = encode(migration); + const initializerEncoded = encode(initializer); const apiEncoded = encode(workload("api")); const workerEncoded = encode(workload("worker")); const webEncoded = encode(workload("web")); @@ -1228,7 +1149,7 @@ if kubectl get namespace "$namespace" --request-timeout=${target.taskState.statu fi printf '%s' ${shQuote(foundationEncoded)} | base64 -d >"$tmp/foundation.yaml" printf '%s' ${shQuote(runtimeEncoded)} | base64 -d >"$tmp/runtime.yaml" -printf '%s' ${shQuote(migrationEncoded)} | base64 -d >"$tmp/migration.yaml" +printf '%s' ${shQuote(initializerEncoded)} | base64 -d >"$tmp/initializer.yaml" printf '%s' ${shQuote(apiEncoded)} | base64 -d >"$tmp/api.yaml" printf '%s' ${shQuote(workerEncoded)} | base64 -d >"$tmp/worker.yaml" printf '%s' ${shQuote(webEncoded)} | base64 -d >"$tmp/web.yaml" @@ -1249,22 +1170,22 @@ if ! kubectl apply --server-side --force-conflicts --field-manager=${shQuote(tar failure_code=runtime-secret-apply-failed exit 44 fi -if [ "$step" = all ] || [ "$step" = migration ]; then - current_phase=migration-apply +if [ "$step" = all ] || [ "$step" = init ]; then + current_phase=initializer-apply write_status running "$current_phase" start-running null check_canceled - kubectl -n "$namespace" delete job/${target.migration.jobName} --ignore-not-found --wait=false >/dev/null 2>&1 || true - if ! kubectl apply --server-side --force-conflicts --field-manager=${shQuote(target.fieldManager)} -f "$tmp/migration.yaml" >"$tmp/migration-apply.out" 2>"$tmp/migration-apply.err"; then failure_code=migration-apply-failed; exit 45; fi - current_phase=migration-wait + kubectl -n "$namespace" delete job/${target.initializer.jobName} --ignore-not-found --wait=false >/dev/null 2>&1 || true + if ! kubectl apply --server-side --force-conflicts --field-manager=${shQuote(target.fieldManager)} -f "$tmp/initializer.yaml" >"$tmp/initializer-apply.out" 2>"$tmp/initializer-apply.err"; then failure_code=initializer-apply-failed; exit 45; fi + current_phase=initializer-wait write_status running "$current_phase" start-running null - migration_deadline=$(( $(date +%s) + ${target.waitTimeoutSeconds} )) + initializer_deadline=$(( $(date +%s) + ${target.waitTimeoutSeconds} )) while :; do check_canceled - if ! kubectl -n "$namespace" get job/${target.migration.jobName} --request-timeout=${Math.min(target.taskState.statusRequestTimeoutSeconds, 5)}s -o json >"$tmp/migration-job.json" 2>"$tmp/migration-wait.err"; then - failure_code=migration-job-query-failed + if ! kubectl -n "$namespace" get job/${target.initializer.jobName} --request-timeout=${Math.min(target.taskState.statusRequestTimeoutSeconds, 5)}s -o json >"$tmp/initializer-job.json" 2>"$tmp/initializer-wait.err"; then + failure_code=initializer-job-query-failed exit 46 fi - if ! migration_condition="$(python3 - "$tmp/migration-job.json" <<'PY' + if ! initializer_condition="$(python3 - "$tmp/initializer-job.json" <<'PY' import json, pathlib, sys payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) conditions = { @@ -1282,16 +1203,16 @@ else: print("running") PY )"; then - failure_code=migration-job-condition-invalid + failure_code=initializer-job-condition-invalid exit 46 fi - case "$migration_condition" in + case "$initializer_condition" in complete) break ;; - failure-target) failure_code=migration-job-failure-target; exit 46 ;; - failed) failure_code=migration-job-failed; exit 46 ;; + failure-target) failure_code=initializer-job-failure-target; exit 46 ;; + failed) failure_code=initializer-job-failed; exit 46 ;; esac - if [ "$(date +%s)" -ge "$migration_deadline" ]; then - failure_code=migration-job-timeout + if [ "$(date +%s)" -ge "$initializer_deadline" ]; then + failure_code=initializer-job-timeout exit 46 fi sleep 1 @@ -1567,8 +1488,8 @@ function sourceSummary(target: TestTargetSpec, commit: string | null, context?: commitPolicy: target.source.commitPolicy, commit, images: context === undefined - ? { apiTemplate: target.source.apiImage, workerTemplate: target.source.workerImage, webTemplate: target.source.webImage, migrationTemplate: target.source.migrationImage } - : { api: context.apiImage, worker: context.workerImage, web: context.webImage, migration: context.migrationImage }, + ? { apiTemplate: target.source.apiImage, workerTemplate: target.source.workerImage, webTemplate: target.source.webImage, initializerTemplate: target.source.initializerImage } + : { api: context.apiImage, worker: context.workerImage, web: context.webImage, initializer: context.initializerImage }, }; } @@ -1662,10 +1583,6 @@ function manifestFingerprint(manifest: Record[], stableRequest: return `sha256:${createHash("sha256").update(JSON.stringify(normalized)).digest("hex")}`; } -function placeholderSecrets(): SecretMaterial { - return { databaseURL: "postgresql://redacted@db.invalid/pikaoa_test?sslmode=require", sessionSecret: "", administratorPassword: "", employeePassword: "" }; -} - function secretSources(target: TestTargetSpec): SecretSourceSpec[] { return [target.database.connection, target.runtime.sessionSecret, target.runtime.administrator.password, target.runtime.employee.password]; } @@ -1760,6 +1677,18 @@ function imageReference(value: unknown, path: string): string { return image; } +function imageRepository(value: unknown, path: string): string { + const image = nonEmpty(value, path); + if (/\s/u.test(image) || image.includes("@") || /:[^/]+$/u.test(image)) throw new Error(`${path} 必须是不带 tag/digest 的镜像仓库`); + return image; +} + +function relativeSourcePath(value: unknown, path: string): string { + const sourcePath = nonEmpty(value, path); + if (sourcePath.startsWith("/") || sourcePath.split(/[\\/]/u).includes("..")) throw new Error(`${path} 必须是无 .. 的仓库相对路径`); + return sourcePath; +} + function otlpGrpcEndpoint(value: unknown, path: string): string { const endpoint = nonEmpty(value, path); if (endpoint.includes("://") || !/^[A-Za-z0-9.-]+:[1-9][0-9]{0,4}$/u.test(endpoint)) throw new Error(`${path} 必须是无 scheme 的 OTLP gRPC host:port`); diff --git a/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts b/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts index 7244c5d8..e769fe11 100644 --- a/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts +++ b/scripts/src/platform-infra-pipelines-as-code-bootstrap.test.ts @@ -120,19 +120,19 @@ test("unselected parse and repository errors remain warnings while selected obje expect(() => parsePacConfigDocument(invalidConsumerDocument, { consumerId: "selfmedia-production-nc01" })).toThrow("namespace"); const invalidRepositoryDocument = pacConfigDocument(); - const invalidRepository = (invalidRepositoryDocument.repositories as Array>).find((item) => item.id === "pikaoa-nc01"); + const invalidRepository = (invalidRepositoryDocument.repositories as Array>).find((item) => item.id === "pikaoa-test-nc01"); if (invalidRepository === undefined) throw new Error("pikaoa repository fixture is required"); invalidRepository.cloneUrl = "https://gitea.example.invalid/mirrors/pikaoa.git"; const isolated = parsePacConfigDocument(invalidRepositoryDocument, { consumerId: "selfmedia-nc01" }); expect(isolated.validationWarnings).toContainEqual({ code: "pac-unselected-repository-invalid", - object: { kind: "repository", id: "pikaoa-nc01" }, + object: { kind: "repository", id: "pikaoa-test-nc01" }, consumer: null, - configPath: "config/platform-infra/pipelines-as-code.yaml#repositories.pikaoa-nc01", + configPath: "config/platform-infra/pipelines-as-code.yaml#repositories.pikaoa-test-nc01", blocking: false, evidence: expect.stringMatching(/^type=string,length=\d+,sha256=[0-9a-f]{64}$/u), }); - expect(validPacConsumers(isolated).some((consumer) => consumer.repositoryRef === "pikaoa-nc01")).toBe(false); + expect(validPacConsumers(isolated).some((consumer) => consumer.repositoryRef === "pikaoa-test-nc01")).toBe(false); expect(() => parsePacConfigDocument(invalidRepositoryDocument, { consumerId: "pikaoa-nc01" })).toThrow("cloneUrl"); const duplicateSelectedRepository = pacConfigDocument(); diff --git a/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts b/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts index b35ac4e8..00e9525f 100644 --- a/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts +++ b/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts @@ -17,6 +17,9 @@ import { canonicalizePacPipelineSpec, firstPacSourceArtifactDrift, pacSourceArtifactSafeError, + pacSourceArtifactProvenanceFromManifest, + pipelineRunLabels, + pipelineRunWorkspaces, pacValueEvidence, pacSourceArtifactYaml, optionalPipelineRunWorkspaces, @@ -96,6 +99,46 @@ function runtimeInput(commit: string | null = sourceCommit): Record { + test("accepts PikaOA test runtime provenance", () => { + const expected = { + configRef: "config/pikaoa.yaml#testRuntime.targets.NC01", + effectiveConfigSha256: `sha256:${"a".repeat(64)}`, + renderer: "pikaoa-test-runtime", + mode: "embedded-pipeline-spec", + } as const; + const manifest = { + metadata: { annotations: pipelineProvenanceAnnotations(expected) }, + }; + expect(pacSourceArtifactProvenanceFromManifest(manifest)).toEqual(expected); + }); + + test("uses PikaOA-specific PipelineRun labels", () => { + const binding = { consumer: { id: "pikaoa-test-nc01", node: "NC01" } } as Parameters[0]; + expect(pipelineRunLabels(binding, "pikaoa")).toEqual({ + "app.kubernetes.io/name": "pikaoa-test-nc01-pac", + "app.kubernetes.io/part-of": "pikaoa", + "unidesk.ai/source-commit": "{{ revision }}", + "pikaoa.unidesk.io/source-commit": "{{ revision }}", + "pikaoa.unidesk.io/trigger": "pipelines-as-code", + }); + }); + + test("maps the PikaOA shared workspace to the declared RWO PVC size", () => { + const binding = { + consumer: { sourceArtifact: { renderer: "pikaoa-test-runtime" } }, + repository: { params: { workspace_pvc_size: "8Gi" } }, + } as Parameters[0]; + expect(pipelineRunWorkspaces(binding, { workspaces: [{ name: "workspace" }] })).toEqual([{ + name: "workspace", + volumeClaimTemplate: { + spec: { + accessModes: ["ReadWriteOnce"], + resources: { requests: { storage: "8Gi" } }, + }, + }, + }]); + }); + test("omits empty PipelineRun workspaces and preserves declared bindings", () => { expect(optionalPipelineRunWorkspaces([])).toEqual({}); const workspaces = [{ name: "source", emptyDir: {} }]; diff --git a/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts b/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts index cc319174..ba6054f8 100644 --- a/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts +++ b/scripts/src/platform-infra-pipelines-as-code-source-artifact.ts @@ -19,9 +19,10 @@ import { stableJsonSha256, stableJsonValue } from "./stable-json"; import { pipelineProvenanceAnnotations, pipelineProvenanceFromManifest, withPipelineProvenanceAnnotations } from "./pipeline-provenance"; import { renderSub2RankDesiredPipeline, sub2RankOwningSourceRemote } from "./platform-infra-sub2rank-pipeline"; import { renderSelfMediaDesiredPipeline, selfMediaSourceWorktreeRemote } from "./selfmedia-delivery-renderer"; +import { renderPikaoaTestDesiredPipeline, pikaoaTestSourceWorktreeRemote } from "./pikaoa-test-delivery-renderer"; export type PacSourceArtifactMode = "embedded-pipeline-spec" | "remote-pipeline-annotation"; -export type PacSourceArtifactRenderer = "agentrun-control-plane" | "hwlab-runtime-lane" | "sub2rank-platform-service" | "selfmedia-runtime"; +export type PacSourceArtifactRenderer = "agentrun-control-plane" | "hwlab-runtime-lane" | "sub2rank-platform-service" | "selfmedia-runtime" | "pikaoa-test-runtime"; export type PacSourceArtifactAction = "plan" | "check" | "write" | "status" | "verify-runtime"; export type PacSourceArtifactRuntimeAlignment = "not-requested" | "aligned" | "drifted" | "missing" | "unavailable"; @@ -499,11 +500,13 @@ function renderDesiredArtifact(binding: PacSourceArtifactBinding, sourceWorktree ? renderHwlabDesiredPipeline(binding, sourceWorktree) : sourceArtifact.renderer === "sub2rank-platform-service" ? renderSub2RankDesiredPipeline(binding) - : renderSelfMediaPipeline(binding, sourceWorktree); + : sourceArtifact.renderer === "selfmedia-runtime" + ? renderSelfMediaPipeline(binding, sourceWorktree) + : renderPikaoaTestPipeline(binding, sourceWorktree); const pipeline = sourceArtifact.renderer === "hwlab-runtime-lane" ? rendered.pipeline : withPipelineProvenanceAnnotations(rendered.pipeline, rendered.provenance); - if (!provenanceEquals(provenanceFromManifest(pipeline), rendered.provenance)) { + if (!provenanceEquals(pacSourceArtifactProvenanceFromManifest(pipeline), rendered.provenance)) { throw new Error(`${sourceArtifact.renderer} did not render the declared Pipeline provenance contract`); } const desiredSpec = requiredRecord(pipeline.spec, "rendered Pipeline.spec"); @@ -583,6 +586,19 @@ function renderSelfMediaPipeline(binding: PacSourceArtifactBinding, sourceWorktr }; } +function renderPikaoaTestPipeline(binding: PacSourceArtifactBinding, sourceWorktree: string): { pipeline: Record; provenance: PacSourceArtifactProvenance } { + const rendered = renderPikaoaTestDesiredPipeline(binding, sourceWorktree); + return { + pipeline: rendered.pipeline, + provenance: { + configRef: rendered.configRef, + effectiveConfigSha256: rendered.effectiveConfigSha256, + renderer: "pikaoa-test-runtime", + mode: "embedded-pipeline-spec", + }, + }; +} + function renderHwlabPipelineFromOwningYaml(spec: HwlabRuntimeLaneSpec, sourceWorktree: string): Record { const temporaryRoot = mkdtempSync(resolve(tmpdir(), "unidesk-pac-source-artifact-")); const temporarySource = resolve(temporaryRoot, "source"); @@ -659,7 +675,9 @@ function embeddedPipelineRun(binding: PacSourceArtifactBinding, desiredSpec: Rec ? "sub2rank" : provenance.renderer === "selfmedia-runtime" ? "selfmedia" - : "agentrun", + : provenance.renderer === "pikaoa-test-runtime" + ? "pikaoa" + : "agentrun", ), }, spec: { @@ -732,7 +750,7 @@ function pipelineRunAnnotations(binding: PacSourceArtifactBinding, provenance: P }; } -function pipelineRunLabels(binding: PacSourceArtifactBinding, partOf: "agentrun" | "hwlab" | "sub2rank" | "selfmedia"): Record { +export function pipelineRunLabels(binding: PacSourceArtifactBinding, partOf: "agentrun" | "hwlab" | "sub2rank" | "selfmedia" | "pikaoa"): Record { if (partOf === "agentrun") { return { "app.kubernetes.io/part-of": "agentrun", @@ -751,6 +769,15 @@ function pipelineRunLabels(binding: PacSourceArtifactBinding, partOf: "agentrun" "selfmedia.pikapython.com/trigger": "pipelines-as-code", }; } + if (partOf === "pikaoa") { + return { + "app.kubernetes.io/name": `${binding.consumer.id}-pac`, + "app.kubernetes.io/part-of": "pikaoa", + "unidesk.ai/source-commit": "{{ revision }}", + "pikaoa.unidesk.io/source-commit": "{{ revision }}", + "pikaoa.unidesk.io/trigger": "pipelines-as-code", + }; + } if (partOf === "sub2rank") { return { "app.kubernetes.io/name": "sub2rank", @@ -825,7 +852,7 @@ function pipelineRunParams(binding: PacSourceArtifactBinding, desiredSpec: Recor if (name === "source-stage-ref") return { name, value: "{{ source_snapshot_prefix }}/{{ revision }}" }; if (name === "git-url") return { name, value: "{{ repo_url }}" }; const repositoryParam = name.replaceAll("-", "_"); - if (binding.consumer.sourceArtifact.renderer === "selfmedia-runtime" && Object.prototype.hasOwnProperty.call(param, "default")) { + if ((binding.consumer.sourceArtifact.renderer === "selfmedia-runtime" || binding.consumer.sourceArtifact.renderer === "pikaoa-test-runtime") && Object.prototype.hasOwnProperty.call(param, "default")) { return { name, value: param.default }; } if (Object.prototype.hasOwnProperty.call(binding.repository.params, repositoryParam)) { @@ -836,11 +863,11 @@ function pipelineRunParams(binding: PacSourceArtifactBinding, desiredSpec: Recor }); } -function pipelineRunWorkspaces(binding: PacSourceArtifactBinding, desiredSpec: Record): Record[] { +export function pipelineRunWorkspaces(binding: PacSourceArtifactBinding, desiredSpec: Record): Record[] { if (desiredSpec.workspaces === undefined) return []; return arrayRecords(desiredSpec.workspaces, "Pipeline.spec.workspaces").map((workspace) => { const name = requiredString(workspace.name, "Pipeline.spec.workspaces[].name"); - if (name === "source") { + if (name === "source" || (name === "workspace" && binding.consumer.sourceArtifact.renderer === "pikaoa-test-runtime")) { return { name, volumeClaimTemplate: { @@ -875,9 +902,9 @@ function inspectSourceArtifact(sourceWorktree: string, binding: PacSourceArtifac const pipelineDrift = firstPacSourceArtifactDrift(desired.pipeline, actualPipeline); const pipelineRunDrift = firstPacSourceArtifactDrift(desired.pipelineRun, actualPipelineRun); const actualProvenance = sourceArtifact.mode === "remote-pipeline-annotation" - ? provenanceFromManifest(actualPipeline) - : provenanceFromManifest(actualPipelineRun); - const wrapperProvenance = provenanceFromManifest(actualPipelineRun); + ? pacSourceArtifactProvenanceFromManifest(actualPipeline) + : pacSourceArtifactProvenanceFromManifest(actualPipelineRun); + const wrapperProvenance = pacSourceArtifactProvenanceFromManifest(actualPipelineRun); const provenanceAligned = provenanceEquals(actualProvenance, desired.provenance) && provenanceEquals(wrapperProvenance, desired.provenance); const aligned = pipelineDrift === null && pipelineRunDrift === null && provenanceAligned; return { @@ -918,6 +945,7 @@ function owningSourceRemote(binding: PacSourceArtifactBinding): string { } if (binding.consumer.sourceArtifact.renderer === "sub2rank-platform-service") return sub2RankOwningSourceRemote(); if (binding.consumer.sourceArtifact.renderer === "selfmedia-runtime") return selfMediaSourceWorktreeRemote(binding.consumer.node); + if (binding.consumer.sourceArtifact.renderer === "pikaoa-test-runtime") return pikaoaTestSourceWorktreeRemote(binding.consumer.node); if (!isHwlabRuntimeLane(binding.consumer.lane)) throw new PacSourceArtifactError("owning-source-lane-unresolved"); return hwlabRuntimeLaneSpecForNode(binding.consumer.lane, binding.consumer.node).gitUrl; } @@ -1030,13 +1058,14 @@ function assertPipelineIdentity(pipeline: Record, binding: PacS if (metadata.namespace !== binding.consumer.namespace) throw new Error(`rendered Pipeline namespace ${String(metadata.namespace)} does not match consumer ${binding.consumer.namespace}`); } -function provenanceFromManifest(manifest: Record): PacSourceArtifactProvenance | null { +export function pacSourceArtifactProvenanceFromManifest(manifest: Record): PacSourceArtifactProvenance | null { const provenance = pipelineProvenanceFromManifest(manifest); if (provenance === null) return null; if (provenance.renderer !== "agentrun-control-plane" && provenance.renderer !== "hwlab-runtime-lane" && provenance.renderer !== "sub2rank-platform-service" - && provenance.renderer !== "selfmedia-runtime") return null; + && provenance.renderer !== "selfmedia-runtime" + && provenance.renderer !== "pikaoa-test-runtime") return null; if (provenance.mode !== "embedded-pipeline-spec" && provenance.mode !== "remote-pipeline-annotation") return null; return provenance as PacSourceArtifactProvenance; } diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index d3a1f8cc..b4738c89 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -801,7 +801,7 @@ function parseConsumerDeliveryProvenance(value: Record, path: s function parseSourceArtifact(value: Record, path: string): PacSourceArtifactSpec { const mode = y.enumField(value, "mode", path, ["embedded-pipeline-spec", "remote-pipeline-annotation"] as const); - const renderer = y.enumField(value, "renderer", path, ["agentrun-control-plane", "hwlab-runtime-lane", "sub2rank-platform-service", "selfmedia-runtime"] as const); + const renderer = y.enumField(value, "renderer", path, ["agentrun-control-plane", "hwlab-runtime-lane", "sub2rank-platform-service", "selfmedia-runtime", "pikaoa-test-runtime"] as const); const pipelineRunPath = sourceArtifactPath(y.stringField(value, "pipelineRunPath", path), `${path}.pipelineRunPath`); const pipelinePath = value.pipelinePath === undefined ? null : sourceArtifactPath(y.stringField(value, "pipelinePath", path), `${path}.pipelinePath`); const taskRunTemplate = y.objectField(value, "taskRunTemplate", path); @@ -811,6 +811,7 @@ function parseSourceArtifact(value: Record, path: string): PacS if (renderer === "hwlab-runtime-lane" && mode !== "remote-pipeline-annotation") throw new Error(`${path}.renderer hwlab-runtime-lane requires remote-pipeline-annotation`); if (renderer === "sub2rank-platform-service" && mode !== "embedded-pipeline-spec") throw new Error(`${path}.renderer sub2rank-platform-service requires embedded-pipeline-spec`); if (renderer === "selfmedia-runtime" && mode !== "embedded-pipeline-spec") throw new Error(`${path}.renderer selfmedia-runtime requires embedded-pipeline-spec`); + if (renderer === "pikaoa-test-runtime" && mode !== "embedded-pipeline-spec") throw new Error(`${path}.renderer pikaoa-test-runtime requires embedded-pipeline-spec`); return { mode, renderer, @@ -1103,10 +1104,10 @@ function validateConsumerConfig( if (consumer.sourceArtifact?.renderer === "sub2rank-platform-service" && consumer.runnerServiceAccount === null) { throw new Error(`${configLabel}.consumers.${consumer.id}.runnerServiceAccount is required for sub2rank-platform-service`); } - if (consumer.sourceArtifact?.renderer === "selfmedia-runtime") { - if (consumer.runnerServiceAccount === null) throw new Error(`${configLabel}.consumers.${consumer.id}.runnerServiceAccount is required for selfmedia-runtime`); + if (consumer.sourceArtifact?.renderer === "selfmedia-runtime" || consumer.sourceArtifact?.renderer === "pikaoa-test-runtime") { + if (consumer.runnerServiceAccount === null) throw new Error(`${configLabel}.consumers.${consumer.id}.runnerServiceAccount is required for private GitOps runtime renderers`); if (consumer.argoBootstrap?.repositoryCredential === null || consumer.argoBootstrap?.repositoryCredential === undefined) { - throw new Error(`${configLabel}.consumers.${consumer.id}.argoBootstrap.repositoryCredential is required for the private selfmedia repository`); + throw new Error(`${configLabel}.consumers.${consumer.id}.argoBootstrap.repositoryCredential is required for the private repository`); } if (consumer.closeoutGitOpsMirrorFlush) throw new Error(`${configLabel}.consumers.${consumer.id}.closeoutGitOpsMirrorFlush must be false for Gitea writeback GitOps`); if (params.gitops_secret_name !== repository.secretName) throw new Error(`${configLabel}.consumers.${consumer.id}.effective params.gitops_secret_name must match repository secretName`);