diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 4c359290..46b4d847 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -56,26 +56,34 @@ targets: repository: pikasTech/HWLAB branch: v0.3 gitops: - branch: v0.3-d601-gitops + branch: v0.3-gitops path: deploy/gitops/node/d601/runtime-v03 gitMirror: namespace: devops-infra serviceReadName: git-mirror-http serviceWriteName: git-mirror-write cachePvcName: hwlab-git-mirror-cache + cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d601-v03-git-mirror-cache cachePvcStorage: 20Gi servicePort: 8080 - deploymentReplicas: 0 + deploymentReplicas: 1 secretName: git-mirror-github-ssh syncConfigMapName: git-mirror-sync-script syncJobPrefix: git-mirror-hwlab-d601-v03-sync-manual flushJobPrefix: git-mirror-hwlab-d601-v03-flush-manual - readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git - writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + storage: + localPath: + namespace: kube-system + configMapName: local-path-config + provisionerDeployment: local-path-provisioner + helperImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: IfNotPresent tekton: - pipelineName: hwlab-d601-v03-ci-image-publish - serviceAccountName: hwlab-d601-v03-tekton-runner - pipelineRunPrefix: hwlab-d601-v03-ci-poll + pipelineName: hwlab-v03-ci-image-publish + serviceAccountName: hwlab-v03-tekton-runner + pipelineRunPrefix: hwlab-v03-ci-poll toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 sourceKind: dockerfile @@ -124,9 +132,22 @@ targets: buildMode: node-local argo: namespace: argocd - projectName: hwlab-d601 - applicationName: hwlab-d601-v03 - applicationFile: application-d601-v03.yaml + projectName: hwlab-v03 + applicationName: hwlab-node-v03 + applicationFile: application-v03.yaml + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + obsoleteApplications: + - hwlab-d601-v03 + resourceTracking: + manageEndpointBridge: true + repositoryCredential: + enabled: true + repoURL: git@github.com:pikasTech/HWLAB.git + secretName: hwlab-node-v03-repository + sourceSecret: + namespace: hwlab-ci + name: hwlab-git-ssh + key: ssh-privatekey install: enabled: true sourceKind: url diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 2b0602a7..7ddb86e9 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -10,6 +10,13 @@ nodes: gitopsRoot: deploy/gitops/node networkProfile: node-ci-egress downloadProfile: node-default + D601: + route: D601 + kubeRoute: D601:k3s + sourceWorkspace: /home/ubuntu/workspace/hwlab-v03 + gitopsRoot: deploy/gitops/node + networkProfile: d601-node-ci-egress + downloadProfile: d601-node-default lanes: v02: @@ -45,6 +52,8 @@ lanes: - hwlab-gateway - hwlab-edge-proxy - hwlab-agent-skills + observability: + prometheusOperator: true public: webUrl: http://74.48.78.17:19666 apiUrl: http://74.48.78.17:19667 @@ -81,9 +90,69 @@ lanes: - hwlab-gateway - hwlab-edge-proxy - hwlab-agent-skills + observability: + prometheusOperator: true public: webUrl: http://74.48.78.17:20666 apiUrl: http://74.48.78.17:20667 + targets: + D601: + workspace: /home/ubuntu/workspace/hwlab-v03 + cicdRepo: /home/ubuntu/workspace/hwlab-v03-cicd.git + cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock + app: hwlab-node-v03 + pipeline: hwlab-v03-ci-image-publish + pipelineRunPrefix: hwlab-v03-ci-poll + serviceAccountName: hwlab-v03-tekton-runner + controlPlaneFieldManager: unidesk-hwlab-d601-v03-control-plane + git: + url: git@github.com:pikasTech/HWLAB.git + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + argo: + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + gitopsBranch: v0.3-gitops + catalogPath: deploy/artifact-catalog.d601-v03.json + runtime: + path: deploy/gitops/node/d601/runtime-v03 + namespace: hwlab-v03 + renderDir: runtime-v03 + tektonDir: tekton-v03 + argoApplicationFile: application-v03.yaml + registryPrefix: 127.0.0.1:5000/hwlab + baseImage: 127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim + baseImageSource: node:20-bookworm-slim + buildkit: + sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless + stepEnv: + HOME: /tekton/home + XDG_CONFIG_HOME: /tekton/home/.config + observability: + prometheusOperator: false + public: + webUrl: https://v03.hwpod.com + apiUrl: https://v03.hwpod.com + externalPostgres: + provider: PK01 + configRef: config/platform-db/postgres-pk01.yaml + serviceName: pk01-platform-postgres + endpointAddress: 82.156.23.220 + port: 5432 + sslmode: require + database: hwlab_d601_v03 + cloudApi: + secretName: hwlab-cloud-api-v03-db + secretKey: database-url + sourceRef: hwlab/d601-v03-cloud-api-db.env + envKey: DATABASE_URL + role: hwlab_d601_v03_app + openfga: + secretName: hwlab-v03-openfga + secretKey: datastore-uri + sourceRef: hwlab/d601-v03-openfga-db.env + envKey: DATASTORE_URI + authnKey: authn-preshared-key + role: hwlab_d601_v03_app networkProfiles: node-ci-egress: @@ -153,12 +222,70 @@ networkProfiles: - ::1 - host.docker.internal - 127.0.0.1:5000 + d601-node-ci-egress: + proxy: + http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + all: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 10.0.0.0/8 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + dockerBuildProxy: + http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + all: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local downloadProfiles: node-default: git: proxyMode: inherit retries: 3 + timeoutSeconds: 240 + npm: + registry: https://registry.npmjs.org/ + retries: 3 + fetchTimeoutSeconds: 120 + pip: + indexUrl: https://pypi.org/simple + retries: 3 + timeoutSeconds: 120 + docker: + registryMirrors: [] + pullRetries: 3 + curl: + retries: 3 + connectTimeoutSeconds: 10 + maxTimeSeconds: 120 + d601-node-default: + git: + proxyMode: inherit + retries: 3 + timeoutSeconds: 60 npm: registry: https://registry.npmjs.org/ retries: 3 diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index b6711e4b..bd2f071b 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -8,6 +8,7 @@ metadata: relatedIssues: - 280 - 281 + - 1119 cluster: role: primary @@ -84,7 +85,7 @@ postgres: purpose: admin-and-secret-sync - id: D601-public cidr: 36.49.29.73/32 - purpose: platform-infra-standby-app + purpose: platform-infra-and-hwlab-v03-app tuning: maxConnections: 50 sharedBuffers: 512MB @@ -135,6 +136,11 @@ postgres: user: sub2api address: 36.49.29.73/32 method: scram-sha-256 + - type: hostssl + database: hwlab_d601_v03 + user: hwlab_d601_v03_app + address: 36.49.29.73/32 + method: scram-sha-256 secrets: source: master-local @@ -154,6 +160,20 @@ secrets: SUB2API_DB_NAME: sub2api randomHex: SUB2API_DB_PASSWORD: 32 + - name: hwlab-d601-v03-db-credentials + sourceRef: platform-db/hwlab-d601-v03-db.env + type: env + requiredKeys: + - HWLAB_D601_V03_DB_USER + - HWLAB_D601_V03_DB_PASSWORD + - HWLAB_D601_V03_DB_NAME + createIfMissing: + enabled: true + values: + HWLAB_D601_V03_DB_USER: hwlab_d601_v03_app + HWLAB_D601_V03_DB_NAME: hwlab_d601_v03 + randomHex: + HWLAB_D601_V03_DB_PASSWORD: 32 objects: roles: @@ -166,12 +186,26 @@ objects: createdb: false createrole: false superuser: false + - name: hwlab_d601_v03_app + passwordRef: + sourceRef: platform-db/hwlab-d601-v03-db.env + key: HWLAB_D601_V03_DB_PASSWORD + login: true + attributes: + createdb: false + createrole: false + superuser: false databases: - name: sub2api owner: sub2api encoding: UTF8 locale: C.UTF-8 extensions: [] + - name: hwlab_d601_v03 + owner: hwlab_d601_v03_app + encoding: UTF8 + locale: C.UTF-8 + extensions: [] exports: connectionStrings: @@ -190,6 +224,36 @@ exports: - scope: platform-infra secret: sub2api-secrets key: DATABASE_URL + - name: hwlab-d601-v03-cloud-api-database-url + sourceSecretRef: platform-db/hwlab-d601-v03-db.env + render: + envKey: DATABASE_URL + format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: hwlab/d601-v03-cloud-api-db.env + key: DATABASE_URL + mode: update-or-insert + consumers: + - scope: hwlab-v03 + secret: hwlab-cloud-api-v03-db + key: database-url + - name: hwlab-d601-v03-openfga-datastore-uri + sourceSecretRef: platform-db/hwlab-d601-v03-db.env + render: + envKey: DATASTORE_URI + format: postgresql://$(HWLAB_D601_V03_DB_USER):$(HWLAB_D601_V03_DB_PASSWORD)@$(PGHOST):5432/$(HWLAB_D601_V03_DB_NAME)?sslmode=require + variables: + PGHOST: 82.156.23.220 + writeToSecretSource: + sourceRef: hwlab/d601-v03-openfga-db.env + key: DATASTORE_URI + mode: update-or-insert + consumers: + - scope: hwlab-v03 + secret: hwlab-v03-openfga + key: datastore-uri backup: phase: minimum-restoreable diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 72b8571d..0baba8ad 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -65,6 +65,29 @@ interface ImageRewriteSpec { target: string; } +interface LocalPathStorageSpec { + namespace: string; + configMapName: string; + provisionerDeployment: string; + helperImage: string; + imagePullPolicy: "Always" | "IfNotPresent" | "Never"; +} + +interface ArgoRepositoryCredentialSpec { + enabled: boolean; + repoURL: string; + secretName: string; + sourceSecret: { + namespace: string; + name: string; + key: string; + }; +} + +interface ArgoResourceTrackingSpec { + manageEndpointBridge: boolean; +} + interface ControlPlaneTargetSpec { id: string; node: string; @@ -79,6 +102,7 @@ interface ControlPlaneTargetSpec { serviceReadName: string; serviceWriteName: string; cachePvcName: string; + cacheHostPath: string | null; cachePvcStorage: string; servicePort: number; deploymentReplicas: number; @@ -89,6 +113,9 @@ interface ControlPlaneTargetSpec { readUrl: string; writeUrl: string; }; + storage: { + localPath: LocalPathStorageSpec; + } | null; tekton: { pipelineName: string; serviceAccountName: string; @@ -112,6 +139,10 @@ interface ControlPlaneTargetSpec { projectName: string; applicationName: string; applicationFile: string; + repoURL: string; + obsoleteApplications: readonly string[]; + resourceTracking: ArgoResourceTrackingSpec; + repositoryCredential: ArgoRepositoryCredentialSpec | null; install: { enabled: boolean; sourceKind: "url"; @@ -239,9 +270,17 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta const tekton = record(components.tekton); const ciNamespace = record(components.ciNamespace); const registry = record(components.registry); + const storage = record(components.storage); + const localPath = record(storage.localPath); + const storageReady = target.storage === null || boolField(localPath, "ready"); + const argoRepositoryCredential = record(argo.repositoryCredential); + const argoResourceTracking = record(argo.resourceTracking); + const argoRepositoryCredentialReady = !boolField(argoRepositoryCredential, "required") || boolField(argoRepositoryCredential, "ready"); + const argoResourceTrackingReady = boolField(argoResourceTracking, "ready"); const ok = result.exitCode === 0 && boolField(tekton, "installed") && boolField(ciNamespace, "exists") + && storageReady && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") @@ -253,7 +292,9 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") - && boolField(argoInstall, "statefulSetsReady"); + && boolField(argoInstall, "statefulSetsReady") + && argoResourceTrackingReady + && argoRepositoryCredentialReady; return { ok, command: "hwlab nodes control-plane infra status", @@ -277,6 +318,9 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta argoInstalled: boolField(argo, "installed"), argoProjectExists: boolField(argo, "projectExists"), argoApplicationExists: boolField(argo, "applicationExists"), + argoResourceTrackingReady, + argoRepositoryCredentialReady, + storageLocalPathReady: storageReady, argoCrdsReady: boolField(argoInstall, "crdsReady"), argoDeploymentsReady: boolField(argoInstall, "deploymentsReady"), argoStatefulSetsReady: boolField(argoInstall, "statefulSetsReady"), @@ -284,7 +328,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta toolsImageReady: boolField(registry, "toolsImageReady"), }, result: compactCommandResult(result), - next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace), + next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace, storageReady), }; } @@ -316,7 +360,7 @@ function infraApply(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, tar next: applyNext(node, target, imageStatus), }; } - const script = applyScript(yaml); + const script = applyScript(yaml, target); const result = runTransK3s(node.kubeRoute, script, options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { @@ -433,6 +477,7 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; const argo = record(record(status.components).argo); const argoInstall = record(argo.install); + const argoResourceTracking = record(argo.resourceTracking); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "argo", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = boolField(argo, "installed") @@ -440,7 +485,8 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") - && boolField(argoInstall, "statefulSetsReady"); + && boolField(argoInstall, "statefulSetsReady") + && boolField(argoResourceTracking, "ready"); return { ok, command: "hwlab nodes control-plane infra argo status", @@ -461,6 +507,7 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge crdsReady: boolField(argoInstall, "crdsReady"), deploymentsReady: boolField(argoInstall, "deploymentsReady"), statefulSetsReady: boolField(argoInstall, "statefulSetsReady"), + resourceTrackingReady: boolField(argoResourceTracking, "ready"), }, argo, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, @@ -485,6 +532,7 @@ function argoApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, o imageRewrites: target.argo.install.imageRewrites, preloadImages: target.argo.install.preloadImages, requiredCrds: target.argo.install.requiredCrds, + obsoleteApplications: target.argo.obsoleteApplications, desired: manifestObjectSummary(desired), desiredSha256: sha256Short(desiredYaml), stateDir: remoteJobStateDir(target, "argo"), @@ -687,6 +735,28 @@ function argoInstallSpec(raw: Record, path: string): ControlPla }; } +function argoRepositoryCredentialSpec(raw: Record | undefined, path: string): ArgoRepositoryCredentialSpec | null { + if (raw === undefined) return null; + const sourceSecret = asRecord(raw.sourceSecret, `${path}.sourceSecret`); + return { + enabled: booleanField(raw, "enabled", path), + repoURL: stringField(raw, "repoURL", path), + secretName: stringField(raw, "secretName", path), + sourceSecret: { + namespace: stringField(sourceSecret, "namespace", `${path}.sourceSecret`), + name: stringField(sourceSecret, "name", `${path}.sourceSecret`), + key: stringField(sourceSecret, "key", `${path}.sourceSecret`), + }, + }; +} + +function argoResourceTrackingSpec(raw: Record | undefined, path: string): ArgoResourceTrackingSpec { + if (raw === undefined) return { manageEndpointBridge: false }; + return { + manageEndpointBridge: booleanField(raw, "manageEndpointBridge", path), + }; +} + function imageRewriteSpec(raw: Record, path: string): ImageRewriteSpec { const rewrite = { source: stringField(raw, "source", path), @@ -757,6 +827,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const source = asRecord(raw.source, `${path}.source`); const gitops = asRecord(raw.gitops, `${path}.gitops`); const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`); + const storage = raw.storage === undefined ? null : storageSpec(asRecord(raw.storage, `${path}.storage`), `${path}.storage`); const tekton = asRecord(raw.tekton, `${path}.tekton`); const argo = asRecord(raw.argo, `${path}.argo`); const toolsImage = asRecord(tekton.toolsImage, `${path}.tekton.toolsImage`); @@ -780,6 +851,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa serviceReadName, serviceWriteName, cachePvcName: stringField(gitMirror, "cachePvcName", `${path}.gitMirror`), + cacheHostPath: optionalStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`) ?? null, cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), deploymentReplicas: nonNegativeIntegerField(gitMirror, "deploymentReplicas", `${path}.gitMirror`), @@ -790,6 +862,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, }, + storage, tekton: { pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), @@ -801,11 +874,30 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa projectName: stringField(argo, "projectName", `${path}.argo`), applicationName: stringField(argo, "applicationName", `${path}.argo`), applicationFile: stringField(argo, "applicationFile", `${path}.argo`), + repoURL: optionalStringField(argo, "repoURL", `${path}.argo`) ?? (optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`), + obsoleteApplications: optionalStringArrayField(argo, "obsoleteApplications", `${path}.argo`), + resourceTracking: argoResourceTrackingSpec(argo.resourceTracking === undefined ? undefined : asRecord(argo.resourceTracking, `${path}.argo.resourceTracking`), `${path}.argo.resourceTracking`), + repositoryCredential: argoRepositoryCredentialSpec(argo.repositoryCredential === undefined ? undefined : asRecord(argo.repositoryCredential, `${path}.argo.repositoryCredential`), `${path}.argo.repositoryCredential`), install: argoInstallSpec(asRecord(argo.install, `${path}.argo.install`), `${path}.argo.install`), }, }; } +function storageSpec(raw: Record, path: string): ControlPlaneTargetSpec["storage"] { + const localPath = asRecord(raw.localPath, `${path}.localPath`); + const imagePullPolicy = optionalStringField(localPath, "imagePullPolicy", `${path}.localPath`) ?? "IfNotPresent"; + if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") throw new Error(`${path}.localPath.imagePullPolicy must be Always, IfNotPresent, or Never`); + return { + localPath: { + namespace: stringField(localPath, "namespace", `${path}.localPath`), + configMapName: stringField(localPath, "configMapName", `${path}.localPath`), + provisionerDeployment: stringField(localPath, "provisionerDeployment", `${path}.localPath`), + helperImage: stringField(localPath, "helperImage", `${path}.localPath`), + imagePullPolicy, + }, + }; +} + function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record[] { const labels = { "app.kubernetes.io/part-of": "hwlab-node-control-plane", @@ -832,27 +924,15 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, data: { "repositories.json": JSON.stringify([{ repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), - "sync.sh": "#!/bin/sh\nset -eu\necho d601-hwlab-git-mirror-sync-placeholder\ncat /etc/git-mirror/repositories.json\n", - "flush.sh": "#!/bin/sh\nset -eu\necho d601-hwlab-git-mirror-flush-placeholder\ncat /etc/git-mirror/repositories.json\n", + "server.js": gitMirrorServerJs(), + "sync.sh": gitMirrorSyncScript(target), + "flush.sh": gitMirrorFlushScript(target), }, }, service(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), service(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), gitMirrorDeployment(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target, "read"), gitMirrorDeployment(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target, "write"), - { - apiVersion: "tekton.dev/v1", - kind: "Pipeline", - metadata: { name: target.tekton.pipelineName, namespace: target.ciNamespace, labels }, - spec: { - params: [ - { name: "source-commit", type: "string" }, - { name: "source-branch", type: "string", default: target.source.branch }, - { name: "gitops-branch", type: "string", default: target.gitops.branch }, - ], - tasks: [{ name: "bootstrap-placeholder", taskSpec: { steps: [{ name: "notice", image: target.tekton.toolsImage.output, script: "#!/bin/sh\nset -eu\necho d601-hwlab-v03-pipeline-placeholder\n" }] } }], - }, - }, { apiVersion: "v1", kind: "Namespace", metadata: { name: target.argo.namespace, labels } }, { apiVersion: "v1", @@ -861,13 +941,97 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa data: { "project.yaml": Bun.YAML.stringify(argoProjectSkeleton(target)), [target.argo.applicationFile]: Bun.YAML.stringify(argoApplicationSkeleton(target)), + "argocd-cm.yaml": Bun.YAML.stringify(argoConfigMap(target, labels)), + "obsolete-applications.json": JSON.stringify(target.argo.obsoleteApplications, null, 2), "note.txt": "Argo CD CRDs/controller are installed by the node control-plane bootstrap path when available; this ConfigMap preserves the desired Application until Argo is ready.\n", }, }, ); + const obsoleteApplications = argoObsoleteApplicationsConfigMap(target, labels); + if (obsoleteApplications !== null) manifests.push(obsoleteApplications); return manifests; } +function argoConfigMap(target: ControlPlaneTargetSpec, labels: Record): Record { + return { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: "argocd-cm", + namespace: target.argo.namespace, + labels: { ...labels, "app.kubernetes.io/name": "argocd-cm", "app.kubernetes.io/part-of": "argocd" }, + }, + data: { + "resource.exclusions": argoResourceExclusions(target), + }, + }; +} + +function argoResourceExclusions(target: ControlPlaneTargetSpec): string { + const endpointBlock = target.argo.resourceTracking.manageEndpointBridge + ? [] + : [ + "### Network resources created by the Kubernetes control plane and excluded to reduce the number of watched events and UI clutter", + "- apiGroups:", + " - ''", + " - discovery.k8s.io", + " kinds:", + " - Endpoints", + " - EndpointSlice", + ]; + return [ + ...endpointBlock, + "### Internal Kubernetes resources excluded to reduce watch volume", + "- apiGroups:", + " - coordination.k8s.io", + " kinds:", + " - Lease", + "### Internal Kubernetes Authz/Authn resources excluded to reduce watched events", + "- apiGroups:", + " - authentication.k8s.io", + " - authorization.k8s.io", + " kinds:", + " - SelfSubjectReview", + " - TokenReview", + " - LocalSubjectAccessReview", + " - SelfSubjectAccessReview", + " - SelfSubjectRulesReview", + " - SubjectAccessReview", + "### Intermediate Certificate Request excluded to reduce watched events", + "- apiGroups:", + " - certificates.k8s.io", + " kinds:", + " - CertificateSigningRequest", + "- apiGroups:", + " - cert-manager.io", + " kinds:", + " - CertificateRequest", + "### Cilium internal resources excluded to reduce UI clutter", + "- apiGroups:", + " - cilium.io", + " kinds:", + " - CiliumIdentity", + " - CiliumEndpoint", + " - CiliumEndpointSlice", + "### Kyverno intermediate and reporting resources excluded to reduce watched events", + "- apiGroups:", + " - kyverno.io", + " - reports.kyverno.io", + " - wgpolicyk8s.io", + " kinds:", + " - PolicyReport", + " - ClusterPolicyReport", + " - EphemeralReport", + " - ClusterEphemeralReport", + " - AdmissionReport", + " - ClusterAdmissionReport", + " - BackgroundScanReport", + " - ClusterBackgroundScanReport", + " - UpdateRequest", + "", + ].join("\n"); +} + function service(name: string, namespace: string, labels: Record, port: number): Record { return { apiVersion: "v1", @@ -878,6 +1042,9 @@ function service(name: string, namespace: string, labels: Record } function gitMirrorDeployment(name: string, namespace: string, labels: Record, target: ControlPlaneTargetSpec, mode: "read" | "write"): Record { + const serverJs = gitMirrorServerJs(); + const syncScript = gitMirrorSyncScript(target); + const flushScript = gitMirrorFlushScript(target); return { apiVersion: "apps/v1", kind: "Deployment", @@ -886,10 +1053,35 @@ function gitMirrorDeployment(name: string, namespace: string, labels: Record[] { - return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; + const labels = { + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "hwlab.pikastech.local/node": target.node, + "hwlab.pikastech.local/lane": target.lane, + }; + const manifest = [argoConfigMap(target, labels), argoProjectSkeleton(target), argoApplicationSkeleton(target)]; + const obsoleteApplications = argoObsoleteApplicationsConfigMap(target, labels); + if (obsoleteApplications !== null) manifest.push(obsoleteApplications); + return manifest; +} + +function argoObsoleteApplicationsConfigMap(target: ControlPlaneTargetSpec, labels: Record): Record | null { + if (target.argo.obsoleteApplications.length === 0) return null; + return { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: `${target.argo.applicationName}-obsolete-applications`, namespace: target.argo.namespace, labels }, + data: { + "applications.json": JSON.stringify(target.argo.obsoleteApplications, null, 2), + "note.txt": "Applications listed here are deleted by UniDesk node-local Argo apply so one YAML-defined application owns the runtime resources.\n", + }, + }; } function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record { @@ -906,7 +1119,7 @@ function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record= 0 ? crlf + 4 : (lf >= 0 ? lf + 2 : -1);", + " if (headerEnd < 0) return null;", + " const headerText = buffer.slice(0, headerEnd).toString('latin1').trim();", + " const rest = buffer.slice(headerEnd);", + " const headers = {};", + " let status = 200;", + " for (const line of headerText.split(/\\r?\\n/u)) {", + " const index = line.indexOf(':');", + " if (index < 0) continue;", + " const key = line.slice(0, index).trim();", + " const value = line.slice(index + 1).trim();", + " if (key.toLowerCase() === 'status') { const parsed = Number.parseInt(value.split(' ')[0] || '', 10); if (Number.isInteger(parsed)) status = parsed; }", + " else headers[key] = value;", + " }", + " return { status, headers, rest };", + "}", + "function handleGit(req, res) {", + " const url = new URL(req.url || '/', 'http://git-mirror.local');", + " const chunks = [];", + " req.on('data', (chunk) => chunks.push(chunk));", + " req.on('error', (error) => { res.writeHead(500, { 'content-type': 'text/plain' }); res.end(String(error && error.message || error)); });", + " req.on('end', () => {", + " let body = Buffer.concat(chunks);", + " const encoding = String(req.headers['content-encoding'] || '').toLowerCase();", + " try {", + " if (encoding === 'gzip' || encoding === 'x-gzip') body = zlib.gunzipSync(body);", + " else if (encoding === 'deflate') body = zlib.inflateSync(body);", + " else if (encoding === 'br') body = zlib.brotliDecompressSync(body);", + " else if (encoding && encoding !== 'identity') throw new Error(`unsupported content-encoding: ${encoding}`);", + " } catch (error) {", + " res.writeHead(415, { 'content-type': 'text/plain' });", + " res.end(String(error && error.message || error));", + " return;", + " }", + " const env = { ...process.env, GIT_PROJECT_ROOT: projectRoot, GIT_HTTP_EXPORT_ALL: '1', PATH_INFO: decodeURIComponent(url.pathname), REQUEST_METHOD: req.method || 'GET', QUERY_STRING: url.search.slice(1), CONTENT_TYPE: req.headers['content-type'] || '', CONTENT_LENGTH: String(body.length), REMOTE_USER: 'git' };", + " delete env.HTTP_CONTENT_ENCODING;", + " const child = spawn('git', ['http-backend'], { env });", + " let pending = Buffer.alloc(0);", + " let headersSent = false;", + " child.stderr.on('data', (chunk) => process.stderr.write(chunk));", + " child.on('error', (error) => { if (!headersSent) { res.writeHead(500, { 'content-type': 'text/plain' }); headersSent = true; } res.end(String(error && error.message || error)); });", + " child.stdout.on('data', (chunk) => {", + " if (headersSent) { res.write(chunk); return; }", + " pending = Buffer.concat([pending, chunk]);", + " const parsed = parseHeaders(pending);", + " if (!parsed) return;", + " headersSent = true;", + " res.writeHead(parsed.status, parsed.headers);", + " if (parsed.rest.length) res.write(parsed.rest);", + " });", + " child.on('close', (code) => {", + " if (!headersSent) { res.writeHead(code === 0 ? 200 : 500, { 'content-type': 'text/plain' }); headersSent = true; if (pending.length) res.write(pending); }", + " res.end();", + " });", + " child.stdin.end(body);", + " });", + "}", + "http.createServer((req, res) => { if ((req.url || '').startsWith('/healthz')) return sendHealth(res); return handleGit(req, res); }).listen(port, '0.0.0.0', () => { console.log(JSON.stringify({ event: 'git-mirror-http-started', port, projectRoot, mode: process.env.GIT_MIRROR_MODE || null })); });", + "", + ].join("\n"); +} + +function gitMirrorSshSetupScriptLines(): string[] { + return [ + "key_file=${GIT_SSH_KEY_FILE:-/etc/git-mirror/ssh-privatekey}", + "if [ -s \"$key_file\" ]; then", + " chmod 600 \"$key_file\"", + " export GIT_SSH_COMMAND=\"ssh -i $key_file -o StrictHostKeyChecking=accept-new -p 443\"", + "else", + " export GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=accept-new -p 443\"", + "fi", + ]; +} + +function gitMirrorSyncScript(target: ControlPlaneTargetSpec): string { + return [ + "#!/bin/sh", + "set -eu", + "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "mkdir -p /cache/pikasTech", + ...gitMirrorSshSetupScriptLines(), + `repository=${shQuote(target.source.repository)}`, + `source_branch=${shQuote(target.source.branch)}`, + `gitops_branch=${shQuote(target.gitops.branch)}`, + "repo=\"/cache/${repository}.git\"", + "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", + "mkdir -p \"$(dirname \"$repo\")\"", + "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", + " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", + "else", + " rm -rf \"$repo\"", + " git init --bare \"$repo\"", + " git --git-dir=\"$repo\" remote add origin \"$remote\"", + "fi", + "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", + "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", + "git --git-dir=\"$repo\" config http.receivepack true", + "timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", + "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", + "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", + "if timeout 180 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", + " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + " if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then", + " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", + " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", + " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", + " fi", + "fi", + "git --git-dir=\"$repo\" update-server-info", + "local_source=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${source_branch}^{commit}\" 2>/dev/null || true)", + "github_source=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\" 2>/dev/null || true)", + "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", + "json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }", + "cat > /cache/HWLAB.last-sync.json </dev/null || true)", + "if [ -n \"$local_gitops\" ]; then", + " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", + " git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", + "fi", + "git --git-dir=\"$repo\" update-server-info", + "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", + "json_ref() { if [ -n \"$1\" ]; then printf '\"%s\"' \"$1\"; else printf null; fi; }", + "cat > /cache/HWLAB.last-flush.json </dev/null || printf '{}') +if [ "$local_path_enabled" = true ]; then + kubectl -n "$local_path_ns" get cm "$local_path_configmap" -o 'go-template={{ index .data "helperPod.yaml" }}' >/tmp/hwlab-local-path-helper.yaml 2>/tmp/hwlab-local-path-helper.err || true + python3 - "$local_path_ns" "$local_path_configmap" "$local_path_deployment" "$local_path_helper_image" "$local_path_pull_policy" /tmp/hwlab-local-path-helper.yaml <<'PY' >/tmp/hwlab-local-path-fragment.json +import json, pathlib, re, subprocess, sys +ns, configmap, deployment, expected_image, expected_policy, helper_path = sys.argv[1:7] +text = pathlib.Path(helper_path).read_text(errors="replace") if pathlib.Path(helper_path).exists() else "" +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +def exists(args): + return run(args).returncode == 0 +def ready_deploy(): + data = run(["kubectl", "-n", ns, "get", "deployment", deployment, "-o", "json"]) + if data.returncode != 0: + return False + obj=json.loads(data.stdout) + desired=int(obj.get("spec", {}).get("replicas") or 0) + ready=int(obj.get("status", {}).get("readyReplicas") or 0) + return desired > 0 and ready == desired +image_match = re.search(r"(?m)^\\s*image:\\s*['\\\"]?([^'\\\"\\s]+)", text) +policy_match = re.search(r"(?m)^\\s*imagePullPolicy:\\s*['\\\"]?([^'\\\"\\s]+)", text) +current_image = image_match.group(1) if image_match else None +current_policy = policy_match.group(1) if policy_match else None +config_exists = exists(["kubectl", "-n", ns, "get", "configmap", configmap]) +deployment_exists = exists(["kubectl", "-n", ns, "get", "deployment", deployment]) +payload = { + "enabled": True, + "namespace": ns, + "configMap": configmap, + "provisionerDeployment": deployment, + "configMapExists": config_exists, + "deploymentExists": deployment_exists, + "deploymentReady": ready_deploy(), + "expectedHelperImage": expected_image, + "helperImage": current_image, + "expectedImagePullPolicy": expected_policy, + "imagePullPolicy": current_policy, + "ready": config_exists and deployment_exists and ready_deploy() and current_image == expected_image and current_policy == expected_policy, +} +print(json.dumps(payload)) +PY +else + printf '{"enabled":false,"ready":true}\\n' >/tmp/hwlab-local-path-fragment.json +fi +local_path_fragment=$(cat /tmp/hwlab-local-path-fragment.json 2>/dev/null || printf '{"enabled":false,"ready":true}') +if [ "$argo_repo_credential_enabled" = true ]; then + python3 - "$argo_ns" "$argo_repo_credential_secret" "$argo_repo_credential_url" "$argo_repo_credential_source_ns" "$argo_repo_credential_source_name" "$argo_repo_credential_source_key" <<'PY' >/tmp/hwlab-argo-repository-credential-fragment.json +import json, subprocess, sys +argo_ns, secret, repo_url, source_ns, source_name, source_key = sys.argv[1:7] +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +repo = run(["kubectl", "-n", argo_ns, "get", "secret", secret, "-o", "json"]) +source = run(["kubectl", "-n", source_ns, "get", "secret", source_name, "-o", "json"]) +repo_payload = json.loads(repo.stdout) if repo.returncode == 0 and repo.stdout.strip() else {} +source_payload = json.loads(source.stdout) if source.returncode == 0 and source.stdout.strip() else {} +labels = repo_payload.get("metadata", {}).get("labels", {}) if repo_payload else {} +data = repo_payload.get("data", {}) if repo_payload else {} +source_data = source_payload.get("data", {}) if source_payload else {} +payload = { + "required": True, + "ready": repo.returncode == 0 and source.returncode == 0 and labels.get("argocd.argoproj.io/secret-type") == "repository" and all(key in data for key in ["type", "url", "sshPrivateKey"]) and source_key in source_data, + "secret": secret, + "repoURL": repo_url, + "sourceSecret": {"namespace": source_ns, "name": source_name, "key": source_key, "present": source.returncode == 0 and source_key in source_data}, + "valuesPrinted": False, +} +print(json.dumps(payload)) +PY +else + printf '{"required":false,"ready":true,"valuesPrinted":false}\\n' >/tmp/hwlab-argo-repository-credential-fragment.json +fi +argo_repository_credential_fragment=$(cat /tmp/hwlab-argo-repository-credential-fragment.json 2>/dev/null || printf '{"required":false,"ready":true,"valuesPrinted":false}') +python3 - "$argo_ns" "$argo_manage_endpoint_bridge" <<'PY' >/tmp/hwlab-argo-resource-tracking-fragment.json +import json, subprocess, sys +namespace, manage = sys.argv[1:3] +expect_manage = manage == "true" +result = subprocess.run(["kubectl", "-n", namespace, "get", "configmap", "argocd-cm", "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +payload = {"manageEndpointBridge": expect_manage, "configMap": "argocd-cm", "exists": result.returncode == 0} +def endpoint_resources_excluded(exclusions): + for item in exclusions.split("- apiGroups:"): + if "kinds:" not in item: + continue + kinds = [line.strip()[2:].strip() for line in item.splitlines() if line.strip().startswith("- ")] + if "Endpoints" in kinds or "EndpointSlice" in kinds: + return True + return False +if result.returncode == 0: + data = json.loads(result.stdout).get("data", {}) + exclusions = data.get("resource.exclusions", "") + payload["endpointResourcesExcluded"] = endpoint_resources_excluded(exclusions) + payload["endpointIgnoreUpdates"] = "resource.customizations.ignoreResourceUpdates.Endpoints" in data + payload["endpointSliceIgnoreUpdates"] = "resource.customizations.ignoreResourceUpdates.discovery.k8s.io_EndpointSlice" in data +else: + payload["endpointResourcesExcluded"] = None + payload["endpointIgnoreUpdates"] = None + payload["endpointSliceIgnoreUpdates"] = None +payload["ready"] = payload["exists"] and (not expect_manage or (payload["endpointResourcesExcluded"] is False and payload["endpointIgnoreUpdates"] is False and payload["endpointSliceIgnoreUpdates"] is False)) +print(json.dumps(payload)) +PY +argo_resource_tracking_fragment=$(cat /tmp/hwlab-argo-resource-tracking-fragment.json 2>/dev/null || printf '{"ready":false,"parseError":true}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"tekton":{"installed":$(kubectl get crd pipelines.tekton.dev pipelineruns.tekton.dev >/dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"storage":{"localPath":$local_path_fragment},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"resourceTracking":$argo_resource_tracking_fragment,"repositoryCredential":$argo_repository_credential_fragment,"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} JSON `; } -function applyScript(yaml: string): string { +function applyScript(yaml: string, target: ControlPlaneTargetSpec): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); + const localPath = target.storage?.localPath ?? null; + const helperPod = localPath === null ? "" : localPathHelperPodYaml(localPath); + const helperEncoded = Buffer.from(helperPod, "utf8").toString("base64"); + const argoRepositoryCredential = target.argo.repositoryCredential; return ` set +e manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml) printf %s ${shQuote(encoded)} | base64 -d >"$manifest" kubectl apply --server-side --field-manager=unidesk-hwlab-node-control-plane -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err rc=$? -python3 - "$rc" <<'PY' +storage_rc=0 +repo_credential_rc=0 +if [ "$rc" -eq 0 ] && [ ${localPath === null ? "false" : "true"} = true ]; then + local_path_ns=${shQuote(localPath?.namespace ?? "")} + local_path_configmap=${shQuote(localPath?.configMapName ?? "")} + local_path_deployment=${shQuote(localPath?.provisionerDeployment ?? "")} + helper_path=$(mktemp /tmp/hwlab-local-path-helper.XXXXXX.yaml) + printf %s ${shQuote(helperEncoded)} | base64 -d >"$helper_path" + patch_path=$(mktemp /tmp/hwlab-local-path-helper.XXXXXX.json) + python3 - "$helper_path" "$patch_path" <<'PY' +import json, pathlib, sys +helper=pathlib.Path(sys.argv[1]).read_text() +pathlib.Path(sys.argv[2]).write_text(json.dumps({"data":{"helperPod.yaml": helper}}, ensure_ascii=False)) +PY + ( + kubectl -n "$local_path_ns" patch configmap "$local_path_configmap" --type merge -p "$(cat "$patch_path")" + patch_rc=$? + if [ "$patch_rc" -eq 0 ]; then + kubectl -n "$local_path_ns" rollout restart deployment "$local_path_deployment" || true + kubectl -n "$local_path_ns" get pod -o name | grep '^pod/helper-pod-create-pvc-' | xargs -r kubectl -n "$local_path_ns" delete --ignore-not-found=true + fi + exit "$patch_rc" + ) >/tmp/hwlab-node-local-path.out 2>/tmp/hwlab-node-local-path.err + storage_rc=$? + rm -f "$helper_path" "$patch_path" +fi +if [ "$rc" -eq 0 ] && [ ${argoRepositoryCredential?.enabled === true ? "true" : "false"} = true ]; then + argo_ns=${shQuote(target.argo.namespace)} + repo_secret=${shQuote(argoRepositoryCredential?.secretName ?? "")} + repo_url=${shQuote(argoRepositoryCredential?.repoURL ?? "")} + source_ns=${shQuote(argoRepositoryCredential?.sourceSecret.namespace ?? "")} + source_secret=${shQuote(argoRepositoryCredential?.sourceSecret.name ?? "")} + source_key=${shQuote(argoRepositoryCredential?.sourceSecret.key ?? "")} + repo_key_file=$(mktemp /tmp/hwlab-argo-repository-key.XXXXXX) + ( + set -eu + kubectl -n "$source_ns" get secret "$source_secret" -o "jsonpath={.data.$source_key}" | base64 -d >"$repo_key_file" + kubectl -n "$argo_ns" create secret generic "$repo_secret" --from-literal=type=git --from-literal=url="$repo_url" --from-file=sshPrivateKey="$repo_key_file" --dry-run=client -o yaml \\ + | kubectl label --local -f - argocd.argoproj.io/secret-type=repository app.kubernetes.io/part-of=hwlab-node-control-plane hwlab.pikastech.local/node=${shQuote(target.node)} hwlab.pikastech.local/lane=${shQuote(target.lane)} -o yaml \\ + | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-argo-repository -f - + ) >/tmp/hwlab-node-argo-repository.out 2>/tmp/hwlab-node-argo-repository.err + repo_credential_rc=$? + rm -f "$repo_key_file" +fi +python3 - "$rc" "$storage_rc" "$repo_credential_rc" <<'PY' import json, pathlib, sys out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else '' err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else '' -print(json.dumps({'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}, ensure_ascii=False)) +storage_out=pathlib.Path('/tmp/hwlab-node-local-path.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-local-path.out').exists() else '' +storage_err=pathlib.Path('/tmp/hwlab-node-local-path.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-local-path.err').exists() else '' +repo_out=pathlib.Path('/tmp/hwlab-node-argo-repository.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-argo-repository.out').exists() else '' +repo_err=pathlib.Path('/tmp/hwlab-node-argo-repository.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-argo-repository.err').exists() else '' +print(json.dumps({'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'storage': {'localPathApplyExitCode': int(sys.argv[2]), 'stdoutPreview': storage_out[-2000:], 'stderrPreview': storage_err[-2000:]}, 'argoRepositoryCredential': {'applyExitCode': int(sys.argv[3]), 'stdoutPreview': repo_out[-2000:], 'stderrPreview': repo_err[-2000:], 'valuesPrinted': False}, 'runtimeRolloutTriggered': False, 'pk01Touched': False}, ensure_ascii=False)) PY rm -f "$manifest" -exit "$rc" +if [ "$rc" -ne 0 ]; then exit "$rc"; fi +if [ "$storage_rc" -ne 0 ]; then exit "$storage_rc"; fi +exit "$repo_credential_rc" `; } +function localPathHelperPodYaml(spec: LocalPathStorageSpec): string { + return [ + "apiVersion: v1", + "kind: Pod", + "metadata:", + " name: helper-pod", + "spec:", + " containers:", + " - name: helper-pod", + ` image: "${spec.helperImage}"`, + ` imagePullPolicy: ${spec.imagePullPolicy}`, + "", + ].join("\n"); +} + function toolsImageStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, timeoutSeconds: number): { registryReady: boolean; toolsImageReady: boolean; @@ -1134,6 +1719,7 @@ function statusNext( gitMirror: Record, argo: Record, ciNamespace: Record, + storageReady: boolean, ): Record { const bootstrapMissing = !boolField(ciNamespace, "exists") || !boolField(gitMirror, "namespaceExists") @@ -1143,6 +1729,7 @@ function statusNext( const blockers: string[] = []; if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); + if (!storageReady) blockers.push("local-path-storage-not-ready"); if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); const argoInstall = record(argo.install); if (!boolField(argo, "installed")) blockers.push("argocd-not-installed"); @@ -1162,6 +1749,9 @@ function statusNext( if (!boolField(registry, "toolsImageReady")) { next.buildToolsImage = "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。"; } + if (!storageReady) { + next.fixLocalPathStorage = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; + } if (!boolField(argo, "installed")) { next.installArgo = "准备受控 D601 Argo CD 安装入口后再进入 runtime rollout。"; } @@ -1267,6 +1857,7 @@ function argoApplyStartScript(node: ControlPlaneNodeSpec, target: ControlPlaneTa const desiredEncoded = Buffer.from(desiredYaml, "utf8").toString("base64"); const rewritesEncoded = Buffer.from(JSON.stringify(target.argo.install.imageRewrites), "utf8").toString("base64"); const preloadEncoded = Buffer.from(JSON.stringify(target.argo.install.preloadImages), "utf8").toString("base64"); + const obsoleteApplications = shellSingleQuotedLines(target.argo.obsoleteApplications); return ` set -eu state_dir=${shQuote(stateDir)} @@ -1283,11 +1874,19 @@ namespace=${shQuote(target.argo.namespace)} manifest_url=${shQuote(target.argo.install.manifestUrl)} field_manager=${shQuote(target.argo.install.fieldManager)} readiness_timeout=${shQuote(String(target.argo.install.readinessTimeoutSeconds))} +repo_credential_enabled=${target.argo.repositoryCredential?.enabled === true ? "true" : "false"} +repo_secret=${shQuote(target.argo.repositoryCredential?.secretName ?? "")} +repo_url=${shQuote(target.argo.repositoryCredential?.repoURL ?? "")} +source_ns=${shQuote(target.argo.repositoryCredential?.sourceSecret.namespace ?? "")} +source_secret=${shQuote(target.argo.repositoryCredential?.sourceSecret.name ?? "")} +source_key=${shQuote(target.argo.repositoryCredential?.sourceSecret.key ?? "")} +active_app=${shQuote(target.argo.applicationName)} log="$state_dir/job.log" status="$state_dir/status.json" install_yaml="$state_dir/install.yaml" rendered_yaml="$state_dir/install.rendered.yaml" desired_yaml="$state_dir/desired.yaml" +obsolete_apps_file="$state_dir/obsolete-applications.txt" write_status() { state="$1"; shift message="$1"; shift || true @@ -1303,11 +1902,23 @@ ${proxyExportBlock(node)} printf %s ${shQuote(desiredEncoded)} | base64 -d >"$desired_yaml" printf %s ${shQuote(rewritesEncoded)} | base64 -d >"$state_dir/image-rewrites.json" printf %s ${shQuote(preloadEncoded)} | base64 -d >"$state_dir/preload-images.json" + cat >"$obsolete_apps_file" <<'OBSOLETE_APPS' +${obsoleteApplications} +OBSOLETE_APPS kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply --server-side --field-manager="$field_manager" -f - || exit "$?" python3 - "$state_dir/preload-images.json" "$state_dir/image-rewrites.json" <<'PY' >"$state_dir/pull-images.sh" import json, pathlib, shlex, sys preload=json.loads(pathlib.Path(sys.argv[1]).read_text()) rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) +def registry_probe(image): + prefix="127.0.0.1:5000/" + if not image.startswith(prefix): + return None + repo_tag=image[len(prefix):] + if ":" not in repo_tag: + return None + repo, tag=repo_tag.rsplit(":", 1) + return "http://127.0.0.1:5000/v2/" + repo + "/manifests/" + tag print("#!/bin/sh") print("set -eu") seen=set() @@ -1317,6 +1928,16 @@ for item in rewrites: if target in seen: continue seen.add(target) + probe=registry_probe(target) + if probe: + print("if curl -fsS --max-time 5 " + shlex.quote(probe) + " >/dev/null 2>&1; then") + print(" echo " + shlex.quote("image already present: " + target)) + print("else") + print(" docker pull " + shlex.quote(pull)) + print(" docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) + print(" docker push " + shlex.quote(target)) + print("fi") + continue print("docker pull " + shlex.quote(pull)) print("docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) print("docker push " + shlex.quote(target)) @@ -1344,7 +1965,30 @@ PY sleep 5 done kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null || exit "$?" - kubectl apply --server-side --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" + if [ "$repo_credential_enabled" = true ]; then + repo_key_file=$(mktemp /tmp/hwlab-argo-repository-key.XXXXXX) + kubectl -n "$source_ns" get secret "$source_secret" -o "jsonpath={.data.$source_key}" | base64 -d >"$repo_key_file" || exit "$?" + kubectl -n "$namespace" create secret generic "$repo_secret" --from-literal=type=git --from-literal=url="$repo_url" --from-file=sshPrivateKey="$repo_key_file" --dry-run=client -o yaml \\ + | kubectl label --local -f - argocd.argoproj.io/secret-type=repository app.kubernetes.io/part-of=hwlab-node-control-plane hwlab.pikastech.local/node=${shQuote(target.node)} hwlab.pikastech.local/lane=${shQuote(target.lane)} -o yaml \\ + | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-argo-repository -f - || exit "$?" + rm -f "$repo_key_file" + fi + kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" + while IFS= read -r obsolete_app; do + [ -n "$obsolete_app" ] || continue + if [ "$obsolete_app" = "$active_app" ]; then + echo "refusing to delete active Argo application listed as obsolete: $obsolete_app" >&2 + exit 42 + fi + kubectl -n "$namespace" delete application "$obsolete_app" --ignore-not-found=true || exit "$?" + done <"$obsolete_apps_file" + kubectl -n "$namespace" rollout restart deployment argocd-repo-server || exit "$?" + kubectl -n "$namespace" rollout status deployment argocd-repo-server --timeout=180s || exit "$?" + kubectl -n "$namespace" rollout restart statefulset argocd-application-controller || exit "$?" + if ! kubectl -n "$namespace" rollout status statefulset argocd-application-controller --timeout=120s; then + kubectl -n "$namespace" delete pod -l app.kubernetes.io/name=argocd-application-controller --ignore-not-found=true --wait=false || exit "$?" + kubectl -n "$namespace" rollout status statefulset argocd-application-controller --timeout=180s || exit "$?" + fi write_status succeeded argocd-install-applied } >>"$log" 2>&1 || { rc=$? @@ -1421,7 +2065,7 @@ function manifestObjectSummary(manifest: readonly Record[]): Re } function runTransK3s(kubeRoute: string, script: string, timeoutSeconds: number): CommandResult { - return runCommand(["/root/.local/bin/trans", kubeRoute, "script", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); + return runCommand([rootPath("scripts", "trans"), kubeRoute, "script", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); } function proxyExportBlock(node: ControlPlaneNodeSpec): string { @@ -1450,6 +2094,15 @@ function shellJsonArray(items: readonly string[]): string { return JSON.stringify([...items]); } +function shellSingleQuotedLines(items: readonly string[]): string { + for (const item of items) validateKubernetesName(item, "argo.obsoleteApplications"); + return items.join("\n"); +} + +function validateKubernetesName(value: string, path: string): void { + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} contains invalid Kubernetes metadata.name: ${value}`); +} + function parseRemoteJson(text: string): unknown { const trimmed = text.trim(); if (trimmed.length === 0) return null; @@ -1515,6 +2168,11 @@ function stringArrayField(obj: Record, key: string, path: strin return [...value] as string[]; } +function optionalStringArrayField(obj: Record, key: string, path: string): string[] { + if (obj[key] === undefined) return []; + return stringArrayField(obj, key, path); +} + function stringRecordField(obj: Record, path: string): Record { const result: Record = {}; for (const [key, value] of Object.entries(obj)) { diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index f8d3a875..ef2f9584 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -32,6 +32,7 @@ export interface HwlabDownloadProfileSpec { readonly git: { readonly proxyMode: "inherit" | "direct" | "none"; readonly retries: number; + readonly timeoutSeconds: number; }; readonly npm: { readonly registry: string; @@ -54,6 +55,36 @@ export interface HwlabDownloadProfileSpec { }; } +export interface HwlabRuntimeExternalPostgresComponentSpec { + readonly secretName: string; + readonly secretKey: string; + readonly sourceRef: string; + readonly envKey: string; + readonly role: string; + readonly authnKey?: string; + readonly schema?: string; +} + +export interface HwlabRuntimeExternalPostgresSpec { + readonly provider: string; + readonly configRef: string; + readonly serviceName: string; + readonly endpointAddress: string; + readonly port: number; + readonly sslmode: "require"; + readonly database: string; + readonly cloudApi: HwlabRuntimeExternalPostgresComponentSpec; + readonly openfga: HwlabRuntimeExternalPostgresComponentSpec; +} + +export interface HwlabRuntimeBuildkitSpec { + readonly sidecarImage: string; +} + +export interface HwlabRuntimeObservabilitySpec { + readonly prometheusOperator: boolean; +} + export interface HwlabRuntimeLaneSpec { readonly lane: HwlabRuntimeLane; readonly nodeId: string; @@ -74,6 +105,7 @@ export interface HwlabRuntimeLaneSpec { readonly gitUrl: string; readonly gitReadUrl: string; readonly gitWriteUrl: string; + readonly argoRepoUrl: string; readonly gitopsBranch: string; readonly catalogPath: string; readonly runtimePath: string; @@ -83,9 +115,14 @@ export interface HwlabRuntimeLaneSpec { readonly argoApplicationFile: string; readonly registryPrefix: string; readonly baseImage: string; + readonly baseImageSource?: string; readonly serviceIds: readonly string[]; readonly publicWebUrl: string; readonly publicApiUrl: string; + readonly stepEnv: Record; + readonly buildkit?: HwlabRuntimeBuildkitSpec; + readonly externalPostgres?: HwlabRuntimeExternalPostgresSpec; + readonly observability: HwlabRuntimeObservabilitySpec; readonly networkProfileId: string; readonly downloadProfileId: string; readonly networkProfile: HwlabNetworkProfileSpec; @@ -109,6 +146,7 @@ interface HwlabLaneConfig { readonly serviceAccountName: string; readonly controlPlaneFieldManager: string; readonly git: { readonly url: string; readonly readUrl: string; readonly writeUrl: string }; + readonly argo: { readonly repoURL?: string }; readonly gitopsBranch: string; readonly catalogPath: string; readonly runtime: { readonly path: string; readonly namespace: string; readonly renderDir: string }; @@ -116,14 +154,20 @@ interface HwlabLaneConfig { readonly argoApplicationFile: string; readonly registryPrefix: string; readonly baseImage: string; + readonly baseImageSource?: string; readonly serviceIds: readonly string[]; readonly public: { readonly webUrl: string; readonly apiUrl: string }; + readonly stepEnv: Record; + readonly buildkit?: HwlabRuntimeBuildkitSpec; + readonly externalPostgres?: HwlabRuntimeExternalPostgresSpec; + readonly observability: HwlabRuntimeObservabilitySpec; } interface HwlabNodeLaneConfig { readonly requiredNoProxy: readonly string[]; readonly nodes: Record; readonly lanes: Record; + readonly laneTargets: Partial>>; readonly networkProfiles: Record; readonly downloadProfiles: Record; } @@ -160,6 +204,12 @@ function optionalStringField(obj: Record, key: string, path: st return value; } +function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + function sortedRecordEntries(value: unknown, path: string): Array<[string, Record]> { return Object.entries(asRecord(value, path)).map(([key, item]) => [key, asRecord(item, `${path}.${key}`)]); } @@ -168,6 +218,22 @@ function unique(values: readonly string[]): string[] { return [...new Set(values)]; } +function mergeOptionalRecord(base: unknown, override: unknown): Record | undefined { + if (base === undefined && override === undefined) return undefined; + const baseRecord = base === undefined ? {} : asRecord(base, "base"); + const overrideRecord = override === undefined ? {} : asRecord(override, "override"); + return { ...baseRecord, ...overrideRecord }; +} + +function optionalStringRecord(value: unknown, path: string): Record { + if (value === undefined) return {}; + const raw = asRecord(value, path); + return Object.fromEntries(Object.entries(raw).map(([key, item]) => { + if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return [key, item]; + })); +} + function proxyConfig(raw: Record, id: string, key: "proxy" | "dockerBuildProxy", requiredNoProxy: readonly string[]): HwlabProxySpec { const proxy = asRecord(raw[key], `networkProfiles.${id}.${key}`); return { @@ -199,7 +265,11 @@ function downloadProfileConfig(id: string, raw: Record): HwlabD } return { id, - git: { proxyMode, retries: numberField(git, "retries", `downloadProfiles.${id}.git`) }, + git: { + proxyMode, + retries: numberField(git, "retries", `downloadProfiles.${id}.git`), + timeoutSeconds: numberField(git, "timeoutSeconds", `downloadProfiles.${id}.git`), + }, npm: { registry: stringField(npm, "registry", `downloadProfiles.${id}.npm`), retries: numberField(npm, "retries", `downloadProfiles.${id}.npm`), @@ -240,6 +310,7 @@ function isSupportedLaneId(id: string): id is HwlabRuntimeLane { function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLaneConfig { const git = asRecord(raw.git, `lanes.${id}.git`); + const argo = raw.argo === undefined ? {} : asRecord(raw.argo, `lanes.${id}.argo`); const runtime = asRecord(raw.runtime, `lanes.${id}.runtime`); const publicUrls = asRecord(raw.public, `lanes.${id}.public`); const minor = numberField(raw, "minor", `lanes.${id}`); @@ -264,6 +335,9 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa readUrl: stringField(git, "readUrl", `lanes.${id}.git`), writeUrl: stringField(git, "writeUrl", `lanes.${id}.git`), }, + argo: { + repoURL: optionalStringField(argo, "repoURL", `lanes.${id}.argo`), + }, gitopsBranch: stringField(raw, "gitopsBranch", `lanes.${id}`), catalogPath: stringField(raw, "catalogPath", `lanes.${id}`), runtime: { @@ -275,11 +349,80 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa argoApplicationFile: stringField(raw, "argoApplicationFile", `lanes.${id}`), registryPrefix: stringField(raw, "registryPrefix", `lanes.${id}`), baseImage: stringField(raw, "baseImage", `lanes.${id}`), + baseImageSource: optionalStringField(raw, "baseImageSource", `lanes.${id}`), serviceIds: stringArrayField(raw, "serviceIds", `lanes.${id}`), public: { webUrl: stringField(publicUrls, "webUrl", `lanes.${id}.public`), apiUrl: stringField(publicUrls, "apiUrl", `lanes.${id}.public`), }, + stepEnv: optionalStringRecord(raw.stepEnv, `lanes.${id}.stepEnv`), + buildkit: buildkitConfig(raw.buildkit, `lanes.${id}.buildkit`), + externalPostgres: externalPostgresConfig(raw.externalPostgres, `lanes.${id}.externalPostgres`), + observability: observabilityConfig(raw.observability, `lanes.${id}.observability`), + }; +} + +function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record, targetRaw: Record): HwlabLaneConfig { + const merged: Record = { + ...baseRaw, + ...targetRaw, + node: nodeId, + git: mergeOptionalRecord(baseRaw.git, targetRaw.git), + argo: mergeOptionalRecord(baseRaw.argo, targetRaw.argo), + runtime: mergeOptionalRecord(baseRaw.runtime, targetRaw.runtime), + public: mergeOptionalRecord(baseRaw.public, targetRaw.public), + stepEnv: mergeOptionalRecord(baseRaw.stepEnv, targetRaw.stepEnv) ?? {}, + buildkit: mergeOptionalRecord(baseRaw.buildkit, targetRaw.buildkit), + externalPostgres: mergeOptionalRecord(baseRaw.externalPostgres, targetRaw.externalPostgres), + observability: mergeOptionalRecord(baseRaw.observability, targetRaw.observability), + }; + delete merged.targets; + return laneConfig(id, merged); +} + +function buildkitConfig(value: unknown, path: string): HwlabRuntimeBuildkitSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + sidecarImage: stringField(raw, "sidecarImage", path), + }; +} + +function externalPostgresComponentConfig(value: unknown, path: string): HwlabRuntimeExternalPostgresComponentSpec { + const raw = asRecord(value, path); + return { + secretName: stringField(raw, "secretName", path), + secretKey: stringField(raw, "secretKey", path), + sourceRef: stringField(raw, "sourceRef", path), + envKey: stringField(raw, "envKey", path), + role: stringField(raw, "role", path), + authnKey: optionalStringField(raw, "authnKey", path), + schema: optionalStringField(raw, "schema", path), + }; +} + +function externalPostgresConfig(value: unknown, path: string): HwlabRuntimeExternalPostgresSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + const sslmode = stringField(raw, "sslmode", path); + if (sslmode !== "require") throw new Error(`${path}.sslmode must be require`); + return { + provider: stringField(raw, "provider", path), + configRef: stringField(raw, "configRef", path), + serviceName: stringField(raw, "serviceName", path), + endpointAddress: stringField(raw, "endpointAddress", path), + port: numberField(raw, "port", path), + sslmode, + database: stringField(raw, "database", path), + cloudApi: externalPostgresComponentConfig(raw.cloudApi, `${path}.cloudApi`), + openfga: externalPostgresComponentConfig(raw.openfga, `${path}.openfga`), + }; +} + +function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservabilitySpec { + const raw = asRecord(value, path); + return { + prometheusOperator: booleanField(raw, "prometheusOperator", path), }; } @@ -298,18 +441,27 @@ function readHwlabNodeLaneConfig(): HwlabNodeLaneConfig { const downloadProfiles = Object.fromEntries( sortedRecordEntries(parsed.downloadProfiles, "downloadProfiles").map(([id, item]) => [id, downloadProfileConfig(id, item)]), ); - const lanes = Object.fromEntries(sortedRecordEntries(parsed.lanes, "lanes").map(([id, item]) => { + const laneEntries = sortedRecordEntries(parsed.lanes, "lanes"); + const lanes = Object.fromEntries(laneEntries.map(([id, item]) => { if (!isSupportedLaneId(id)) throw new Error(`lanes.${id} is not supported by this CLI build`); return [id, laneConfig(id, item)]; })) as Record; + const laneTargets: Partial>> = {}; + for (const [id, item] of laneEntries) { + if (!isSupportedLaneId(id) || item.targets === undefined) continue; + laneTargets[id] = Object.fromEntries(sortedRecordEntries(item.targets, `lanes.${id}.targets`).map(([nodeId, target]) => [ + nodeId, + laneTargetConfig(id, nodeId, item, target), + ])); + } for (const node of Object.values(nodes)) { if (networkProfiles[node.networkProfileId] === undefined) throw new Error(`nodes.${node.id}.networkProfile references missing profile ${node.networkProfileId}`); if (downloadProfiles[node.downloadProfileId] === undefined) throw new Error(`nodes.${node.id}.downloadProfile references missing profile ${node.downloadProfileId}`); } - for (const lane of Object.values(lanes)) { + for (const lane of [...Object.values(lanes), ...Object.values(laneTargets).flatMap((targets) => Object.values(targets))]) { if (nodes[lane.node] === undefined) throw new Error(`lanes.${lane.id}.node references missing node ${lane.node}`); } - return { requiredNoProxy, nodes, lanes, networkProfiles, downloadProfiles }; + return { requiredNoProxy, nodes, lanes, laneTargets, networkProfiles, downloadProfiles }; } const HWLAB_NODE_LANE_CONFIG = readHwlabNodeLaneConfig(); @@ -338,6 +490,7 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { gitUrl: config.git.url, gitReadUrl: config.git.readUrl, gitWriteUrl: config.git.writeUrl, + argoRepoUrl: config.argo.repoURL ?? config.git.readUrl, gitopsBranch: config.gitopsBranch, catalogPath: config.catalogPath, runtimePath: config.runtime.path, @@ -347,9 +500,14 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { argoApplicationFile: config.argoApplicationFile, registryPrefix: config.registryPrefix, baseImage: config.baseImage, + ...(config.baseImageSource === undefined ? {} : { baseImageSource: config.baseImageSource }), serviceIds: config.serviceIds, publicWebUrl: config.public.webUrl, publicApiUrl: config.public.apiUrl, + stepEnv: config.stepEnv, + ...(config.buildkit === undefined ? {} : { buildkit: config.buildkit }), + ...(config.externalPostgres === undefined ? {} : { externalPostgres: config.externalPostgres }), + observability: config.observability, networkProfileId: networkProfile.id, downloadProfileId: downloadProfile.id, networkProfile, @@ -369,6 +527,15 @@ export function hwlabRuntimeLaneSpec(lane: HwlabRuntimeLane): HwlabRuntimeLaneSp return RUNTIME_LANE_SPECS[lane]; } +export function hwlabRuntimeLaneSpecForNode(lane: HwlabRuntimeLane, nodeId: string): HwlabRuntimeLaneSpec { + const targetSpec = HWLAB_NODE_LANE_CONFIG.laneTargets[lane]?.[nodeId]; + if (targetSpec !== undefined) return buildRuntimeLaneSpec(targetSpec); + const defaultSpec = RUNTIME_LANE_SPECS[lane]; + if (defaultSpec.nodeId === nodeId) return defaultSpec; + const knownNodes = unique([defaultSpec.nodeId, ...Object.keys(HWLAB_NODE_LANE_CONFIG.laneTargets[lane] ?? {})]).join(", "); + throw new Error(`lane ${lane} has no target for node ${nodeId}; known nodes: ${knownNodes}`); +} + export function hwlabRuntimeLaneIds(): HwlabRuntimeLane[] { return Object.keys(RUNTIME_LANE_SPECS) as HwlabRuntimeLane[]; } diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 06094af5..c2964aa3 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -1,14 +1,23 @@ import { existsSync, readFileSync } from "node:fs"; -import { repoRoot, type Config } from "./config"; +import { join } from "node:path"; +import { repoRoot, rootPath, type Config } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { runHwlabG14Command } from "./hwlab-g14"; import { hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; -import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; type DelegatedNodeDomain = "control-plane" | "git-mirror"; +type NodeRuntimeRenderLocation = "node-host" | "local"; + +interface NodeRuntimeRenderResult { + readonly result: CommandResult; + readonly renderDir: string; + readonly worktreeDir: string; + readonly location: NodeRuntimeRenderLocation; +} interface NodeSecretOptions { action: SecretAction; @@ -101,9 +110,12 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane plan --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", @@ -124,6 +136,30 @@ export function hwlabNodeHelp(): Record { async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { const scoped = parseNodeScopedDelegatedOptions(domain, args); + const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane); + if (domain === "control-plane" && scoped.action === "plan") { + return nodeRuntimeControlPlanePlan(scoped); + } + if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) { + if (scoped.action === "status") return nodeRuntimeControlPlaneStatus(scoped); + if (scoped.action === "apply" || scoped.action === "trigger-current" || scoped.action === "refresh" || scoped.action === "sync") { + if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped); + return nodeRuntimeControlPlaneRun(scoped); + } + return nodeRuntimeUnsupportedAction(scoped); + } + if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) { + return { + ok: false, + command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mutation: false, + degradedReason: "node-scoped-git-mirror-action-not-enabled", + message: "D601 runtime GitOps is declared in UniDesk YAML and triggered by node-scoped control-plane commands; this command intentionally does not delegate D601 git-mirror actions to G14.", + expected: nodeRuntimeExpected(scoped.spec), + }; + } if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") { return runNodeEndpointBridge(scoped); } @@ -146,8 +182,10 @@ function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: stri confirm: boolean; dryRun: boolean; wait: boolean; + rerun: boolean; timeoutSeconds: number; originalArgs: string[]; + spec: HwlabRuntimeLaneSpec; } { const [actionRaw] = args; if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`); @@ -155,8 +193,7 @@ function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: stri assertNodeId(node); const laneRaw = requiredOption(args, "--lane"); if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); - const spec = hwlabRuntimeLaneSpec(laneRaw); - if (spec.nodeId !== node) throw new Error(`lane ${laneRaw} is configured for node ${spec.nodeId}; got --node ${node}`); + const spec = hwlabRuntimeLaneSpecForNode(laneRaw, node); const confirm = args.includes("--confirm"); const dryRun = args.includes("--dry-run"); if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`); @@ -168,8 +205,1936 @@ function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: stri confirm, dryRun, wait: args.includes("--wait"), + rerun: args.includes("--rerun"), timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600), originalArgs: [...args], + spec, + }; +} + +function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record { + return { + configPath: hwlabRuntimeLaneConfigPath(), + node: spec.nodeId, + nodeRoute: spec.nodeRoute, + nodeKubeRoute: spec.nodeKubeRoute, + lane: spec.lane, + sourceBranch: spec.sourceBranch, + workspace: spec.workspace, + cicdRepo: spec.cicdRepo, + git: { + sourceUrl: spec.gitUrl, + readUrl: spec.gitReadUrl, + writeUrl: spec.gitWriteUrl, + }, + argo: { + repoURL: spec.argoRepoUrl, + }, + gitopsBranch: spec.gitopsBranch, + catalogPath: spec.catalogPath, + runtimePath: spec.runtimePath, + runtimeNamespace: spec.runtimeNamespace, + renderDir: spec.runtimeRenderDir, + pipeline: spec.pipeline, + pipelineRunPrefix: spec.pipelineRunPrefix, + serviceAccount: spec.serviceAccountName, + argoApplication: spec.app, + registryPrefix: spec.registryPrefix, + baseImage: { + image: spec.baseImage, + sourceImage: spec.baseImageSource ?? null, + }, + buildkit: spec.buildkit === undefined ? null : { + sidecarImage: spec.buildkit.sidecarImage, + }, + dockerBuildProxy: { + http: spec.networkProfile.dockerBuildProxy.http, + https: spec.networkProfile.dockerBuildProxy.https, + all: spec.networkProfile.dockerBuildProxy.all, + noProxy: spec.networkProfile.dockerBuildProxy.noProxy, + }, + stepEnv: spec.stepEnv, + public: { + webUrl: spec.publicWebUrl, + apiUrl: spec.publicApiUrl, + }, + downloadProfile: { + id: spec.downloadProfileId, + git: spec.downloadProfile.git, + npm: spec.downloadProfile.npm, + }, + observability: spec.observability, + externalPostgres: spec.externalPostgres === undefined ? null : { + provider: spec.externalPostgres.provider, + configRef: spec.externalPostgres.configRef, + serviceName: spec.externalPostgres.serviceName, + endpointAddress: spec.externalPostgres.endpointAddress, + port: spec.externalPostgres.port, + sslmode: spec.externalPostgres.sslmode, + database: spec.externalPostgres.database, + cloudApi: { + secretName: spec.externalPostgres.cloudApi.secretName, + secretKey: spec.externalPostgres.cloudApi.secretKey, + sourceRef: spec.externalPostgres.cloudApi.sourceRef, + envKey: spec.externalPostgres.cloudApi.envKey, + role: spec.externalPostgres.cloudApi.role, + }, + openfga: { + secretName: spec.externalPostgres.openfga.secretName, + secretKey: spec.externalPostgres.openfga.secretKey, + sourceRef: spec.externalPostgres.openfga.sourceRef, + envKey: spec.externalPostgres.openfga.envKey, + authnKey: spec.externalPostgres.openfga.authnKey ?? null, + role: spec.externalPostgres.openfga.role, + schema: spec.externalPostgres.openfga.schema ?? null, + }, + valuesPrinted: false, + }, + localPostgres: { + shouldRender: spec.externalPostgres === undefined, + expectedAbsent: spec.externalPostgres !== undefined, + }, + }; +} + +function nodeRuntimeControlPlanePlan(scoped: ReturnType): Record { + return { + ok: true, + command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, + mode: "plan", + mutation: false, + node: scoped.node, + lane: scoped.lane, + expected: nodeRuntimeExpected(scoped.spec), + checks: { + nodeScopedTargetConfigured: true, + externalPostgresDeclared: scoped.spec.externalPostgres !== undefined, + secretValuesPrinted: false, + runtimeNamespace: scoped.spec.runtimeNamespace, + localPostgresExpectedAbsent: scoped.spec.externalPostgres !== undefined, + }, + next: { + infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`, + status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, + platformDbStatus: scoped.spec.externalPostgres === undefined + ? null + : `bun scripts/cli.ts platform-db postgres status --config ${scoped.spec.externalPostgres.configRef}`, + }, + }; +} + +function compactRuntimeCommand(result: CommandResult): Record { + return { + exitCode: result.exitCode, + stdoutBytes: Buffer.byteLength(result.stdout), + stderrBytes: Buffer.byteLength(result.stderr), + stdoutTail: result.stdout.trim().slice(-2000), + stderrTail: result.stderr.trim().slice(-2000), + timedOut: result.timedOut, + }; +} + +function nodeRuntimeUnsupportedAction(scoped: ReturnType): Record { + return { + ok: false, + command: `hwlab nodes control-plane ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mutation: false, + degradedReason: "unsupported-node-scoped-runtime-action", + message: "node-scoped runtime currently supports plan/status/apply/refresh/sync/trigger-current", + expected: nodeRuntimeExpected(scoped.spec), + }; +} + +function transPath(): string { + return join(repoRoot, "scripts", "trans"); +} + +function runNodeHostScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { + return runCommand([transPath(), spec.nodeRoute, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function runNodeHostScriptAsync(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, label: string): CommandResult { + const token = nodeRuntimeRenderToken(); + const stateDir = `/tmp/unidesk-hwlab-node-runtime/${label}-${token}`; + const statusPath = `${stateDir}/status.json`; + const stdoutPath = `${stateDir}/stdout.log`; + const stderrPath = `${stateDir}/stderr.log`; + const runnerPath = `${stateDir}/run.sh`; + const startScript = [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + `status_path=${shellQuote(statusPath)}`, + `stdout_path=${shellQuote(stdoutPath)}`, + `stderr_path=${shellQuote(stderrPath)}`, + `runner_path=${shellQuote(runnerPath)}`, + "rm -rf \"$state_dir\"", + "mkdir -p \"$state_dir\"", + "cat > \"$runner_path\" <<'UNIDESK_REMOTE_RUNNER'", + script, + "UNIDESK_REMOTE_RUNNER", + "chmod 700 \"$runner_path\"", + "python3 - \"$status_path\" <<'PY'", + "import datetime, json, sys", + "json.dump({'state':'running','ok':None,'pid':None,'startedAt':datetime.datetime.utcnow().isoformat()+'Z','finishedAt':None,'exitCode':None}, open(sys.argv[1], 'w'), indent=2)", + "PY", + "(", + " set +e", + " sh \"$runner_path\" >\"$stdout_path\" 2>\"$stderr_path\"", + " code=$?", + " python3 - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$code\" <<'PY'", + "import datetime, json, pathlib, sys", + "status_path, stdout_path, stderr_path, code = sys.argv[1:5]", + "def tail(path):", + " try:", + " text = pathlib.Path(path).read_text(errors='replace')", + " except FileNotFoundError:", + " return ''", + " return text[-12000:]", + "payload = {'state':'finished','ok':code == '0','pid':None,'startedAt':None,'finishedAt':datetime.datetime.utcnow().isoformat()+'Z','exitCode':int(code),'stdout':tail(stdout_path),'stderr':tail(stderr_path)}", + "try:", + " current = json.load(open(status_path))", + " payload['startedAt'] = current.get('startedAt')", + "except Exception:", + " pass", + "json.dump(payload, open(status_path, 'w'), indent=2)", + "PY", + ") >/dev/null 2>&1 &", + "pid=$!", + "python3 - \"$status_path\" \"$pid\" <<'PY'", + "import json, sys", + "path, pid = sys.argv[1:3]", + "payload = json.load(open(path))", + "payload['pid'] = int(pid)", + "json.dump(payload, open(path, 'w'), indent=2)", + "PY", + "printf '{\"ok\":true,\"state\":\"started\",\"pid\":%s,\"statusPath\":\"%s\"}\\n' \"$pid\" \"$status_path\"", + ].join("\n"); + const start = runNodeHostScript(spec, startScript, 55); + if (!isCommandSuccess(start)) return start; + const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000; + let lastStatus: CommandResult | null = null; + let payload: Record = {}; + while (Date.now() <= deadline) { + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + const status = runNodeHostScript(spec, `cat ${shellQuote(statusPath)} 2>/dev/null || printf '{"state":"missing","ok":false,"exitCode":127,"stdout":"","stderr":"missing async status"}\\n'`, 55); + lastStatus = status; + payload = parseJsonObject(status.stdout.trim()); + if (payload.state === "finished" || payload.ok === true || payload.ok === false) { + return commandResultFromAsync(spec, payload, statusPath, false); + } + } + const kill = runNodeHostScript(spec, [ + "set +e", + `status_path=${shellQuote(statusPath)}`, + "pid=$(python3 - \"$status_path\" <<'PY' 2>/dev/null", + "import json, sys", + "try: print(json.load(open(sys.argv[1])).get('pid') or '')", + "except Exception: print('')", + "PY", + ")", + "if [ -n \"$pid\" ]; then kill \"$pid\" 2>/dev/null || true; fi", + "cat \"$status_path\" 2>/dev/null || true", + ].join("\n"), 55); + return { + command: [transPath(), spec.nodeRoute, "script", "--", ""], + cwd: repoRoot, + exitCode: 124, + stdout: typeof payload.stdout === "string" ? payload.stdout : "", + stderr: [ + `remote async command timed out after ${timeoutSeconds}s; statusPath=${statusPath}`, + typeof payload.stderr === "string" ? payload.stderr : "", + lastStatus?.stderr.trim() ?? "", + kill.stderr.trim(), + ].filter(Boolean).join("\n"), + signal: null, + timedOut: true, + }; +} + +function parseJsonObject(text: string): Record { + try { + const parsed = JSON.parse(text) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; + } catch { + return {}; + } +} + +function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record, statusPath: string, timedOut: boolean): CommandResult { + return { + command: [transPath(), spec.nodeRoute, "script", "--", ""], + cwd: repoRoot, + exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1, + stdout: typeof payload.stdout === "string" ? payload.stdout : "", + stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`, + signal: null, + timedOut, + }; +} + +function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult { + return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 }); +} + +function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { + return runCommand([transPath(), spec.nodeKubeRoute, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function isCommandSuccess(result: CommandResult): boolean { + return result.exitCode === 0 && !result.timedOut; +} + +function shortSha(value: string): string { + return value.slice(0, 12).toLowerCase(); +} + +function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { + return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; +} + +function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string { + const suffix = `/${spec.runtimeRenderDir}`; + if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length); + return spec.gitopsRoot; +} + +function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { + const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 }); + if (!isCommandSuccess(result)) return { sourceCommit: null, result }; + const match = /[0-9a-f]{40}/iu.exec(statusText(result)); + return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; +} + +function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { + const gitRetries = Math.max(1, spec.downloadProfile.git.retries); + const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); + return [ + `cicd_repo=${shellQuote(spec.cicdRepo)}`, + `cicd_url=${shellQuote(spec.gitUrl)}`, + `cicd_branch=${shellQuote(spec.sourceBranch)}`, + `cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`, + `git_retries=${shellQuote(String(gitRetries))}`, + `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, + "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", + "retry_git() {", + " op=$1", + " shift", + " attempt=1", + " while [ \"$attempt\" -le \"$git_retries\" ]; do", + " echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2", + " run_git \"$@\"", + " code=$?", + " if [ \"$code\" -eq 0 ]; then return 0; fi", + " echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2", + " attempt=$((attempt + 1))", + " done", + " return 1", + "}", + "mkdir -p \"$(dirname \"$cicd_repo\")\"", + "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", + " :", + "elif [ -e \"$cicd_repo\" ]; then", + " echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", + " exit 41", + "else", + " retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", + "fi", + "git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"", + "git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", + "if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then", + " rm -rf \"$cicd_repo\"", + " retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", + " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", + " retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune", + "fi", + ].join("\n"); +} + +function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record { + if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped); + if (scoped.action === "sync") return nodeRuntimeSync(scoped); + if (scoped.action === "apply") return nodeRuntimeApply(scoped); + if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped); + return nodeRuntimeUnsupportedAction(scoped); +} + +function nodeRuntimeApply(scoped: ReturnType): Record { + const spec = scoped.spec; + const head = resolveNodeRuntimeLaneHead(spec); + const sourceCommit = head.sourceCommit; + if (sourceCommit === null) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "source-head", + degradedReason: "node-runtime-source-head-unresolved", + headProbe: compactRuntimeCommand(head.result), + }; + } + const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds); + if (secrets !== null && secrets.ok !== true) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + phase: "external-postgres-secret-sync", + sourceCommit, + secrets, + degradedReason: "external-postgres-secret-sync-failed", + }; + } + const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds); + if (baseImage !== null && baseImage.ok !== true) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + phase: "base-image-seed", + sourceCommit, + secrets, + baseImage, + degradedReason: "node-runtime-base-image-seed-failed", + }; + } + const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds); + if (!isCommandSuccess(render.result)) { + return { + ok: false, + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + phase: "source-render", + sourceCommit, + secrets, + baseImage, + renderDir: render.renderDir, + renderLocation: render.location, + render: compactRuntimeCommand(render.result), + degradedReason: "node-runtime-control-plane-render-failed", + }; + } + const apply = render.location === "local" + ? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds) + : applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds); + const cleanup = render.location === "local" + ? cleanupLocalNodeRuntimeRenderDir(spec, render) + : cleanupNodeRuntimeRenderDir(spec, render.renderDir); + return { + ok: isCommandSuccess(apply), + command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-apply", + mutation: !scoped.dryRun && isCommandSuccess(apply), + sourceCommit, + expected: nodeRuntimeExpected(spec), + secrets, + baseImage, + renderDir: render.renderDir, + renderLocation: render.location, + render: compactRuntimeCommand(render.result), + apply: compactRuntimeCommand(apply), + cleanupRenderDir: compactRuntimeCommand(cleanup), + degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed", + next: scoped.dryRun + ? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` } + : { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; +} + +function nodeRuntimeRefresh(scoped: ReturnType): Record { + const spec = scoped.spec; + const script = [ + "set +e", + `app=${shellQuote(spec.app)}`, + "argo_namespace=argocd", + `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`, + "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)", + "if [ \"$dry_run\" = true ]; then", + " output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)", + "else", + " output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)", + "fi", + "code=$?", + "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)", + "printf 'app\\t%s\\n' \"$app\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'before\\t%s\\n' \"$before\"", + "printf 'after\\t%s\\n' \"$after\"", + "printf 'annotateExitCode\\t%s\\n' \"$code\"", + "printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", + "exit \"$code\"", + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const fields = keyValueLinesFromText(statusText(result)); + return { + ok: isCommandSuccess(result), + command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-refresh", + mutation: !scoped.dryRun && isCommandSuccess(result), + argoApplication: fields.app || spec.app, + before: fields.before || null, + after: fields.after || null, + annotateExitCode: numericField(fields.annotateExitCode), + result: compactRuntimeCommand(result), + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, + }; +} + +function nodeRuntimeSync(scoped: ReturnType): Record { + const spec = scoped.spec; + const operation = { + operation: { + initiatedBy: { username: "unidesk-hwlab-node-cli" }, + retry: { limit: 1 }, + sync: { + prune: true, + revision: spec.gitopsBranch, + source: { + repoURL: spec.argoRepoUrl, + targetRevision: spec.gitopsBranch, + path: spec.runtimePath, + }, + syncOptions: ["CreateNamespace=true"], + syncStrategy: { hook: {} }, + }, + }, + }; + const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64"); + const script = [ + "set +e", + `app=${shellQuote(spec.app)}`, + "argo_namespace=argocd", + `runtime_namespace=${shellQuote(spec.runtimeNamespace)}`, + `dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`, + `patch_b64=${shellQuote(patchB64)}`, + "terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)", + "patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)", + "printf '{\"operation\":null}' > \"$terminate_patch_file\"", + "printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"", + "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", + "operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)", + "terminate_code=0", + "terminate_output=", + "terminated_operation=false", + "if [ \"$operation_phase\" = Running ]; then", + " if [ \"$dry_run\" = true ]; then", + " terminated_operation=would-terminate", + " else", + " terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)", + " terminate_code=$?", + " if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi", + " sleep 2", + " fi", + "fi", + "failed_hook_count=0", + "deleted_hook_count=0", + "failed_hooks=", + "deleted_hooks=", + "hook_delete_errors=", + "for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do", + " tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)", + " failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)", + " case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac", + " case \"$failed\" in *True*) ;; *) continue ;; esac", + " hook_name=${job#job.batch/}", + " failed_hook_count=$((failed_hook_count + 1))", + " if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi", + " if [ \"$dry_run\" = false ]; then", + " delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)", + " delete_code=$?", + " if [ \"$delete_code\" -eq 0 ]; then", + " deleted_hook_count=$((deleted_hook_count + 1))", + " if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi", + " else", + " if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi", + " fi", + " fi", + "done", + "if [ \"$dry_run\" = true ]; then", + " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)", + "else", + " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)", + "fi", + "code=$?", + "after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", + "printf 'app\\t%s\\n' \"$app\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'before\\t%s\\n' \"$before\"", + "printf 'after\\t%s\\n' \"$after\"", + "printf 'operationPhase\\t%s\\n' \"$operation_phase\"", + "printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"", + "printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"", + "printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", + "printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"", + "printf 'failedHooks\\t%s\\n' \"$failed_hooks\"", + "printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"", + "printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"", + "printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"", + "printf 'patchExitCode\\t%s\\n' \"$code\"", + "printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"", + "rm -f \"$patch_file\" \"$terminate_patch_file\"", + "exit \"$code\"", + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const fields = keyValueLinesFromText(statusText(result)); + return { + ok: isCommandSuccess(result), + command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : "confirmed-sync", + mutation: !scoped.dryRun && isCommandSuccess(result), + argoApplication: fields.app || spec.app, + syncSource: { + repoURL: spec.argoRepoUrl, + targetRevision: spec.gitopsBranch, + path: spec.runtimePath, + }, + before: fields.before || null, + after: fields.after || null, + operationRecovery: { + operationPhase: fields.operationPhase || null, + terminatedOperation: fields.terminatedOperation || "false", + terminateExitCode: numericField(fields.terminateExitCode), + terminateOutput: fields.terminateOutput || null, + failedHookCount: numericField(fields.failedHookCount), + failedHooks: commaListField(fields.failedHooks), + deletedHookCount: numericField(fields.deletedHookCount), + deletedHooks: commaListField(fields.deletedHooks), + hookDeleteErrors: fields.hookDeleteErrors || null, + }, + patchExitCode: numericField(fields.patchExitCode), + patchOutput: fields.patchOutput || null, + result: compactRuntimeCommand(result), + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` }, + }; +} + +function nodeRuntimeTriggerCurrent(scoped: ReturnType): Record { + const spec = scoped.spec; + printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" }); + const head = resolveNodeRuntimeLaneHead(spec); + const sourceCommit = head.sourceCommit; + if (sourceCommit === null) { + printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" }); + return { + ok: false, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "source-head", + degradedReason: "node-runtime-source-head-unresolved", + headProbe: compactRuntimeCommand(head.result), + }; + } + const pipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit); + printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun }); + printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun }); + const before = getNodeRuntimePipelineRun(spec, pipelineRun); + printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null }); + if (scoped.dryRun) { + return { + ok: true, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "dry-run", + sourceCommit, + pipelineRun, + rerun: scoped.rerun, + before, + manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun), + next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` }, + }; + } + if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) { + return { + ok: true, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "confirmed-trigger", + sourceCommit, + pipelineRun, + before, + skipped: true, + reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running", + rerunAvailable: true, + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` }, + }; + } + printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun }); + const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false }); + if (refresh.ok !== true) { + printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "failed", sourceCommit, pipelineRun }); + return { + ok: false, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "control-plane-apply", + sourceCommit, + pipelineRun, + refresh, + degradedReason: "node-runtime-control-plane-apply-before-trigger-failed", + }; + } + printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun }); + printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun }); + const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun); + const after = getNodeRuntimePipelineRun(spec, pipelineRun); + const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True"); + const createOk = isCommandSuccess(create) || createObserved; + printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined }); + return { + ok: createOk, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "confirmed-trigger", + mutation: createOk, + sourceCommit, + pipelineRun, + rerun: scoped.rerun, + before, + refresh, + create: compactRuntimeCommand(create), + after, + createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined, + degradedReason: createOk ? undefined : "node-runtime-pipelinerun-create-failed", + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` }, + }; +} + +function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { + const spec = scoped.spec; + const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); + const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run"); + const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null; + const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null; + const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit)); + const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60); + const namespaceExists = namespace.exitCode === 0; + const postgresObjects = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60) + : null; + const localPostgresObjects = postgresObjects === null + ? [] + : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec)); + const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60); + const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60); + const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60); + const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u); + const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun); + const workloads = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60) + : null; + const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + const bridge = externalPostgresBridgeStatus(spec, namespaceExists); + const secrets = externalPostgresSecretStatus(spec, namespaceExists); + const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0; + const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready)); + const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy"; + const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True"; + return { + ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady, + command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, + mode: "node-scoped-runtime-status", + mutation: false, + node: scoped.node, + lane: scoped.lane, + sourceCommit, + pipelineRun, + expected: nodeRuntimeExpected(spec), + sourceHead: head === null + ? { ok: sourceCommit !== null, value: sourceCommit, source: "option" } + : { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) }, + controlPlane: { + ready: controlPlaneReady, + serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) }, + pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) }, + }, + argo: { + ready: argoReady, + application: spec.app, + repoURL, + expectedRepoURL: spec.argoRepoUrl, + targetRevision, + path, + syncRevision, + syncStatus, + health, + result: compactRuntimeCommand(argo), + }, + pipelineRun: pipelineRunProbe, + runtime: { + namespace: spec.runtimeNamespace, + namespaceExists, + localPostgresObjects, + localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0, + workloadNames, + workloadCount: workloadNames.length, + workloadResult: workloads === null ? null : compactRuntimeCommand(workloads), + externalPostgresBridge: bridge, + externalPostgresSecrets: secrets, + }, + probes: { + namespace: compactRuntimeCommand(namespace), + postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects), + }, + degradedReason: controlPlaneReady + ? runtimeReady + ? argoReady + ? pipelineRunReady ? undefined : "pipelinerun-not-succeeded" + : "argo-not-synced-healthy" + : namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing" + : "control-plane-not-ready", + next: { + plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, + apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, + triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, + }, + }; +} + +function nodeRuntimeRenderToken(): string { + return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-"); +} + +function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { + if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds); + return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds); +} + +function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean { + return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId; +} + +function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { + const token = nodeRuntimeRenderToken(); + const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; + const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; + const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); + const script = [ + "set -eu", + runtimeLaneCicdRepoEnsureScript(spec), + `source_commit=${shellQuote(sourceCommit)}`, + `render_dir=${shellQuote(renderDir)}`, + `worktree_dir=${shellQuote(worktreeDir)}`, + `overlay_b64=${shellQuote(overlay)}`, + "cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }", + "trap cleanup_render_worktree EXIT", + `test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`, + "rm -rf \"$render_dir\" \"$worktree_dir\"", + "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", + "git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"", + "git -C \"$worktree_dir\" checkout --detach \"$source_commit\"", + "cd \"$worktree_dir\"", + "if [ ! -d node_modules/yaml ]; then", + " if [ -f package-lock.json ]; then", + " npm ci --ignore-scripts --no-audit --prefer-offline", + " elif [ -f bun.lock ] || [ -f bun.lockb ]; then", + " bun install --frozen-lockfile --ignore-scripts", + " else", + " echo \"control-plane render cannot install dependencies: no lockfile found\" >&2", + " exit 42", + " fi", + "fi", + "node - \"$overlay_b64\" <<'NODE'", + "const fs = require('fs');", + "const YAML = require('yaml');", + "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", + "const path = 'deploy/deploy.yaml';", + "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", + "doc.nodes = doc.nodes || {};", + "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", + "doc.lanes = doc.lanes || {};", + "const lane = doc.lanes[overlay.lane] || {};", + "const downloadStack = {", + " ...(lane.envRecipe?.downloadStack || {}),", + " httpProxy: overlay.dockerProxyHttp,", + " httpsProxy: overlay.dockerProxyHttps,", + " noProxy: overlay.dockerNoProxyList,", + "};", + "doc.lanes[overlay.lane] = {", + " ...lane,", + " node: overlay.nodeId,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " namespace: overlay.runtimeNamespace,", + " endpoint: overlay.publicApiUrl,", + " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", + " artifactCatalog: overlay.catalogPath,", + " runtimePath: overlay.runtimePath,", + " imageTagMode: 'full',", + " sourceRepo: overlay.gitUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", + "};", + "fs.writeFileSync(path, YAML.stringify(doc));", + "NODE", + "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", + [ + "node scripts/run-bun.mjs \"$render_script\"", + `--lane ${shellQuote(spec.lane)}`, + `--node ${shellQuote(spec.nodeId)}`, + `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, + `--catalog-path ${shellQuote(spec.catalogPath)}`, + "--image-tag-mode full", + `--source-revision ${shellQuote(sourceCommit)}`, + `--source-repo ${shellQuote(spec.gitUrl)}`, + `--source-branch ${shellQuote(spec.sourceBranch)}`, + `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, + `--git-read-url ${shellQuote(spec.gitReadUrl)}`, + `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, + `--registry-prefix ${shellQuote(spec.registryPrefix)}`, + `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, + `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, + `--out ${shellQuote(renderDir)}`, + ].join(" "), + ...nodeRuntimePipelinePostprocessScript(), + ].join("\n"); + return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" }; +} + +function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { + const token = nodeRuntimeRenderToken(); + const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; + const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; + const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); + const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); + const script = [ + "set -eu", + `source_url=${shellQuote(spec.gitUrl)}`, + `source_branch=${shellQuote(spec.sourceBranch)}`, + `source_commit=${shellQuote(sourceCommit)}`, + `render_dir=${shellQuote(renderDir)}`, + `worktree_dir=${shellQuote(worktreeDir)}`, + `overlay_b64=${shellQuote(overlay)}`, + `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, + "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", + "rm -rf \"$render_dir\" \"$worktree_dir\"", + "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", + "echo \"phase=local-git-clone-worktree\" >&2", + "run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"", + "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"", + "cd \"$worktree_dir\"", + "if [ ! -d node_modules/yaml ]; then", + " echo \"phase=local-install-yaml\" >&2", + " npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3", + "fi", + "node - \"$overlay_b64\" <<'NODE'", + "const fs = require('fs');", + "const YAML = require('yaml');", + "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", + "const path = 'deploy/deploy.yaml';", + "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", + "doc.nodes = doc.nodes || {};", + "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", + "doc.lanes = doc.lanes || {};", + "const lane = doc.lanes[overlay.lane] || {};", + "const downloadStack = {", + " ...(lane.envRecipe?.downloadStack || {}),", + " httpProxy: overlay.dockerProxyHttp,", + " httpsProxy: overlay.dockerProxyHttps,", + " noProxy: overlay.dockerNoProxyList,", + "};", + "doc.lanes[overlay.lane] = {", + " ...lane,", + " node: overlay.nodeId,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " namespace: overlay.runtimeNamespace,", + " endpoint: overlay.publicApiUrl,", + " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", + " artifactCatalog: overlay.catalogPath,", + " runtimePath: overlay.runtimePath,", + " imageTagMode: 'full',", + " sourceRepo: overlay.gitUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", + "};", + "fs.writeFileSync(path, YAML.stringify(doc));", + "NODE", + "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", + "echo \"phase=local-gitops-render\" >&2", + [ + "node scripts/run-bun.mjs \"$render_script\"", + `--lane ${shellQuote(spec.lane)}`, + `--node ${shellQuote(spec.nodeId)}`, + `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, + `--catalog-path ${shellQuote(spec.catalogPath)}`, + "--image-tag-mode full", + `--source-revision ${shellQuote(sourceCommit)}`, + `--source-repo ${shellQuote(spec.gitUrl)}`, + `--source-branch ${shellQuote(spec.sourceBranch)}`, + `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, + `--git-read-url ${shellQuote(spec.gitReadUrl)}`, + `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, + `--registry-prefix ${shellQuote(spec.registryPrefix)}`, + `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, + `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, + `--out ${shellQuote(renderDir)}`, + ].join(" "), + ...nodeRuntimePipelinePostprocessScript(), + ].join("\n"); + return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" }; +} + +function nodeRuntimePipelinePostprocessScript(): string[] { + return [ + "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'", + "const fs = require('fs');", + "const path = require('path');", + "const renderDir = process.argv[2];", + "const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", + "const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');", + "let text = fs.readFileSync(pipelinePath, 'utf8');", + "let YAML = null;", + "try { YAML = require('yaml'); } catch {}", + "const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');", + "const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;", + "const yamlString = (value) => JSON.stringify(String(value));", + "const proxyEnv = {", + " HTTP_PROXY: overlay.proxyHttp,", + " HTTPS_PROXY: overlay.proxyHttps,", + " ALL_PROXY: overlay.proxyAll,", + " NO_PROXY: overlay.noProxy,", + " http_proxy: overlay.proxyHttp,", + " https_proxy: overlay.proxyHttps,", + " all_proxy: overlay.proxyAll,", + " no_proxy: overlay.noProxy,", + "};", + "const dockerProxyEnv = {", + " HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,", + " HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,", + " HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,", + "};", + "const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };", + "function deployYamlOverlayScript() {", + " const runtimeOverlay = JSON.stringify({", + " nodeId: overlay.nodeId,", + " lane: overlay.lane,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " gitopsRoot: overlay.gitopsRoot,", + " runtimePath: overlay.runtimePath,", + " runtimeRenderDir: overlay.runtimeRenderDir,", + " runtimeNamespace: overlay.runtimeNamespace,", + " catalogPath: overlay.catalogPath,", + " gitUrl: overlay.gitUrl,", + " publicWebUrl: overlay.publicWebUrl,", + " publicApiUrl: overlay.publicApiUrl,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " dockerProxyHttp: overlay.dockerProxyHttp,", + " dockerProxyHttps: overlay.dockerProxyHttps,", + " dockerNoProxyList: overlay.dockerNoProxyList,", + " npmRegistry: overlay.npmRegistry,", + " npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,", + " });", + " return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'", + "const fs = require('fs');", + "const YAML = require('yaml');", + "const overlay = ${runtimeOverlay};", + "const file = 'deploy/deploy.yaml';", + "if (!fs.existsSync(file)) {", + " console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));", + " process.exit(45);", + "}", + "const doc = YAML.parse(fs.readFileSync(file, 'utf8'));", + "doc.nodes = doc.nodes || {};", + "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", + "doc.lanes = doc.lanes || {};", + "const lane = doc.lanes[overlay.lane] || {};", + "const envRecipe = lane.envRecipe || {};", + "const downloadStack = {", + " ...(envRecipe.downloadStack || {}),", + " httpProxy: overlay.dockerProxyHttp,", + " httpsProxy: overlay.dockerProxyHttps,", + " noProxy: overlay.dockerNoProxyList,", + "};", + "if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;", + "if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;", + "doc.lanes[overlay.lane] = {", + " ...lane,", + " node: overlay.nodeId,", + " sourceBranch: overlay.sourceBranch,", + " gitopsBranch: overlay.gitopsBranch,", + " namespace: overlay.runtimeNamespace,", + " endpoint: overlay.publicApiUrl,", + " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", + " artifactCatalog: overlay.catalogPath,", + " runtimePath: overlay.runtimePath,", + " imageTagMode: 'full',", + " sourceRepo: overlay.gitUrl,", + " externalPostgres: overlay.externalPostgres,", + " envRecipe: { ...envRecipe, downloadStack },", + "};", + "fs.writeFileSync(file, YAML.stringify(doc));", + "console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));", + "NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;", + "}", + "function runtimePathOverlayScript() {", + " const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;", + " const targetPath = String(overlay.runtimePath || '');", + " if (!targetPath || sourcePath === targetPath) return '';", + " return [", + " `if [ ! -d ${shellSingle(targetPath)} ]; then`,", + " ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,", + " ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,", + " ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,", + " ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,", + " `fi`,", + " ].join('\\n');", + "}", + "function stepEnvBootstrapScript() {", + " const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);", + " if (entries.length === 0) return '';", + " const lines = ['# unidesk-step-env-bootstrap'];", + " for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);", + " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');", + " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');", + " return lines.join('\\n');", + "}", + "function runtimeGitopsPostprocessScript() {", + " const runtimeOverlay = JSON.stringify({", + " gitopsRoot: overlay.gitopsRoot,", + " runtimePath: overlay.runtimePath,", + " runtimeRenderDir: overlay.runtimeRenderDir,", + " runtimeNamespace: overlay.runtimeNamespace,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " });", + " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'", + "const fs = require('fs');", + "const path = require('path');", + "const YAML = require('yaml');", + "const overlay = ${runtimeOverlay};", + "const runtimePath = String(overlay.runtimePath || '');", + "const renderDir = String(overlay.runtimeRenderDir || '');", + "const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';", + "const candidates = [...new Set([", + " runtimePath,", + " renderDir,", + " overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',", + " legacyRuntimePath,", + "].filter(Boolean))];", + "const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;", + "if (!runtimePath || !fs.existsSync(sourcePath)) {", + " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));", + " process.exit(47);", + "}", + "if (sourcePath !== runtimePath) {", + " fs.rmSync(runtimePath, { recursive: true, force: true });", + " fs.mkdirSync(path.dirname(runtimePath), { recursive: true });", + " fs.cpSync(sourcePath, runtimePath, { recursive: true });", + " fs.rmSync(sourcePath, { recursive: true, force: true });", + "}", + "for (const candidate of candidates) {", + " if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });", + "}", + "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", + "function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }", + "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", + "function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }", + "function patchKustomization() {", + " const file = path.join(runtimePath, 'kustomization.yaml');", + " if (!fs.existsSync(file)) return false;", + " const doc = readYaml(file) || {};", + " const resources = Array.isArray(doc.resources) ? doc.resources : [];", + " const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));", + " if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); return true; }", + " return false;", + "}", + "function patchExternalPostgres() {", + " const pg = overlay.externalPostgres;", + " if (!pg || !pg.serviceName) return false;", + " const file = path.join(runtimePath, 'external-postgres.yaml');", + " if (!fs.existsSync(file)) return false;", + " const doc = readYaml(file);", + " const items = listItems(doc).filter(Boolean);", + " let changed = false;", + " const endpointSliceName = String(pg.serviceName) + '-host';", + " for (const item of items) {", + " if (!item || typeof item !== 'object') continue;", + " item.metadata = item.metadata || {};", + " item.metadata.namespace = overlay.runtimeNamespace;", + " item.metadata.labels = item.metadata.labels || {};", + " item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;", + " if (item.kind === 'Service') {", + " item.metadata.name = pg.serviceName;", + " item.spec = item.spec || {};", + " item.spec.ports = [{ name: 'postgres', port: pg.port, targetPort: pg.port, protocol: 'TCP' }];", + " delete item.spec.selector;", + " changed = true;", + " }", + " if (item.kind === 'EndpointSlice') {", + " item.metadata.name = endpointSliceName;", + " item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;", + " item.addressType = 'IPv4';", + " item.ports = [{ name: 'postgres', port: pg.port, protocol: 'TCP' }];", + " item.endpoints = [{ addresses: [pg.endpointAddress], conditions: { ready: true } }];", + " changed = true;", + " }", + " }", + " if (changed) writeYaml(file, normalizeList(items));", + " return changed;", + "}", + "const kustomizationChanged = patchKustomization();", + "const externalPostgresChanged = patchExternalPostgres();", + "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, kustomizationChanged, externalPostgresChanged }));", + "NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;", + "}", + "function runtimeGitopsVerifyScript() {", + " const runtimeOverlay = JSON.stringify({", + " runtimePath: overlay.runtimePath,", + " runtimeNamespace: overlay.runtimeNamespace,", + " externalPostgres: overlay.externalPostgres,", + " observability: overlay.observability,", + " });", + " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'", + "const fs = require('fs');", + "const path = require('path');", + "const YAML = require('yaml');", + "const overlay = ${runtimeOverlay};", + "const runtimePath = String(overlay.runtimePath || '');", + "function fail(reason, extra = {}) {", + " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));", + " process.exit(48);", + "}", + "if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');", + "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", + "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", + "const checks = [];", + "const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');", + "if (overlay.observability && overlay.observability.prometheusOperator === false) {", + " if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');", + " const resources = readYaml(kustomizationPath).resources || [];", + " if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });", + " checks.push('observability-disabled');", + "}", + "const pg = overlay.externalPostgres;", + "if (pg && pg.serviceName) {", + " const file = path.join(runtimePath, 'external-postgres.yaml');", + " if (!fs.existsSync(file)) fail('external-postgres-missing');", + " const items = listItems(readYaml(file)).filter(Boolean);", + " const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);", + " const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');", + " if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });", + " if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });", + " const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;", + " const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;", + " const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;", + " if (Number(servicePort) !== Number(pg.port) || Number(endpointPort) !== Number(pg.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: pg.port });", + " if (String(endpointAddress) !== String(pg.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: pg.endpointAddress });", + " checks.push('external-postgres-bridge');", + "}", + "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));", + "NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;", + "}", + "function patchScript(script) {", + " let result = String(script || '');", + " const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';", + " if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", + " result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);", + " }", + " const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';", + " if (result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", + " result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);", + " }", + " result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);", + " result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", + " result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", + " result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", + " result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", + " result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);", + " result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);", + " const legacyRuntimePath = (() => {", + " const runtimePath = String(overlay.runtimePath || '');", + " const parts = runtimePath.split('/').filter(Boolean);", + " if (parts.length < 2) return '';", + " const leaf = parts.at(-1);", + " const parent = parts.slice(0, -2).join('/');", + " return parent && leaf ? `${parent}/${leaf}` : '';", + " })();", + " if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {", + " result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);", + " result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');", + " result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');", + " }", + " const bootstrap = stepEnvBootstrapScript();", + " if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {", + " result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);", + " }", + " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {", + " let next = match;", + " if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", + " if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", + " return next;", + " });", + " result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {", + " if (match.includes('--check')) return runtimeGitopsVerifyScript();", + " return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;", + " });", + " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", + " return result;", + "}", + "function patchManifestObject(doc) {", + " if (!doc || typeof doc !== 'object') return false;", + " if (doc.kind !== 'Pipeline' || !doc.spec) return false;", + " const defaults = {", + " 'git-url': overlay.gitUrl,", + " 'git-read-url': overlay.gitReadUrl,", + " 'git-write-url': overlay.gitWriteUrl,", + " 'catalog-path': overlay.catalogPath,", + " 'runtime-path': overlay.runtimePath,", + " 'registry-prefix': overlay.registryPrefix,", + " };", + " for (const param of doc.spec?.params || []) {", + " if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];", + " }", + " doc.metadata = doc.metadata || {};", + " doc.metadata.annotations = doc.metadata.annotations || {};", + " doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;", + " doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;", + " for (const task of doc.spec?.tasks || []) {", + " for (const sidecar of task.taskSpec?.sidecars || []) {", + " if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;", + " }", + " for (const step of task.taskSpec?.steps || []) {", + " if (Array.isArray(step.env)) {", + " for (const env of step.env) {", + " if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];", + " }", + " }", + " step.env = Array.isArray(step.env) ? step.env : [];", + " const existingEnv = new Set(step.env.map((env) => env.name));", + " for (const [name, value] of Object.entries(stepEnv)) {", + " if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });", + " }", + " if (typeof step.script === 'string') step.script = patchScript(step.script);", + " }", + " }", + " return true;", + "}", + "function patchStructuredPipeline() {", + " try {", + " const doc = JSON.parse(text);", + " if (!patchManifestObject(doc)) return false;", + " text = JSON.stringify(doc, null, 2) + '\\n';", + " return true;", + " } catch {}", + " if (YAML) {", + " try {", + " const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);", + " const changed = docs.some((doc) => patchManifestObject(doc));", + " if (!changed) return false;", + " text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';", + " return true;", + " } catch {}", + " }", + " return false;", + "}", + "const structured = patchStructuredPipeline();", + "function replaceParamDefault(name, value) {", + " const namePattern = escapeRegExp(name);", + " text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);", + " text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);", + "}", + "replaceParamDefault('git-url', overlay.gitUrl);", + "replaceParamDefault('git-read-url', overlay.gitReadUrl);", + "replaceParamDefault('git-write-url', overlay.gitWriteUrl);", + "replaceParamDefault('catalog-path', overlay.catalogPath);", + "replaceParamDefault('runtime-path', overlay.runtimePath);", + "replaceParamDefault('registry-prefix', overlay.registryPrefix);", + "text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);", + "text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);", + "if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);", + "text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", + "text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", + "text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", + "text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", + "for (const [name, value] of Object.entries(stepEnv)) {", + " if (value === undefined) continue;", + " text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);", + " text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);", + "}", + "const bootstrap = stepEnvBootstrapScript();", + "if (bootstrap) {", + " text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {", + " const previous = fullText.slice(Math.max(0, offset - 500), offset);", + " if (previous.includes('unidesk-step-env-bootstrap')) return match;", + " return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;", + " });", + "}", + "if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {", + " text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);", + "}", + "const quotedRoot = JSON.stringify(overlay.gitopsRoot);", + "const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');", + "const replacements = [", + " ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],", + " ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],", + "];", + "let changed = false;", + "for (const [needle, replacement] of replacements) {", + " if (!text.includes(needle)) continue;", + " text = text.split(needle).join(replacement);", + " changed = true;", + "}", + "if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }", + "fs.writeFileSync(pipelinePath, text);", + "function patchArgoYaml(filePath) {", + " if (!YAML || !fs.existsSync(filePath)) return;", + " const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", + " let changed = false;", + " for (const doc of docs) {", + " if (!doc || typeof doc !== 'object') continue;", + " if (doc.kind === 'AppProject') {", + " doc.spec = doc.spec || {};", + " doc.spec.sourceRepos = [overlay.argoRepoUrl];", + " changed = true;", + " }", + " if (doc.kind === 'Application') {", + " doc.spec = doc.spec || {};", + " doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };", + " changed = true;", + " }", + " }", + " if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", + "}", + "patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));", + "patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));", + "NODE", + ]; +} + +function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { + const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http); + return { + nodeId: spec.nodeId, + lane: spec.lane, + sourceBranch: spec.sourceBranch, + gitopsBranch: spec.gitopsBranch, + gitopsRoot: nodeRuntimeGitopsRoot(spec), + runtimePath: spec.runtimePath, + runtimeRenderDir: spec.runtimeRenderDir, + runtimeNamespace: spec.runtimeNamespace, + catalogPath: spec.catalogPath, + tektonDir: spec.tektonDir, + argoApplicationFile: spec.argoApplicationFile, + argoRepoUrl: spec.argoRepoUrl, + gitUrl: spec.gitUrl, + gitReadUrl: spec.gitReadUrl, + gitWriteUrl: spec.gitWriteUrl, + networkProfileId: spec.networkProfileId, + downloadProfileId: spec.downloadProfileId, + gitSshProxyHost: gitSshProxy?.host, + gitSshProxyPort: gitSshProxy?.port, + proxyHttp: spec.networkProfile.proxy.http, + proxyHttps: spec.networkProfile.proxy.https, + proxyAll: spec.networkProfile.proxy.all, + noProxy: spec.networkProfile.proxy.noProxy.join(","), + dockerProxyHttp: spec.networkProfile.dockerBuildProxy.http, + dockerProxyHttps: spec.networkProfile.dockerBuildProxy.https, + dockerProxyAll: spec.networkProfile.dockerBuildProxy.all, + dockerNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","), + dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy, + npmRegistry: spec.downloadProfile.npm.registry, + npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000, + stepEnv: spec.stepEnv, + observability: spec.observability, + registryPrefix: spec.registryPrefix, + buildkitSidecarImage: spec.buildkit?.sidecarImage, + publicWebUrl: spec.publicWebUrl, + publicApiUrl: spec.publicApiUrl, + externalPostgres: spec.externalPostgres === undefined ? undefined : { + enabled: true, + serviceName: spec.externalPostgres.serviceName, + endpointAddress: spec.externalPostgres.endpointAddress, + port: spec.externalPostgres.port, + }, + }; +} + +function httpProxyEndpoint(value: string): { host: string; port: number } | null { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; + const port = parsed.port === "" ? parsed.protocol === "https:" ? 443 : 80 : Number.parseInt(parsed.port, 10); + if (!Number.isInteger(port) || port <= 0) return null; + return { host: parsed.hostname, port }; +} + + +function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] { + return [ + `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, + `${renderDir}/${spec.tektonDir}/rbac.yaml`, + `${renderDir}/${spec.tektonDir}/pipeline.yaml`, + `${renderDir}/argocd/project.yaml`, + `${renderDir}/argocd/${spec.argoApplicationFile}`, + ]; +} + +function applyNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { + const files = nodeRuntimeControlPlaneFiles(spec, renderDir); + const args = [ + "kubectl", + "apply", + ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), + ...files.flatMap((file) => ["-f", file]), + ]; + return runNodeK3sArgs(spec, args, timeoutSeconds); +} + +function applyLocalNodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandResult { + const manifest = nodeRuntimeControlPlaneFiles(spec, renderDir) + .map((file) => `---\n# Source: ${file}\n${readFileSync(file, "utf8").trimEnd()}\n`) + .join(""); + const args = [ + "kubectl", + "apply", + ...(dryRun ? ["--dry-run=client"] : ["--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`]), + "-f", + "-", + ]; + return runNodeK3sScript(spec, args.map(shellQuote).join(" "), timeoutSeconds, manifest); +} + +function cleanupNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandResult { + const prefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; + const script = [ + "set -eu", + `render_dir=${shellQuote(renderDir)}`, + `prefix=${shellQuote(prefix)}`, + "case \"$render_dir\" in", + " \"$prefix\"*) rm -rf \"$render_dir\" ;;", + " *) echo \"refusing to remove unexpected render dir: $render_dir\" >&2; exit 44 ;;", + "esac", + ].join("\n"); + return runNodeHostScript(spec, script, 60); +} + +function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, render: NodeRuntimeRenderResult): CommandResult { + const renderPrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-`; + const worktreePrefix = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-`; + if (!render.renderDir.startsWith(renderPrefix) || !render.worktreeDir.startsWith(worktreePrefix)) { + return { + command: ["rm", "-rf", ""], + cwd: repoRoot, + exitCode: 44, + stdout: "", + stderr: `refusing to remove unexpected local render paths: ${render.renderDir} ${render.worktreeDir}`, + signal: null, + timedOut: false, + }; + } + return runCommand(["rm", "-rf", render.renderDir, render.worktreeDir], repoRoot, { timeoutMs: 60_000 }); +} + +function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record { + return { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + name: pipelineRun, + namespace: "hwlab-ci", + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": spec.lane, + "hwlab.pikastech.local/source-commit": sourceCommit, + "hwlab.pikastech.local/trigger": "unidesk-node-cli", + }, + annotations: { + "hwlab.pikastech.local/node": spec.nodeId, + "hwlab.pikastech.local/source-branch": spec.sourceBranch, + "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, + "hwlab.pikastech.local/runtime-path": spec.runtimePath, + "hwlab.pikastech.local/network-profile": spec.networkProfileId, + "hwlab.pikastech.local/download-profile": spec.downloadProfileId, + }, + }, + spec: { + pipelineRef: { name: spec.pipeline }, + taskRunTemplate: { + serviceAccountName: spec.serviceAccountName, + podTemplate: { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 }, + }, + }, + params: [ + { name: "git-url", value: spec.gitUrl }, + { name: "git-read-url", value: spec.gitReadUrl }, + { name: "git-write-url", value: spec.gitWriteUrl }, + { name: "source-branch", value: spec.sourceBranch }, + { name: "gitops-branch", value: spec.gitopsBranch }, + { name: "lane", value: spec.lane }, + { name: "catalog-path", value: spec.catalogPath }, + { name: "image-tag-mode", value: "full" }, + { name: "runtime-path", value: spec.runtimePath }, + { name: "revision", value: sourceCommit }, + { name: "registry-prefix", value: spec.registryPrefix }, + { name: "services", value: spec.serviceIds.join(",") }, + { name: "base-image", value: spec.baseImage }, + ], + workspaces: [ + { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, + { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, + ], + }, + }; +} + +function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult { + const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64"); + const script = [ + "set -eu", + `manifest_b64=${shellQuote(manifestB64)}`, + `pipeline_run=${shellQuote(pipelineRun)}`, + `rerun=${rerun ? "true" : "false"}`, + "manifest_path=\"/tmp/$pipeline_run.json\"", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", + "existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", + "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then", + " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + "elif [ \"$existing_status\" = False ]; then", + " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + "fi", + "if kubectl create -f \"$manifest_path\"; then", + " :", + "else", + " code=$?", + " if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then", + " printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2", + " else", + " exit \"$code\"", + " fi", + "fi", + "kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", + ].join("\n"); + return runNodeK3sScript(spec, script, timeoutSeconds); +} + +function printNodeRuntimeTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { + process.stderr.write(`${JSON.stringify({ event: "hwlab.runtime-lane.trigger.progress", at: new Date().toISOString(), lane: spec.lane, node: spec.nodeId, ...data })}\n`); +} + +function getNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record { + const result = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipelinerun", pipelineRun, "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}{.metadata.creationTimestamp}{\"\\n\"}"], 60); + if (result.exitCode !== 0) return { exists: false, name: pipelineRun, result: compactRuntimeCommand(result) }; + const [status = "", reason = "", message = "", createdAt = ""] = result.stdout.split(/\r?\n/u); + return { + exists: true, + name: pipelineRun, + status: status || null, + reason: reason || null, + message: message || null, + createdAt: createdAt || null, + result: compactRuntimeCommand(result), + }; +} + +function syncNodeExternalPostgresSecrets(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { + const pg = spec.externalPostgres; + if (pg === undefined) return null; + const secretRoot = externalPostgresSecretSourceRoot(spec); + const cloudApi = readSecretSourceValue(secretRoot, pg.cloudApi.sourceRef, pg.cloudApi.envKey); + const openfga = readSecretSourceValue(secretRoot, pg.openfga.sourceRef, pg.openfga.envKey); + const missing = [ + ...(cloudApi.ok ? [] : [`${pg.cloudApi.sourceRef}:${pg.cloudApi.envKey}`]), + ...(openfga.ok ? [] : [`${pg.openfga.sourceRef}:${pg.openfga.envKey}`]), + ]; + if (missing.length > 0) { + if (dryRun) { + return { + ok: true, + mode: "dry-run", + mutation: false, + skipped: true, + reason: "external-postgres-secret-source-missing", + missing, + sourceRoot: secretRoot, + valuesPrinted: false, + next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, + }; + } + return { + ok: false, + mode: dryRun ? "dry-run" : "confirmed-secret-sync", + mutation: false, + degradedReason: "external-postgres-secret-source-missing", + missing, + sourceRoot: secretRoot, + valuesPrinted: false, + next: { platformDbApply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configRef} --confirm` }, + }; + } + const setup = runNodeK3sScript(spec, externalPostgresSecretSetupScript(spec, dryRun), timeoutSeconds); + if (!isCommandSuccess(setup)) { + return { + ok: false, + mode: dryRun ? "dry-run" : "confirmed-secret-sync", + mutation: false, + phase: "namespace-authn-setup", + setup: compactRuntimeCommand(setup), + valuesPrinted: false, + }; + } + const manifest = { + apiVersion: "v1", + kind: "List", + items: [ + { + apiVersion: "v1", + kind: "Secret", + metadata: { name: pg.cloudApi.secretName, namespace: spec.runtimeNamespace }, + type: "Opaque", + stringData: { [pg.cloudApi.secretKey]: cloudApi.value }, + }, + { + apiVersion: "v1", + kind: "Secret", + metadata: { name: pg.openfga.secretName, namespace: spec.runtimeNamespace }, + type: "Opaque", + stringData: { [pg.openfga.secretKey]: openfga.value }, + }, + ], + }; + const applyScript = [ + "set -eu", + [ + "kubectl", + "apply", + "--server-side", + "--force-conflicts", + "--field-manager=unidesk-hwlab-node-external-postgres-secret", + ...(dryRun ? ["--dry-run=server"] : []), + "-f", + "-", + ].map(shellQuote).join(" "), + ].join("\n"); + const apply = runNodeK3sScript(spec, applyScript, timeoutSeconds, JSON.stringify(manifest)); + return { + ok: isCommandSuccess(apply), + mode: dryRun ? "dry-run" : "confirmed-secret-sync", + mutation: !dryRun && isCommandSuccess(apply), + namespace: spec.runtimeNamespace, + sourceRoot: secretRoot, + secrets: [ + { name: pg.cloudApi.secretName, key: pg.cloudApi.secretKey, sourceRef: pg.cloudApi.sourceRef, envKey: pg.cloudApi.envKey }, + { name: pg.openfga.secretName, key: pg.openfga.secretKey, sourceRef: pg.openfga.sourceRef, envKey: pg.openfga.envKey, authnKey: pg.openfga.authnKey ?? null }, + ], + setup: compactRuntimeCommand(setup), + apply: compactRuntimeCommand(apply), + valuesPrinted: false, + }; +} + +function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, timeoutSeconds: number): Record | null { + if (spec.baseImageSource === undefined) return null; + const script = [ + "set -eu", + `target=${shellQuote(spec.baseImage)}`, + `source=${shellQuote(spec.baseImageSource)}`, + `dry_run=${shellQuote(dryRun ? "true" : "false")}`, + "repo_tag=${target#*/}", + "repo=${repo_tag%:*}", + "tag=${repo_tag##*:}", + "if [ \"$repo\" = \"$repo_tag\" ]; then tag=latest; fi", + "registry_url=\"http://127.0.0.1:5000/v2/$repo/tags/list\"", + "present=false", + "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then present=true; fi", + "action=none", + "if [ \"$present\" = false ]; then", + " action=seed", + " if [ \"$dry_run\" = false ]; then", + " docker image inspect \"$source\" >/dev/null", + " docker tag \"$source\" \"$target\"", + " docker push \"$target\" >/tmp/hwlab-node-base-image-push.out", + " fi", + "fi", + "after=false", + "if curl -fsS \"$registry_url\" 2>/dev/null | grep -q '\"'\"$tag\"'\"'; then after=true; fi", + "printf 'target\\t%s\\n' \"$target\"", + "printf 'source\\t%s\\n' \"$source\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'presentBefore\\t%s\\n' \"$present\"", + "printf 'presentAfter\\t%s\\n' \"$after\"", + "printf 'action\\t%s\\n' \"$action\"", + "if [ \"$dry_run\" = true ] || [ \"$after\" = true ]; then exit 0; fi", + "exit 1", + ].join("\n"); + const result = runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); + const fields = keyValueLinesFromText(statusText(result)); + return { + ok: isCommandSuccess(result), + dryRun, + target: fields.target ?? spec.baseImage, + source: fields.source ?? spec.baseImageSource, + presentBefore: fields.presentBefore === "true", + presentAfter: fields.presentAfter === "true", + action: fields.action ?? null, + result: compactRuntimeCommand(result), + }; +} + +function externalPostgresSecretSetupScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { + const pg = spec.externalPostgres; + if (pg === undefined) return "true"; + const authnKey = pg.openfga.authnKey ?? "authn-preshared-key"; + return [ + "set -eu", + `namespace=${shellQuote(spec.runtimeNamespace)}`, + `secret=${shellQuote(pg.openfga.secretName)}`, + `authn_key=${shellQuote(authnKey)}`, + `dry_run=${shellQuote(dryRun ? "true" : "false")}`, + "namespace_apply_args=\"--server-side --field-manager=unidesk-hwlab-node-runtime-namespace\"", + "if [ \"$dry_run\" = true ]; then namespace_apply_args=\"$namespace_apply_args --dry-run=server\"; fi", + "kubectl create namespace \"$namespace\" --dry-run=client -o yaml | kubectl apply $namespace_apply_args -f -", + "authn_present=$(kubectl -n \"$namespace\" get secret \"$secret\" -o \"go-template={{ index .data \\\"$authn_key\\\" }}\" 2>/dev/null | wc -c | tr -d ' ')", + "if [ \"${authn_present:-0}\" -gt 0 ]; then", + " action=kept", + "elif [ \"$dry_run\" = true ]; then", + " action=would-generate-authn-key", + "else", + " if command -v openssl >/dev/null 2>&1; then authn_value=$(openssl rand -base64 48); else authn_value=$(head -c 48 /dev/urandom | base64); fi", + " kubectl -n \"$namespace\" create secret generic \"$secret\" --from-literal=\"$authn_key=$authn_value\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=unidesk-hwlab-node-openfga-authn -f - >/dev/null", + " authn_value=", + " action=generated-authn-key", + "fi", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$secret\"", + "printf 'authnKey\\t%s\\n' \"$authn_key\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'valuesPrinted\\tfalse\\n'", + ].join("\n"); +} + +function externalPostgresSecretSourceRoot(spec: HwlabRuntimeLaneSpec): string { + const configRef = spec.externalPostgres?.configRef; + if (configRef === undefined) return join(repoRoot, ".state", "secrets"); + const configPath = rootPath(configRef); + const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error(`${configRef} must be a YAML object`); + const secrets = (parsed as Record).secrets; + if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) throw new Error(`${configRef}.secrets must be an object`); + const root = (secrets as Record).root; + if (typeof root !== "string" || root.length === 0) throw new Error(`${configRef}.secrets.root must be a non-empty string`); + return root.startsWith("/") ? root : rootPath(root); +} + +function readSecretSourceValue(secretRoot: string, sourceRef: string, key: string): { ok: true; value: string; sourcePath: string } | { ok: false; sourcePath: string } { + const sourcePath = join(secretRoot, sourceRef); + if (!existsSync(sourcePath)) return { ok: false, sourcePath }; + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const value = values[key]; + if (value === undefined || value.length === 0) return { ok: false, sourcePath }; + return { ok: true, value, sourcePath }; +} + +function parseEnvFile(text: string): Record { + const values: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + const rawValue = line.slice(eq + 1).trim(); + values[key] = rawValue.startsWith("'") && rawValue.endsWith("'") + ? rawValue.slice(1, -1).replace(/'"'"'/gu, "'") + : rawValue.startsWith("\"") && rawValue.endsWith("\"") + ? rawValue.slice(1, -1) + : rawValue; + } + return values; +} + +function isLocalPostgresObject(name: string, spec: HwlabRuntimeLaneSpec): boolean { + if (!/postgres|pgsql|pgdata/iu.test(name)) return false; + const externalServiceName = spec.externalPostgres?.serviceName; + if (externalServiceName !== undefined && (name === `service/${externalServiceName}` || name === `svc/${externalServiceName}`)) return false; + return true; +} + +function externalPostgresBridgeStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { + const pg = spec.externalPostgres; + if (pg === undefined) return { required: false, ready: true }; + if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; + const service = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "service", pg.serviceName, "-o", "name"], 60); + const endpointSlice = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "endpointslice", `${pg.serviceName}-host`, "-o", "jsonpath={.endpoints[0].addresses[0]}{\"\\n\"}{.ports[0].port}{\"\\n\"}"], 60); + const [address = "", port = ""] = endpointSlice.stdout.split(/\r?\n/u); + return { + required: true, + ready: service.exitCode === 0 && endpointSlice.exitCode === 0 && address === pg.endpointAddress && Number(port) === pg.port, + serviceName: pg.serviceName, + endpointAddress: address || null, + expectedEndpointAddress: pg.endpointAddress, + port: numericField(port), + expectedPort: pg.port, + service: compactRuntimeCommand(service), + endpointSlice: compactRuntimeCommand(endpointSlice), + }; +} + +function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExists: boolean): Record { + const pg = spec.externalPostgres; + if (pg === undefined) return { required: false, ready: true }; + if (!namespaceExists) return { required: true, ready: false, degradedReason: "runtime-namespace-missing" }; + const cloudApi = secretKeyStatus(spec, pg.cloudApi.secretName, pg.cloudApi.secretKey); + const openfgaUri = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.secretKey); + const openfgaAuthn = secretKeyStatus(spec, pg.openfga.secretName, pg.openfga.authnKey ?? "authn-preshared-key"); + return { + required: true, + ready: cloudApi.present && openfgaUri.present && openfgaAuthn.present, + valuesPrinted: false, + cloudApi, + openfga: { + datastoreUri: openfgaUri, + authnPresharedKey: openfgaAuthn, + }, + }; +} + +function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record { + const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60); + return { + secret, + key, + present: result.exitCode === 0 && result.stdout.trim().length > 0, + valueBytesBase64: result.stdout.trim().length, + exitCode: result.exitCode, + stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 500), + valuesPrinted: false, }; } @@ -460,11 +2425,11 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): } function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { - return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); + return runCommand([transPath(), `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { - return runCommand(["/root/.local/bin/trans", node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); + return runCommand([transPath(), node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { @@ -2186,6 +4151,11 @@ function numericField(value: string | undefined): number | null { return Number.isFinite(parsed) ? parsed : null; } +function commaListField(value: string | undefined): string[] { + if (!value) return []; + return value.split(",").map((item) => item.trim()).filter(Boolean); +} + function splitWhitespaceField(value: string | undefined): string[] { if (!value) return []; return value.split(/\s+/u).filter(Boolean); diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index d916cf93..464a35af 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -418,6 +418,8 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri const stageStatus = stringField(lastEvent.status); const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); + const node = stringField(lastEvent.node) ?? commandOption(job.command, "--node") ?? "G14"; + const lane = stringField(lastEvent.lane) ?? commandOption(job.command, "--lane") ?? "v03"; const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail) ? true : stage === "create-pipelinerun" && stageStatus === "failed" @@ -465,13 +467,19 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri slow ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand: pipelineRun - ? `bun scripts/cli.ts hwlab nodes control-plane status --node ${stringField(lastEvent.node) ?? "G14"} --lane ${stringField(lastEvent.lane) ?? "v03"} --pipeline-run ${pipelineRun}` + ? `bun scripts/cli.ts hwlab nodes control-plane status --node ${node} --lane ${lane} --pipeline-run ${pipelineRun}` : job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; } +function commandOption(command: readonly string[], name: string): string | null { + const index = command.indexOf(name); + const value = index >= 0 ? command[index + 1] : undefined; + return typeof value === "string" && value.trim().length > 0 && !value.startsWith("--") ? value.trim() : null; +} + function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary { const nowMs = Date.now(); const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000); diff --git a/scripts/src/platform-db.ts b/scripts/src/platform-db.ts index bf7a8a30..1fed2658 100644 --- a/scripts/src/platform-db.ts +++ b/scripts/src/platform-db.ts @@ -242,6 +242,8 @@ interface RemoteFacts { logDirExists: boolean; roleExists: boolean; databaseExists: boolean; + roleExistsByName?: Record; + databaseExistsByName?: Record; serverVersion: string | null; sslOn: boolean; sslCertFile: string | null; @@ -383,12 +385,14 @@ async function status(config: UniDeskConfig, options: PlatformDbOptions): Promis const facts = remote.parsed; const controllerConnection = controllerConnectionProbe(pg, secrets); const endpointHealthy = controllerConnection.ok === true && controllerConnection.ssl === true; + const rolesReady = facts !== null && pg.objects.roles.every((role, index) => facts.postgres.roleExistsByName?.[role.name] ?? (index === 0 ? facts.postgres.roleExists : false)); + const databasesReady = facts !== null && pg.objects.databases.every((database, index) => facts.postgres.databaseExistsByName?.[database.name] ?? (index === 0 ? facts.postgres.databaseExists : false)); const deploymentHealthy = facts !== null && facts.postgres.packageInstalled && facts.postgres.serviceActive && facts.postgres.sslOn - && facts.postgres.roleExists - && facts.postgres.databaseExists + && rolesReady + && databasesReady && facts.network.port5432Listening && facts.postgres.appConnectionOk === true && facts.postgres.appConnectionSsl === true; @@ -421,6 +425,10 @@ async function status(config: UniDeskConfig, options: PlatformDbOptions): Promis dnsAddresses: facts.network.dns.addresses, roleExists: facts.postgres.roleExists, databaseExists: facts.postgres.databaseExists, + roleExistsByName: facts.postgres.roleExistsByName ?? {}, + databaseExistsByName: facts.postgres.databaseExistsByName ?? {}, + rolesReady, + databasesReady, appConnectionOk: facts.postgres.appConnectionOk, appConnectionSsl: facts.postgres.appConnectionSsl, appConnectionHost: facts.postgres.appConnectionHost, @@ -548,7 +556,7 @@ function readPostgresHostConfig(pathArg: string): PostgresHostConfig { const entries = arrayOfRecords(secrets.entries, `${configPath}.secrets.entries`).map((entry, index) => secretEntry(entry, `${configPath}.secrets.entries[${index}]`)); const roles = arrayOfRecords(objects.roles, `${configPath}.objects.roles`).map((role, index) => roleConfig(role, `${configPath}.objects.roles[${index}]`)); const databases = arrayOfRecords(objects.databases, `${configPath}.objects.databases`).map((database, index) => databaseConfig(database, `${configPath}.objects.databases[${index}]`)); - if (roles.length !== 1 || databases.length !== 1) throw new Error(`${configPath}.objects must declare exactly one role and one database for the first PK01 Sub2API rollout`); + if (roles.length === 0 || databases.length === 0) throw new Error(`${configPath}.objects.roles/databases must each contain at least one item`); const connectionStrings = arrayOfRecords(exportsConfig.connectionStrings, `${configPath}.exports.connectionStrings`).map((item, index) => connectionStringExport(item, `${configPath}.exports.connectionStrings[${index}]`)); return { configPath: relativeConfigPath(configPath), @@ -849,7 +857,7 @@ function connectionStringExport(item: Record, path: string): Co consumers: arrayOfRecords(item.consumers, `${path}.consumers`).map((consumer, index) => ({ scope: stringField(consumer, "scope", `${path}.consumers[${index}]`), secret: stringField(consumer, "secret", `${path}.consumers[${index}]`), - key: envKey(stringField(consumer, "key", `${path}.consumers[${index}]`), `${path}.consumers[${index}].key`), + key: kubernetesSecretKey(stringField(consumer, "key", `${path}.consumers[${index}]`), `${path}.consumers[${index}].key`), })), }; } @@ -880,6 +888,11 @@ function envKey(value: string, path: string): string { return value; } +function kubernetesSecretKey(value: string, path: string): string { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`); + return value; +} + function configSummary(pg: PostgresHostConfig): Record { return { path: pg.configPath, @@ -994,8 +1007,8 @@ function validateSub2ApiSecretConsistency(pg: PostgresHostConfig, inspection: Se const database = pg.objects.databases[0]; const material = inspection.materials.get(role.passwordRef.sourceRef); if (material === undefined) throw new Error(`secret source not found for role passwordRef: ${role.passwordRef.sourceRef}`); - if (material.values.SUB2API_DB_USER !== role.name) throw new Error("SUB2API_DB_USER must match YAML objects.roles[0].name"); - if (material.values.SUB2API_DB_NAME !== database.name) throw new Error("SUB2API_DB_NAME must match YAML objects.databases[0].name"); + if (material.values.SUB2API_DB_USER !== undefined && material.values.SUB2API_DB_USER !== role.name) throw new Error("SUB2API_DB_USER must match YAML objects.roles[0].name"); + if (material.values.SUB2API_DB_NAME !== undefined && material.values.SUB2API_DB_NAME !== database.name) throw new Error("SUB2API_DB_NAME must match YAML objects.databases[0].name"); } function writeConnectionStringExport(pg: PostgresHostConfig, inspection: SecretInspection, item: ConnectionStringExportConfig): Record { @@ -1093,8 +1106,8 @@ function secretSummary(secrets: SecretInspection): Record { } function desiredSummary(pg: PostgresHostConfig, secrets: SecretInspection, facts: RemoteFacts | null): Record { - const role = pg.objects.roles[0]; - const database = pg.objects.databases[0]; + const roleExistsByName = facts?.postgres.roleExistsByName ?? {}; + const databaseExistsByName = facts?.postgres.databaseExistsByName ?? {}; return { package: { name: `postgresql-${pg.postgres.package.version}`, @@ -1131,8 +1144,15 @@ function desiredSummary(pg: PostgresHostConfig, secrets: SecretInspection, facts }, pgHbaManagedBlock: { start: managedHbaStart, end: managedHbaEnd, rules: pg.postgres.auth.pgHba.length }, pgHbaRemoteTransport: "hostssl", - role: { name: role.name, action: facts?.postgres.roleExists ? "none" : "create-or-update" }, - database: { name: database.name, owner: database.owner, action: facts?.postgres.databaseExists ? "none" : "create" }, + roles: pg.objects.roles.map((role, index) => ({ + name: role.name, + action: (facts === null ? false : roleExistsByName[role.name] ?? (index === 0 ? facts.postgres.roleExists : false)) ? "none" : "create-or-update", + })), + databases: pg.objects.databases.map((database, index) => ({ + name: database.name, + owner: database.owner, + action: (facts === null ? false : databaseExistsByName[database.name] ?? (index === 0 ? facts.postgres.databaseExists : false)) ? "none" : "create", + })), }, secrets: secretSummary(secrets), exports: pg.exports.connectionStrings.map((item) => ({ @@ -1234,6 +1254,8 @@ function factsScript(pg: PostgresHostConfig, probe: { user: string; password: st const logDir = pg.postgres.paths.logDir; const role = pg.objects.roles[0].name; const database = pg.objects.databases[0].name; + const roleSqlList = pg.objects.roles.map((item) => `'${item.name.replace(/'/gu, "''")}'`).join(","); + const databaseSqlList = pg.objects.databases.map((item) => `'${item.name.replace(/'/gu, "''")}'`).join(","); const dnsHost = pg.postgres.network.publicDns; const appHost = pg.postgres.network.listenAddresses.find((item) => item !== "127.0.0.1" && item !== "0.0.0.0") ?? "127.0.0.1"; const probeEnv = probe === null @@ -1285,6 +1307,8 @@ if command -v runuser >/dev/null 2>&1 && command -v psql >/dev/null 2>&1 && id p $root_prefix runuser -u postgres -- psql -Atqc "SHOW ssl_key_file;" >"$tmp/sslKeyFile" 2>/dev/null $root_prefix runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_roles WHERE rolname='${role}'" >"$tmp/roleExists" 2>/dev/null $root_prefix runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='${database}'" >"$tmp/databaseExists" 2>/dev/null + $root_prefix runuser -u postgres -- psql -Atqc "SELECT rolname FROM pg_roles WHERE rolname IN (${roleSqlList})" >"$tmp/rolesExisting" 2>/dev/null + $root_prefix runuser -u postgres -- psql -Atqc "SELECT datname FROM pg_database WHERE datname IN (${databaseSqlList})" >"$tmp/databasesExisting" 2>/dev/null fi fi if [ -n "\${APP_PROBE_PASSWORD:-}" ] && command -v psql >/dev/null 2>&1; then @@ -1308,6 +1332,12 @@ def int_or_none(name): return None def yes_file(name, expected="0"): return text(name) == expected +def set_lines(name): + return {line.strip() for line in text(name).splitlines() if line.strip()} +expected_roles = ${JSON.stringify(pg.objects.roles.map((item) => item.name))} +expected_databases = ${JSON.stringify(pg.objects.databases.map((item) => item.name))} +existing_roles = set_lines("rolesExisting") +existing_databases = set_lines("databasesExisting") release_rc = text("releaserc") payload = { "ok": True, @@ -1349,6 +1379,8 @@ payload = { "logDirExists": yes_file("logDirRc"), "roleExists": text("roleExists") == "1", "databaseExists": text("databaseExists") == "1", + "roleExistsByName": {name: name in existing_roles for name in expected_roles}, + "databaseExistsByName": {name: name in existing_databases for name in expected_databases}, "serverVersion": text("serverVersion") or None, "sslOn": text("sslOn").lower() in {"on", "true", "1"}, "sslCertFile": text("sslCertFile") or None, @@ -1386,10 +1418,18 @@ async function startRemoteApplyJob(config: UniDeskConfig, pg: PostgresHostConfig function remoteApplyPayload(pg: PostgresHostConfig, secrets: SecretInspection): Record { const role = pg.objects.roles[0]; const database = pg.objects.databases[0]; - const material = secrets.materials.get(role.passwordRef.sourceRef); - if (material === undefined) throw new Error(`missing material for ${role.passwordRef.sourceRef}`); - const dbPassword = material.values[role.passwordRef.key]; - if (dbPassword === undefined || dbPassword.length === 0) throw new Error(`missing ${role.passwordRef.key}`); + const roles = pg.objects.roles.map((item) => { + const material = secrets.materials.get(item.passwordRef.sourceRef); + if (material === undefined) throw new Error(`missing material for ${item.passwordRef.sourceRef}`); + const password = material.values[item.passwordRef.key]; + if (password === undefined || password.length === 0) throw new Error(`missing ${item.passwordRef.key}`); + return { + name: item.name, + password, + login: item.login, + attributes: item.attributes, + }; + }); return { clusterId: pg.metadata.id, pgVersion: pg.postgres.package.version, @@ -1404,11 +1444,13 @@ function remoteApplyPayload(pg: PostgresHostConfig, secrets: SecretInspection): passwordEncryption: pg.postgres.auth.passwordEncryption, role: { name: role.name, - password: dbPassword, + password: roles[0].password, login: role.login, attributes: role.attributes, }, + roles, database, + databases: pg.objects.databases, backup: pg.backup.logicalDump, hbaMarkers: { start: managedHbaStart, end: managedHbaEnd }, }; @@ -1556,24 +1598,27 @@ if name == "pg_hba": print(f'{rule["type"]} {rule["database"]} {rule["user"]} {rule["address"]} {rule["method"]}') print(markers["end"]) elif name == "sql": - role = data["role"] - name = role["name"] - password = role["password"].replace("'", "''") - attrs = [] - attrs.append("LOGIN" if role.get("login") else "NOLOGIN") - attrs.append("CREATEDB" if role["attributes"].get("createdb") else "NOCREATEDB") - attrs.append("CREATEROLE" if role["attributes"].get("createrole") else "NOCREATEROLE") - attrs.append("SUPERUSER" if role["attributes"].get("superuser") else "NOSUPERUSER") - attr_sql = " ".join(attrs) - print("DO $unidesk$") - print("BEGIN") - print(f" IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{name}') THEN") - print(f" CREATE ROLE {name} {attr_sql} PASSWORD '{password}';") - print(" ELSE") - print(f" ALTER ROLE {name} WITH {attr_sql} PASSWORD '{password}';") - print(" END IF;") - print("END") - print("$unidesk$;") + for role in data.get("roles", [data["role"]]): + name = role["name"] + password = role["password"].replace("'", "''") + attrs = [] + attrs.append("LOGIN" if role.get("login") else "NOLOGIN") + attrs.append("CREATEDB" if role["attributes"].get("createdb") else "NOCREATEDB") + attrs.append("CREATEROLE" if role["attributes"].get("createrole") else "NOCREATEROLE") + attrs.append("SUPERUSER" if role["attributes"].get("superuser") else "NOSUPERUSER") + attr_sql = " ".join(attrs) + print("DO $unidesk$") + print("BEGIN") + print(f" IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{name}') THEN") + print(f" CREATE ROLE {name} {attr_sql} PASSWORD '{password}';") + print(" ELSE") + print(f" ALTER ROLE {name} WITH {attr_sql} PASSWORD '{password}';") + print(" END IF;") + print("END") + print("$unidesk$;") +elif name == "databases": + for db in data.get("databases", [data["database"]]): + print("\t".join([db["name"], db["owner"], db["encoding"], db["locale"]])) elif name == "backup_script": backup = data["backup"] role = data["role"]["name"] @@ -1704,9 +1749,14 @@ render_file sql > "$sql" runuser -u postgres -- psql -v ON_ERROR_STOP=1 < "$sql" write_state running create-database -if ! runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1; then - runuser -u postgres -- createdb -O "$DB_OWNER" -E "$DB_ENCODING" --locale="$DB_LOCALE" --template=template0 "$DB_NAME" -fi +databases_tsv="$job_dir/databases.tsv" +render_file databases > "$databases_tsv" +while IFS=' ' read -r db_name db_owner db_encoding db_locale; do + [ -n "$db_name" ] || continue + if ! runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='$db_name'" | grep -q 1; then + runuser -u postgres -- createdb -O "$db_owner" -E "$db_encoding" --locale="$db_locale" --template=template0 "$db_name" + fi +done < "$databases_tsv" if [ "$BACKUP_ENABLED" = "true" ]; then write_state running configure-backup @@ -1739,8 +1789,16 @@ fi write_state running final-check runuser -u postgres -- psql -Atqc "SHOW server_version;" >/dev/null -runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_roles WHERE rolname='$DB_OWNER'" | grep -q 1 -runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1 +python3 - "$payload" <<'PY' > "$job_dir/final-check.sql" +import json, sys +data=json.load(open(sys.argv[1], encoding="utf-8")) +for role in data.get("roles", [data["role"]]): + print("SELECT 'role', %r, EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %r);" % (role["name"], role["name"])) +for db in data.get("databases", [data["database"]]): + print("SELECT 'database', %r, EXISTS (SELECT 1 FROM pg_database WHERE datname = %r);" % (db["name"], db["name"])) +PY +chmod 0644 "$job_dir/final-check.sql" +runuser -u postgres -- psql -Atq < "$job_dir/final-check.sql" | awk -F'|' '{ if ($3 != "t") exit 1 }' write_state succeeded complete 0 trap - EXIT exit 0 diff --git a/scripts/tran b/scripts/tran index 67c18135..992cef87 100755 --- a/scripts/tran +++ b/scripts/tran @@ -1,10 +1,11 @@ #!/bin/sh set -eu -repo=${UNIDESK_TRAN_REPO_ROOT:-/root/unidesk} -if [ ! -f "$repo/scripts/cli.ts" ]; then - self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) - repo=$(CDPATH= cd -- "$self_dir/.." && pwd) +self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +self_repo=$(CDPATH= cd -- "$self_dir/.." && pwd) +repo=${UNIDESK_TRAN_REPO_ROOT:-$self_repo} +if [ ! -f "$repo/scripts/cli.ts" ] && [ -f /root/unidesk/scripts/cli.ts ]; then + repo=/root/unidesk fi tran_timeout_seconds() { diff --git a/scripts/trans b/scripts/trans index e0e68a83..7c4a1f23 100755 --- a/scripts/trans +++ b/scripts/trans @@ -1,10 +1,11 @@ #!/bin/sh set -eu -repo=${UNIDESK_TRANS_REPO_ROOT:-/root/unidesk} -if [ ! -f "$repo/scripts/cli.ts" ]; then - self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) - repo=$(CDPATH= cd -- "$self_dir/.." && pwd) +self_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +self_repo=$(CDPATH= cd -- "$self_dir/.." && pwd) +repo=${UNIDESK_TRANS_REPO_ROOT:-$self_repo} +if [ ! -f "$repo/scripts/cli.ts" ] && [ -f /root/unidesk/scripts/cli.ts ]; then + repo=/root/unidesk fi exec bun "$repo/scripts/cli.ts" ssh "$@"