From d083e88bb95ce99b42555257ae64ef032d09440b Mon Sep 17 00:00:00 2001 From: root Date: Wed, 8 Jul 2026 06:34:23 +0200 Subject: [PATCH] ymal: compose repeated node lane templates --- config/agentrun.yaml | 384 +++---- config/hwlab-node-lanes.yaml | 1020 ++++++----------- config/hwlab-web-probe-sentinel/profiles.yaml | 554 ++++----- config/platform-infra/host-proxy.yaml | 268 ++--- config/platform-infra/pipelines-as-code.yaml | 369 +++--- scripts/src/agentrun-lanes.ts | 3 +- scripts/src/hwlab-node-lanes.ts | 6 +- .../src/hwlab-node-web-sentinel-config-ref.ts | 3 +- .../src/hwlab-node-web-sentinel-resolver.ts | 3 +- scripts/src/platform-infra-host-proxy.ts | 35 +- .../src/platform-infra-pipelines-as-code.ts | 3 +- scripts/src/trans-host-proxy.ts | 3 +- scripts/src/yaml-composition.ts | 166 +++ 13 files changed, 1232 insertions(+), 1585 deletions(-) create mode 100644 scripts/src/yaml-composition.ts diff --git a/config/agentrun.yaml b/config/agentrun.yaml index dc35e799..c9ef92a8 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -2,11 +2,9 @@ version: 1 kind: AgentRunConfig metadata: name: agentrun - manager: baseUrl: https://agentrun.74-48-78-17.nip.io/ timeoutMs: 15000 - publicExposure: enabled: true proxyName: agentrun-v01-frpc @@ -23,13 +21,11 @@ publicExposure: serviceName: caddy upstreamBaseUrl: http://127.0.0.1:22880 responseHeaderTimeoutSeconds: 60 - auth: env: HWLAB_API_KEY file: /root/.config/hwlab-v02/master-server-admin-api-key.env header: Authorization scheme: Bearer - client: role: render-only transport: direct-http @@ -40,7 +36,7 @@ client: backendProfile: codex workspaceRef: kind: opaque - path: . + path: "." executionPolicy: sandbox: workspace-write approval: never @@ -48,12 +44,10 @@ client: network: enabled secretScope: allowCredentialEcho: false - controlPlane: default: node: G14 lane: v01 - nodes: G14: route: G14 @@ -70,7 +64,6 @@ controlPlane: NC01: route: NC01 kubeRoute: NC01:k3s - lanes: v01: node: G14 @@ -120,7 +113,7 @@ controlPlane: value: 74.48.78.17 bootRepoUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/agentrun.git imageBuild: - context: . + context: "." containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host @@ -131,7 +124,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -145,7 +138,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -275,49 +268,7 @@ controlPlane: key: config.toml providerCredential: profile: codex - v02: - node: D601 - version: v0.2 - source: - statusMode: k3s-git-mirror - repository: pikasTech/agentrun - branch: v0.2 - sourceAuthority: - mode: gitMirrorSnapshot - resolver: k8s-git-mirror - allowHostGit: false - allowHostWorkspace: false - allowGithubDirectInPipeline: false - sourceSnapshot: - stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} - missingObjectPolicy: fail-fast - refreshPolicy: sync-before-snapshot - bootstrapFromBranch: v0.1 - bootstrapTimeoutSeconds: 900 - bootstrapPollSeconds: 15 - remote: git@github.com:pikasTech/agentrun.git - workspace: /home/ubuntu/workspace/agentrun-v02 - runtime: - namespace: agentrun-v02 - managerDeployment: agentrun-mgr - managerService: agentrun-mgr - managerPort: 8080 - internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 - ci: - namespace: agentrun-ci - pipeline: agentrun-v02-ci-image-publish - pipelineRunPrefix: agentrun-v02-ci - serviceAccountName: agentrun-v02-tekton-runner - registryPrefix: 127.0.0.1:5000/agentrun - toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 - buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless - gitops: - branch: v0.2-gitops - path: deploy/gitops/node/d601/runtime-v02 - argoNamespace: argocd - argoApplication: agentrun-d601-v02 - repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git deployment: format: unidesk-yaml-only gitopsRoot: deploy/gitops/node/d601 @@ -345,7 +296,7 @@ controlPlane: value: 74.48.78.17 bootRepoUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git imageBuild: - context: . + context: "." containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host @@ -356,7 +307,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -370,7 +321,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -564,64 +515,20 @@ controlPlane: key: model-catalog.json providerCredential: profile: dsflash-go - - id: tool-github-pr-token - sourceRef: /root/unidesk/.env/gh_token.txt - sourceKey: GH_TOKEN - sourceFormat: raw-token - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-github-pr - key: GH_TOKEN - - id: tool-unidesk-ssh-token - sourceRef: /root/unidesk/.state/docker-compose.env - sourceKey: UNIDESK_SSH_CLIENT_TOKEN - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-unidesk-ssh - key: UNIDESK_SSH_CLIENT_TOKEN - - jd01-v02: - node: JD01 - version: v0.2 - source: - statusMode: k3s-git-mirror - repository: pikasTech/agentrun - branch: v0.2 - sourceAuthority: - mode: gitMirrorSnapshot - resolver: k8s-git-mirror - allowHostGit: false - allowHostWorkspace: false - allowGithubDirectInPipeline: false - sourceSnapshot: - stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} - missingObjectPolicy: fail-fast - refreshPolicy: sync-before-snapshot - bootstrapFromBranch: v0.1 - bootstrapTimeoutSeconds: 900 - bootstrapPollSeconds: 15 + - extends: controlPlane.templates.secrets.githubPrRawToken + - extends: controlPlane.templates.secrets.unideskSshToken + extends: controlPlane.templates.agentrunV02Lane + variables: + NODE: D601 remote: git@github.com:pikasTech/agentrun.git - workspace: /root/workspace/agentrun-v02 - runtime: - namespace: agentrun-v02 - managerDeployment: agentrun-mgr - managerService: agentrun-mgr - managerPort: 8080 - internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 - ci: - namespace: agentrun-ci - pipeline: agentrun-jd01-v02-ci-image-publish - pipelineRunPrefix: agentrun-jd01-v02-ci - serviceAccountName: agentrun-jd01-v02-tekton-runner - registryPrefix: 127.0.0.1:5000/agentrun - toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 - buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless - gitops: - branch: jd01-v0.2-gitops - path: deploy/gitops/node/jd01/runtime-v02 - argoNamespace: argocd - argoApplication: agentrun-jd01-v02 - repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + workspace: /home/ubuntu/workspace/agentrun-v02 + pipeline: agentrun-v02-ci-image-publish + pipelineRunPrefix: agentrun-v02-ci + serviceAccountName: agentrun-v02-tekton-runner + gitopsBranch: v0.2-gitops + gitopsPath: deploy/gitops/node/d601/runtime-v02 + argoApplication: agentrun-d601-v02 + jd01-v02: deployment: format: unidesk-yaml-only gitopsRoot: deploy/gitops/node/jd01 @@ -649,7 +556,7 @@ controlPlane: value: 74.48.78.17 bootRepoUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-agentrun.git imageBuild: - context: . + context: "." containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host @@ -660,7 +567,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -674,7 +581,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -716,7 +623,7 @@ controlPlane: noProxyExtra: - localhost - 127.0.0.1 - - ::1 + - "::1" - .svc - .svc.cluster.local - .cluster.local @@ -864,64 +771,20 @@ controlPlane: key: model-catalog.json providerCredential: profile: dsflash-go - - id: tool-github-pr-token - sourceRef: /root/unidesk/.env/gh_token.txt - sourceKey: GH_TOKEN - sourceFormat: raw-token - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-github-pr - key: GH_TOKEN - - id: tool-unidesk-ssh-token - sourceRef: /root/unidesk/.state/docker-compose.env - sourceKey: UNIDESK_SSH_CLIENT_TOKEN - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-unidesk-ssh - key: UNIDESK_SSH_CLIENT_TOKEN - + - extends: controlPlane.templates.secrets.githubPrRawToken + - extends: controlPlane.templates.secrets.unideskSshToken + extends: controlPlane.templates.agentrunV02Lane + variables: + NODE: JD01 + remote: git@github.com:pikasTech/agentrun.git + workspace: /root/workspace/agentrun-v02 + pipeline: agentrun-jd01-v02-ci-image-publish + pipelineRunPrefix: agentrun-jd01-v02-ci + serviceAccountName: agentrun-jd01-v02-tekton-runner + gitopsBranch: jd01-v0.2-gitops + gitopsPath: deploy/gitops/node/jd01/runtime-v02 + argoApplication: agentrun-jd01-v02 nc01-v02: - node: NC01 - version: v0.2 - source: - statusMode: k3s-git-mirror - repository: pikasTech/agentrun - branch: v0.2 - sourceAuthority: - mode: gitMirrorSnapshot - resolver: k8s-git-mirror - allowHostGit: false - allowHostWorkspace: false - allowGithubDirectInPipeline: false - sourceSnapshot: - stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} - missingObjectPolicy: fail-fast - refreshPolicy: sync-before-snapshot - bootstrapFromBranch: v0.1 - bootstrapTimeoutSeconds: 900 - bootstrapPollSeconds: 15 - remote: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-agentrun.git - workspace: /root/agentrun - runtime: - namespace: agentrun-v02 - managerDeployment: agentrun-mgr - managerService: agentrun-mgr - managerPort: 8080 - internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 - ci: - namespace: agentrun-ci - pipeline: agentrun-nc01-v02-ci-image-publish - pipelineRunPrefix: agentrun-nc01-v02-ci - serviceAccountName: agentrun-nc01-v02-tekton-runner - registryPrefix: 127.0.0.1:5000/agentrun - toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 - buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless - gitops: - branch: nc01-v0.2-gitops - path: deploy/gitops/node/nc01/runtime-v02 - argoNamespace: argocd - argoApplication: agentrun-nc01-v02 - repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git deployment: format: unidesk-yaml-only gitopsRoot: deploy/gitops/node/nc01 @@ -949,7 +812,7 @@ controlPlane: value: 152.53.229.148 bootRepoUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-agentrun.git imageBuild: - context: . + context: "." containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host @@ -960,7 +823,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -974,7 +837,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -1016,7 +879,7 @@ controlPlane: noProxyExtra: - localhost - 127.0.0.1 - - ::1 + - "::1" - .svc - .svc.cluster.local - .cluster.local @@ -1137,64 +1000,20 @@ controlPlane: key: config.toml providerCredential: profile: codex - - id: tool-github-pr-token - sourceRef: /root/unidesk/.env/gh_token.txt - sourceKey: GH_TOKEN - sourceFormat: raw-token - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-github-pr - key: GH_TOKEN - - id: tool-unidesk-ssh-token - sourceRef: /root/unidesk/.state/docker-compose.env - sourceKey: UNIDESK_SSH_CLIENT_TOKEN - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-unidesk-ssh - key: UNIDESK_SSH_CLIENT_TOKEN - + - extends: controlPlane.templates.secrets.githubPrRawToken + - extends: controlPlane.templates.secrets.unideskSshToken + extends: controlPlane.templates.agentrunV02Lane + variables: + NODE: NC01 + remote: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-agentrun.git + workspace: /root/agentrun + pipeline: agentrun-nc01-v02-ci-image-publish + pipelineRunPrefix: agentrun-nc01-v02-ci + serviceAccountName: agentrun-nc01-v02-tekton-runner + gitopsBranch: nc01-v0.2-gitops + gitopsPath: deploy/gitops/node/nc01/runtime-v02 + argoApplication: agentrun-nc01-v02 d518-v02: - node: D518 - version: v0.2 - source: - statusMode: k3s-git-mirror - repository: pikasTech/agentrun - branch: v0.2 - sourceAuthority: - mode: gitMirrorSnapshot - resolver: k8s-git-mirror - allowHostGit: false - allowHostWorkspace: false - allowGithubDirectInPipeline: false - sourceSnapshot: - stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} - missingObjectPolicy: fail-fast - refreshPolicy: sync-before-snapshot - bootstrapFromBranch: v0.1 - bootstrapTimeoutSeconds: 900 - bootstrapPollSeconds: 15 - remote: git@github.com:pikasTech/agentrun.git - workspace: /home/ubuntu/workspace/agentrun-v02 - runtime: - namespace: agentrun-v02 - managerDeployment: agentrun-mgr - managerService: agentrun-mgr - managerPort: 8080 - internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 - ci: - namespace: agentrun-ci - pipeline: agentrun-d518-v02-ci-image-publish - pipelineRunPrefix: agentrun-d518-v02-ci - serviceAccountName: agentrun-d518-v02-tekton-runner - registryPrefix: 127.0.0.1:5000/agentrun - toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 - buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless - gitops: - branch: d518-v0.2-gitops - path: deploy/gitops/node/d518/runtime-v02 - argoNamespace: argocd - argoApplication: agentrun-d518-v02 - repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git deployment: format: unidesk-yaml-only gitopsRoot: deploy/gitops/node/d518 @@ -1227,7 +1046,7 @@ controlPlane: value: 74.48.78.17 bootRepoUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git imageBuild: - context: . + context: "." containerfile: deploy/container/Containerfile repository: agentrun-mgr-env network: host @@ -1238,7 +1057,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -1252,7 +1071,7 @@ controlPlane: noProxy: - localhost - 127.0.0.1 - - ::1 + - "::1" - 127.0.0.1:5000 - localhost:5000 - .svc @@ -1465,18 +1284,77 @@ controlPlane: key: config.toml providerCredential: profile: fake-echo - - id: tool-github-pr-token - sourceRef: /root/unidesk/.env/gh_token.txt - sourceKey: GH_TOKEN - sourceFormat: raw-token - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-github-pr - key: GH_TOKEN - - id: tool-unidesk-ssh-token - sourceRef: /root/unidesk/.state/docker-compose.env - sourceKey: UNIDESK_SSH_CLIENT_TOKEN - targetRef: - namespace: agentrun-v02 - name: agentrun-v01-tool-unidesk-ssh - key: UNIDESK_SSH_CLIENT_TOKEN + - extends: controlPlane.templates.secrets.githubPrRawToken + - extends: controlPlane.templates.secrets.unideskSshToken + extends: controlPlane.templates.agentrunV02Lane + variables: + NODE: D518 + remote: git@github.com:pikasTech/agentrun.git + workspace: /home/ubuntu/workspace/agentrun-v02 + pipeline: agentrun-d518-v02-ci-image-publish + pipelineRunPrefix: agentrun-d518-v02-ci + serviceAccountName: agentrun-d518-v02-tekton-runner + gitopsBranch: d518-v0.2-gitops + gitopsPath: deploy/gitops/node/d518/runtime-v02 + argoApplication: agentrun-d518-v02 + templates: + agentrunV02Lane: + version: v0.2 + node: "${NODE}" + source: + statusMode: k3s-git-mirror + repository: pikasTech/agentrun + branch: v0.2 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: "refs/unidesk/snapshots/agentrun-yaml-lane/{branch}" + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + bootstrapFromBranch: v0.1 + bootstrapTimeoutSeconds: 900 + bootstrapPollSeconds: 15 + remote: "${remote}" + workspace: "${workspace}" + runtime: + namespace: agentrun-v02 + managerDeployment: agentrun-mgr + managerService: agentrun-mgr + managerPort: 8080 + internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 + ci: + namespace: agentrun-ci + pipeline: "${pipeline}" + pipelineRunPrefix: "${pipelineRunPrefix}" + serviceAccountName: "${serviceAccountName}" + registryPrefix: 127.0.0.1:5000/agentrun + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless + gitops: + branch: "${gitopsBranch}" + path: "${gitopsPath}" + argoNamespace: argocd + argoApplication: "${argoApplication}" + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + secrets: + githubPrRawToken: + id: tool-github-pr-token + sourceRef: /root/unidesk/.env/gh_token.txt + sourceKey: GH_TOKEN + sourceFormat: raw-token + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-tool-github-pr + key: GH_TOKEN + unideskSshToken: + id: tool-unidesk-ssh-token + sourceRef: /root/unidesk/.state/docker-compose.env + sourceKey: UNIDESK_SSH_CLIENT_TOKEN + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-tool-unidesk-ssh + key: UNIDESK_SSH_CLIENT_TOKEN \ No newline at end of file diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 96564eb2..5e549d23 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -1,17 +1,13 @@ -# SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. version: 1 kind: HwlabNodeLaneConfig metadata: name: hwlab-node-lanes - defaults: node: G14 lane: v03 - requiredNoProxy: - hyueapi.com - .hyueapi.com - nodes: G14: route: G14 @@ -48,7 +44,6 @@ nodes: gitopsRoot: deploy/gitops/node networkProfile: jd01-node-ci-egress downloadProfile: jd01-node-default - lanes: v02: node: G14 @@ -101,7 +96,7 @@ lanes: allowHostWorkspace: false allowGithubDirectInPipeline: false sourceSnapshot: - stageRefPrefix: refs/unidesk/snapshots/hwlab-node-runtime/{branch} + stageRefPrefix: "refs/unidesk/snapshots/hwlab-node-runtime/{branch}" missingObjectPolicy: fail-fast refreshPolicy: sync-before-snapshot workspace: /root/hwlab-v03 @@ -203,107 +198,11 @@ lanes: hwpodPreinstall: enabled: true configRefs: - preinstall: config/hwlab-hwpod-preinstalls/constart-71freq-c.yaml#hwpodPreinstall - projectManagementSource: config/hwlab-project-management/constart-71freq-mdtodo.yaml#projectManagement.sources[0] - gatewayProfile: config/hwlab-gateway/constart-71freq-d601-v03.yaml#gateway.profile + preinstall: "config/hwlab-hwpod-preinstalls/constart-71freq-c.yaml#hwpodPreinstall" + projectManagementSource: "config/hwlab-project-management/constart-71freq-mdtodo.yaml#projectManagement.sources[0]" + gatewayProfile: "config/hwlab-gateway/constart-71freq-d601-v03.yaml#gateway.profile" webProbe: - browserProxyMode: auto - defaultOrigin: - mode: public - baseUrl: https://hwlab.pikapython.com - authLogin: - maxAttempts: 6 - requestTimeoutMs: 30000 - initialDelayMs: 500 - maxDelayMs: 10000 - alertThresholds: - sameOriginApiSlowMs: 10000 - partialApiSlowMs: 10000 - longLivedStreamOpenSlowMs: 10000 - visibleLoadingSlowMs: 10000 - turnTimingSampleSlackSeconds: 3 - turnElapsedSevereTimeoutSeconds: 120 - domEvaluateTimeoutRedCount: 2 - domEvaluateTimeoutRedWindowMs: 30000 - screenshotTimeoutRedCount: 2 - pageErrorRedCount: 2 - longTaskRedMs: 1000 - longAnimationFrameRedMs: 1000 - eventLoopGapRedMs: 1000 - browserProcessSampleIntervalMs: 1000 - requestRateBucketMs: 10000 - requestRateTotalRedPerMinute: 300 - requestRatePageRedPerMinute: 240 - requestRateApiPathRedPerMinute: 120 - browserTotalRssRedMb: 800 - browserProcessRssRedMb: 600 - browserRssGrowthRedMb: 300 - browserRssGrowthWindowMs: 30000 - playwrightResponsivenessRedMs: 5000 - playwrightResponsivenessTimeoutRedCount: 2 - cdpMetricsTimeoutRedCount: 2 - uncommandedStateChangeCommandWindowMs: 10000 - scrollJumpCommandWindowMs: 8000 - scrollJumpFromY: 250 - scrollJumpToY: 40 - sessionRailFallbackRatio: 0.5 - browserFreezePolicy: - enabled: true - blockerWindowMs: 30000 - memory: - totalRssBlockerMb: 800 - processRssBlockerMb: 500 - growthBlockerMb: 300 - responsiveness: - latencyBlockerMs: 5000 - eventBlockerCount: 2 - cdp: - metricsTimeoutBlockerCount: 2 - kill: - enabled: true - gracefulSignal: SIGTERM - forceSignal: SIGKILL - graceMs: 3000 - pollIntervalMs: 100 - exitCode: 7 - projectManagement: - enabled: true - targetPaths: - - /projects - - /projects/mdtodo - readinessSelectors: - - '[data-testid="project-management-root"]' - - '[data-testid="project-management-mdtodo"]' - naturalApiPathPrefixes: - - /v1/project-management/ - - /v1/workbench/launches - - /v1/agent/chat - commandAllowlist: - - gotoProjectMdtodo - - openMdtodoSourceConfig - - closeMdtodoSourceConfig - - configureMdtodoHwpodSource - - probeMdtodoSource - - reindexMdtodoSource - - selectProjectSource - - selectMdtodoSource - - selectMdtodoFile - - selectMdtodoTask - - expandMdtodoTask - - openMdtodoReportPreview - - toggleMdtodoReportFullscreen - - editMdtodoTaskInline - - editMdtodoTaskTitle - - editMdtodoTaskBody - - toggleMdtodoTaskStatus - - addMdtodoRootTask - - addMdtodoSubTask - - continueMdtodoTask - - deleteMdtodoTask - - launchWorkbenchFromTask - - launchWorkbenchFromMdtodo - launchRoute: /v1/workbench/launches - slowApiBudgetMs: 10000 + extends: templates.hwlabV03.webProbeWorkbench tektonDir: tekton-v03 argoApplicationFile: application-v03.yaml registryPrefix: 127.0.0.1:5000/hwlab @@ -326,7 +225,7 @@ lanes: XDG_CONFIG_HOME: /tekton/home/.config observability: prometheusOperator: false - traceExplorerUrlTemplate: /v1/workbench/traces/{trace_id}/events + traceExplorerUrlTemplate: "/v1/workbench/traces/{trace_id}/events" metricsEndpoint: serviceName: hwlab-cloud-api containerName: hwlab-cloud-api @@ -339,13 +238,13 @@ lanes: sentinels: - id: workbench-dsflash-go-tool-call-10x enabled: true - configRef: config/hwlab-web-probe-sentinels/d601-v03/workbench-dsflash-go-tool-call-10x.yaml#sentinel + configRef: "config/hwlab-web-probe-sentinels/d601-v03/workbench-dsflash-go-tool-call-10x.yaml#sentinel" - id: workbench-auth-session-switch-2users enabled: true - configRef: config/hwlab-web-probe-sentinels/d601-v03/workbench-auth-session-switch-2users.yaml#sentinel + configRef: "config/hwlab-web-probe-sentinels/d601-v03/workbench-auth-session-switch-2users.yaml#sentinel" - id: mdtodo-visual-regression enabled: true - configRef: config/hwlab-web-probe-sentinels/d601-v03/mdtodo-visual-regression.yaml#sentinel + configRef: "config/hwlab-web-probe-sentinels/d601-v03/mdtodo-visual-regression.yaml#sentinel" workbench: enabled: true summaryPath: /v1/web-performance/summary @@ -355,7 +254,8 @@ lanes: - hwlab_agentrun_ - hwlab_webui_ - hwlab_http_ - requiredSeries: [] + requiredSeries: + [] backendLabelDenylist: - unknown maxUnknownEventLines: 0 @@ -366,7 +266,15 @@ lanes: quantile: 0.95 window: 5m minSamples: 5 - groupBy: [namespace, gitops_target, journey, route, backend, transport, entry, outcome] + groupBy: + - namespace + - gitops_target + - journey + - route + - backend + - transport + - entry + - outcome matchLabels: journey: submit_to_first_visible - id: workbench_backend_event_visible_p95 @@ -375,14 +283,28 @@ lanes: quantile: 0.95 window: 5m minSamples: 5 - groupBy: [namespace, gitops_target, event_type, backend, transport, outcome] + groupBy: + - namespace + - gitops_target + - event_type + - backend + - transport + - outcome - id: workbench_session_switch_p95 metric: hwlab:workbench_session_switch:p95_seconds sourceMetric: hwlab_workbench_journey_duration_seconds quantile: 0.95 window: 5m minSamples: 5 - groupBy: [namespace, gitops_target, journey, route, target_state, cache, source, outcome] + groupBy: + - namespace + - gitops_target + - journey + - route + - target_state + - cache + - source + - outcome matchLabels: journey: session_switch_first_visible|session_switch_full_load - id: workbench_open_p95 @@ -391,7 +313,14 @@ lanes: quantile: 0.95 window: 5m minSamples: 5 - groupBy: [namespace, gitops_target, journey, route, cache, auth_state, outcome] + groupBy: + - namespace + - gitops_target + - journey + - route + - cache + - auth_state + - outcome matchLabels: journey: workbench_open_first_visible|workbench_open_full_load - id: workbench_projection_lag_p95 @@ -400,35 +329,70 @@ lanes: quantile: 0.95 window: 5m minSamples: 1 - groupBy: [namespace, gitops_target, node, lane, projection_status, source, status, reason] + groupBy: + - namespace + - gitops_target + - node + - lane + - projection_status + - source + - status + - reason - id: workbench_terminal_projection_delay_p95 metric: hwlab:workbench_terminal_projection_delay:p95_seconds sourceMetric: hwlab_workbench_terminal_projection_delay_seconds quantile: 0.95 window: 5m minSamples: 1 - groupBy: [namespace, gitops_target, node, lane, projection_status, source, status, reason] + groupBy: + - namespace + - gitops_target + - node + - lane + - projection_status + - source + - status + - reason - id: workbench_turn_get_p95 metric: hwlab:workbench_turn_get:p95_seconds sourceMetric: hwlab_workbench_turn_get_duration_seconds quantile: 0.95 window: 5m minSamples: 1 - groupBy: [namespace, gitops_target, node, lane, route, status, degraded_reason] + groupBy: + - namespace + - gitops_target + - node + - lane + - route + - status + - degraded_reason - id: agentrun_result_p95 metric: hwlab:agentrun_result:p95_seconds sourceMetric: hwlab_agentrun_result_duration_seconds quantile: 0.95 window: 5m minSamples: 1 - groupBy: [namespace, gitops_target, node, lane, event_count_bucket, status] + groupBy: + - namespace + - gitops_target + - node + - lane + - event_count_bucket + - status - id: workbench_projector_batch_p95 metric: hwlab:workbench_projector_batch:p95_seconds sourceMetric: hwlab_workbench_projector_batch_duration_seconds quantile: 0.95 window: 5m minSamples: 1 - groupBy: [namespace, gitops_target, node, lane, phase, status] + groupBy: + - namespace + - gitops_target + - node + - lane + - phase + - status warningAlerts: - id: HWLABWorkbenchSubmitFirstVisibleSlow ruleId: workbench_submit_first_visible_p95 @@ -487,24 +451,11 @@ lanes: minSamples: 1 for: 10m runtimeImageRewrites: - - source: fatedier/frpc:v0.68.1 - target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 - - source: openfga/openfga:v1.17.0 - target: 127.0.0.1:5000/hwlab/openfga:v1.17.0 + - extends: templates.hwlabV03.runtimeImageRewrites.frpc + - extends: templates.hwlabV03.runtimeImageRewrites.openfga runtimeImageBuilds: - - id: moonbridge - kind: moonbridge - target: 127.0.0.1:5000/hwlab/moonbridge:1b99888d3dae - sourceRepo: https://github.com/ZhiYi-R/moon-bridge.git - sourceRef: 1b99888d3dae889b79ee602cb875c7907f7e76f2 - builderImage: golang:1.25-bookworm - goProxy: https://goproxy.cn,direct - dockerNetworkMode: host - - id: opencode-git - kind: opencode-git - target: 127.0.0.1:5000/hwlab/opencode:1.17.7-git - sourceImage: ghcr.io/anomalyco/opencode:1.17.7 - dockerNetworkMode: host + - extends: templates.hwlabV03.runtimeImageBuilds.moonbridge + - extends: templates.hwlabV03.runtimeImageBuilds.opencodeGit public: webUrl: https://hwlab.pikapython.com apiUrl: https://hwlab.pikapython.com @@ -519,60 +470,32 @@ lanes: rollout: deployment: hwlab-cloud-api publicExposure: - mode: pk01-caddy-frp - publicBaseUrl: https://hwlab.pikapython.com - hostname: hwlab.pikapython.com - expectedA: 82.156.23.220 + extends: templates.hwlabV03.publicExposurePk01 frpc: - serverAddr: 82.156.23.220 - serverPort: 22000 - tokenSourceRef: platform-infra/pk01-frp.env - tokenSourceKey: FRP_TOKEN - secretName: hwlab-v03-frpc-secrets - secretKey: frpc.toml - tokenKey: token webProxy: - name: hwlab-d601-v03-cloud-web remotePort: 22096 - localIP: hwlab-cloud-web.hwlab-v03.svc.cluster.local - localPort: 8080 apiProxy: - name: hwlab-d601-v03-edge-proxy remotePort: 22095 - localIP: hwlab-edge-proxy.hwlab-v03.svc.cluster.local - localPort: 6667 - caddy: - route: PK01 - configPath: /etc/caddy/Caddyfile - serviceName: caddy - email: ops@pikapython.com - tls: auto - responseHeaderTimeoutSeconds: 600 externalPostgres: + extends: templates.hwlabV03.externalPostgresConsumer provider: PK01 configRef: config/platform-db/postgres-pk01.yaml serviceName: d601-pk01-platform-postgres endpointAddress: 82.156.23.220 - port: 5432 runtimeAccess: routeName: d601-pk01-postgres endpointAddress: d601-tcp-egress-gateway.unidesk.svc.cluster.local port: 25432 - sslmode: require database: hwlab_d601_v03 cloudApi: - secretName: hwlab-cloud-api-v03-db - secretKey: database-url sourceRef: hwlab/d601-v03-cloud-api-db.env - envKey: DATABASE_URL role: hwlab_d601_v03_app openfga: - secretName: hwlab-v03-openfga - secretKey: datastore-uri sourceRef: hwlab/d601-v03-openfga-db.env - envKey: DATASTORE_URI - authnKey: authn-preshared-key role: hwlab_d601_v03_app + variables: + LANE: v03 + NODE: D601 D518: node: D518 workspace: /home/ubuntu/workspace/hwlab-v03 @@ -639,103 +562,11 @@ lanes: queryRetryInitialDelayMs: 250 queryRetryMaxDelayMs: 5000 webProbe: + extends: templates.hwlabV03.webProbeWorkbench browserProxyMode: direct - defaultOrigin: - mode: public - baseUrl: https://hwlab.pikapython.com - authLogin: - maxAttempts: 6 - requestTimeoutMs: 30000 - initialDelayMs: 500 - maxDelayMs: 10000 - alertThresholds: - sameOriginApiSlowMs: 10000 - partialApiSlowMs: 10000 - longLivedStreamOpenSlowMs: 10000 - visibleLoadingSlowMs: 10000 - turnTimingSampleSlackSeconds: 3 - turnElapsedSevereTimeoutSeconds: 120 - domEvaluateTimeoutRedCount: 2 - domEvaluateTimeoutRedWindowMs: 30000 - screenshotTimeoutRedCount: 2 - pageErrorRedCount: 2 - longTaskRedMs: 1000 - longAnimationFrameRedMs: 1000 - eventLoopGapRedMs: 1000 - browserProcessSampleIntervalMs: 1000 - requestRateBucketMs: 10000 - requestRateTotalRedPerMinute: 300 - requestRatePageRedPerMinute: 240 - requestRateApiPathRedPerMinute: 120 - browserTotalRssRedMb: 800 - browserProcessRssRedMb: 600 - browserRssGrowthRedMb: 300 - browserRssGrowthWindowMs: 30000 - playwrightResponsivenessRedMs: 5000 - playwrightResponsivenessTimeoutRedCount: 2 - cdpMetricsTimeoutRedCount: 2 - uncommandedStateChangeCommandWindowMs: 10000 - scrollJumpCommandWindowMs: 8000 - scrollJumpFromY: 250 - scrollJumpToY: 40 - sessionRailFallbackRatio: 0.5 browserFreezePolicy: - enabled: true - blockerWindowMs: 30000 memory: totalRssBlockerMb: 500 - processRssBlockerMb: 500 - growthBlockerMb: 300 - responsiveness: - latencyBlockerMs: 5000 - eventBlockerCount: 2 - cdp: - metricsTimeoutBlockerCount: 2 - kill: - enabled: true - gracefulSignal: SIGTERM - forceSignal: SIGKILL - graceMs: 3000 - pollIntervalMs: 100 - exitCode: 7 - projectManagement: - enabled: true - targetPaths: - - /projects - - /projects/mdtodo - readinessSelectors: - - '[data-testid="project-management-root"]' - - '[data-testid="project-management-mdtodo"]' - naturalApiPathPrefixes: - - /v1/project-management/ - - /v1/workbench/launches - - /v1/agent/chat - commandAllowlist: - - gotoProjectMdtodo - - openMdtodoSourceConfig - - closeMdtodoSourceConfig - - configureMdtodoHwpodSource - - probeMdtodoSource - - reindexMdtodoSource - - selectProjectSource - - selectMdtodoSource - - selectMdtodoFile - - selectMdtodoTask - - expandMdtodoTask - - openMdtodoReportPreview - - toggleMdtodoReportFullscreen - - editMdtodoTaskInline - - editMdtodoTaskTitle - - editMdtodoTaskBody - - toggleMdtodoTaskStatus - - addMdtodoRootTask - - addMdtodoSubTask - - continueMdtodoTask - - deleteMdtodoTask - - launchWorkbenchFromTask - - launchWorkbenchFromMdtodo - launchRoute: /v1/workbench/launches - slowApiBudgetMs: 10000 tektonDir: tekton-v03 argoApplicationFile: application-v03.yaml registryPrefix: 127.0.0.1:5000/hwlab @@ -768,29 +599,16 @@ lanes: sentinels: - id: workbench-dsflash-go-tool-call-10x enabled: true - configRef: config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml#sentinel + configRef: "config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml#sentinel" - id: workbench-fake-echo-session-invariance-10x enabled: true - configRef: config/hwlab-web-probe-sentinels/d518-v03/workbench-fake-echo-session-invariance-10x.yaml#sentinel + configRef: "config/hwlab-web-probe-sentinels/d518-v03/workbench-fake-echo-session-invariance-10x.yaml#sentinel" runtimeImageRewrites: - - source: fatedier/frpc:v0.68.1 - target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 - - source: openfga/openfga:v1.17.0 - target: 127.0.0.1:5000/hwlab/openfga:v1.17.0 + - extends: templates.hwlabV03.runtimeImageRewrites.frpc + - extends: templates.hwlabV03.runtimeImageRewrites.openfga runtimeImageBuilds: - - id: moonbridge - kind: moonbridge - target: 127.0.0.1:5000/hwlab/moonbridge:1b99888d3dae - sourceRepo: https://github.com/ZhiYi-R/moon-bridge.git - sourceRef: 1b99888d3dae889b79ee602cb875c7907f7e76f2 - builderImage: golang:1.25-bookworm - goProxy: https://goproxy.cn,direct - dockerNetworkMode: host - - id: opencode-git - kind: opencode-git - target: 127.0.0.1:5000/hwlab/opencode:1.17.7-git - sourceImage: ghcr.io/anomalyco/opencode:1.17.7 - dockerNetworkMode: host + - extends: templates.hwlabV03.runtimeImageBuilds.moonbridge + - extends: templates.hwlabV03.runtimeImageBuilds.opencodeGit public: webUrl: https://hwlab.pikapython.com apiUrl: https://hwlab.pikapython.com @@ -825,79 +643,45 @@ lanes: enabled: true mode: shadow-produce-only consumeEnabled: false - configRef: config/platform-infra/kafka.yaml#clients.hwlab-v03-cloud-api + configRef: "config/platform-infra/kafka.yaml#clients.hwlab-v03-cloud-api" bootstrapServers: platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092 commandTopic: hwlab.agentrun.command.v1 clientId: hwlab-v03-cloud-api publicExposure: - mode: pk01-caddy-frp - publicBaseUrl: https://hwlab.pikapython.com - hostname: hwlab.pikapython.com - expectedA: 82.156.23.220 + extends: templates.hwlabV03.publicExposurePk01 frpc: - serverAddr: 82.156.23.220 - serverPort: 22000 - tokenSourceRef: platform-infra/pk01-frp.env - tokenSourceKey: FRP_TOKEN - secretName: hwlab-v03-frpc-secrets - secretKey: frpc.toml - tokenKey: token webProxy: - name: hwlab-d518-v03-cloud-web remotePort: 22092 - localIP: hwlab-cloud-web.hwlab-v03.svc.cluster.local - localPort: 8080 apiProxy: - name: hwlab-d518-v03-edge-proxy remotePort: 22091 - localIP: hwlab-edge-proxy.hwlab-v03.svc.cluster.local - localPort: 6667 - caddy: - route: PK01 - configPath: /etc/caddy/Caddyfile - serviceName: caddy - email: ops@pikapython.com - tls: auto - responseHeaderTimeoutSeconds: 600 externalPostgres: + extends: templates.hwlabV03.externalPostgresConsumer provider: PK01 configRef: config/platform-db/postgres-pk01.yaml serviceName: d518-pk01-platform-postgres endpointAddress: 82.156.23.220 - port: 5432 runtimeAccess: routeName: d518-pk01-postgres endpointAddress: d518-tcp-egress-gateway.unidesk.svc.cluster.local port: 25432 - sslmode: require database: hwlab_d518_v03 cloudApi: - secretName: hwlab-cloud-api-v03-db - secretKey: database-url sourceRef: hwlab/d518-v03-cloud-api-db.env - envKey: DATABASE_URL role: hwlab_d518_v03_app openfga: - secretName: hwlab-v03-openfga - secretKey: datastore-uri sourceRef: hwlab/d518-v03-openfga-db.env - envKey: DATASTORE_URI - authnKey: authn-preshared-key role: hwlab_d518_v03_app + variables: + LANE: v03 + NODE: D518 JD01: node: JD01 workspace: /root/workspace/hwlab-v03 sourceAuthority: - mode: giteaSnapshot - resolver: gitea-mirror + extends: templates.hwlabV03.sourceAuthorityGiteaSnapshot giteaMirrorRepoKey: hwlab-jd01-v03 - allowHostGit: false - allowHostWorkspace: false - allowGithubDirectInPipeline: false sourceSnapshot: - stageRefPrefix: refs/unidesk/snapshots/gitea-actions/hwlab-node-runtime/{branch} - missingObjectPolicy: fail-fast - refreshPolicy: gitea-controlled-snapshot + extends: templates.hwlabV03.sourceSnapshotGiteaControlled sourceWorkspace: git: remoteName: github @@ -927,41 +711,10 @@ lanes: stateDir: /var/lib/unidesk/hwlab-source-workspace install: timeoutSeconds: 300 - command: | - set -eu - . /etc/unidesk/proxy.env 2>/dev/null || true - export HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy - npm_version=11.17.0 - tmp_dir=$(mktemp -d) - trap 'rm -rf "$tmp_dir"' EXIT - curl -fsSL --retry 3 --connect-timeout 20 --max-time 300 "https://registry.npmmirror.com/npm/-/npm-${npm_version}.tgz" -o "$tmp_dir/npm.tgz" - mkdir -p /usr/local/lib/node_modules - rm -rf /usr/local/lib/node_modules/npm - tar -xzf "$tmp_dir/npm.tgz" -C "$tmp_dir" - mv "$tmp_dir/package" /usr/local/lib/node_modules/npm - ln -sfn /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm - ln -sfn /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx - npm --version - npx --version + command: "set -eu\n. /etc/unidesk/proxy.env 2>/dev/null || true\nexport HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy\nnpm_version=11.17.0\ntmp_dir=$(mktemp -d)\ntrap 'rm -rf \"$tmp_dir\"' EXIT\ncurl -fsSL --retry 3 --connect-timeout 20 --max-time 300 \"https://registry.npmmirror.com/npm/-/npm-${npm_version}.tgz\" -o \"$tmp_dir/npm.tgz\"\nmkdir -p /usr/local/lib/node_modules\nrm -rf /usr/local/lib/node_modules/npm\ntar -xzf \"$tmp_dir/npm.tgz\" -C \"$tmp_dir\"\nmv \"$tmp_dir/package\" /usr/local/lib/node_modules/npm\nln -sfn /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm\nln -sfn /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx\nnpm --version\nnpx --version\n" install: executor: host - dependencyCommand: | - set -eu - npm install --no-save --package-lock=false --ignore-scripts --no-audit --no-fund playwright@1.59.1 - if [ -n "${HTTP_PROXY:-}" ]; then - cat >/etc/apt/apt.conf.d/99unidesk-host-proxy </etc/apt/apt.conf.d/99unidesk-host-proxy <>(configPath); + const root = materializeYamlComposition(readYamlRecord>(configPath), { label: configPath }).value; validateConfigEnvelope(root, configPath); const controlPlane = recordField(root, "controlPlane", configPath); const defaultTarget = recordField(controlPlane, "default", `${configPath}.controlPlane`); diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 37079160..9297b86a 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -5,6 +5,7 @@ // Responsibility: YAML source-of-truth parsing for HWLAB node/lane Workbench observability. import { readFileSync } from "node:fs"; import { rootPath } from "./config"; +import { materializeYamlComposition } from "./yaml-composition"; export type HwlabRuntimeLane = string; @@ -1685,7 +1686,10 @@ function runtimeImageBuildsConfig(value: unknown, path: string): HwlabRuntimeIma function readHwlabNodeLaneConfig(): HwlabNodeLaneConfig { const path = rootPath(HWLAB_NODE_LANE_CONFIG_PATH); const raw = readFileSync(path, "utf8"); - const parsed = asRecord(Bun.YAML.parse(raw) as unknown, HWLAB_NODE_LANE_CONFIG_PATH); + const parsed = asRecord( + materializeYamlComposition(Bun.YAML.parse(raw) as unknown, { label: HWLAB_NODE_LANE_CONFIG_PATH }).value, + HWLAB_NODE_LANE_CONFIG_PATH, + ); validateConfigEnvelope(parsed); const defaultTarget = parseDefaultTarget(asRecord(parsed.defaults, `${HWLAB_NODE_LANE_CONFIG_PATH}.defaults`)); const requiredNoProxy = stringArrayField(parsed, "requiredNoProxy", HWLAB_NODE_LANE_CONFIG_PATH); diff --git a/scripts/src/hwlab-node-web-sentinel-config-ref.ts b/scripts/src/hwlab-node-web-sentinel-config-ref.ts index 4820a227..00606b6d 100644 --- a/scripts/src/hwlab-node-web-sentinel-config-ref.ts +++ b/scripts/src/hwlab-node-web-sentinel-config-ref.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { rootPath } from "./config"; +import { materializeYamlComposition } from "./yaml-composition"; export interface WebProbeSentinelTemplateContext { readonly nodeId: string; @@ -34,7 +35,7 @@ export function readWebProbeSentinelConfigRef(context: WebProbeSentinelTemplateC const absPath = rootPath(parsed.file); if (!existsSync(absPath)) throw new Error(`${parsed.file} does not exist`); const text = readFileSync(absPath, "utf8"); - const doc = Bun.YAML.parse(text) as unknown; + const doc = materializeYamlComposition(Bun.YAML.parse(text) as unknown, { label: parsed.file }).value; const vars = { ...builtinVars, ...documentVars(doc, builtinVars) }; const path = renderTemplateString(parsed.path, vars, `${parsed.file}#path`); const target = valueAtConfigPath(doc, path); diff --git a/scripts/src/hwlab-node-web-sentinel-resolver.ts b/scripts/src/hwlab-node-web-sentinel-resolver.ts index c2f63fe1..46260e44 100644 --- a/scripts/src/hwlab-node-web-sentinel-resolver.ts +++ b/scripts/src/hwlab-node-web-sentinel-resolver.ts @@ -4,6 +4,7 @@ import { existsSync, readFileSync } from "node:fs"; import { rootPath } from "./config"; import { readWebProbeSentinelConfigRefTarget, type WebProbeSentinelTemplateContext } from "./hwlab-node-web-sentinel-config-ref"; import { HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS, type HwlabRuntimeLaneSpec, type HwlabRuntimeWebProbeSentinelConfigRefKey, type HwlabRuntimeWebProbeSentinelRegistryItemSpec } from "./hwlab-node-lanes"; +import { materializeYamlComposition } from "./yaml-composition"; export interface ResolvedWebProbeSentinel { readonly id: string; @@ -169,7 +170,7 @@ export function readConfigRefTarget(ref: string, context?: WebProbeSentinelTempl } const absPath = rootPath(file); if (!existsSync(absPath)) throw new Error(`${file} does not exist`); - const doc = Bun.YAML.parse(readFileSync(absPath, "utf8")) as unknown; + const doc = materializeYamlComposition(Bun.YAML.parse(readFileSync(absPath, "utf8")) as unknown, { label: file }).value; return valueAtPath(doc, path); } diff --git a/scripts/src/platform-infra-host-proxy.ts b/scripts/src/platform-infra-host-proxy.ts index 87d987c8..8303e01e 100644 --- a/scripts/src/platform-infra-host-proxy.ts +++ b/scripts/src/platform-infra-host-proxy.ts @@ -9,6 +9,7 @@ import type { RenderedCliResult } from "./output"; import { shQuote } from "./platform-infra-public-service"; import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; import { transHostProxyEnvSummary } from "./trans-host-proxy"; +import { materializeYamlComposition } from "./yaml-composition"; const configPath = "config/platform-infra/host-proxy.yaml"; @@ -161,7 +162,7 @@ function parseOptions(args: string[]): HostProxyOptions { } function readHostProxyConfig(): HostProxyConfig { - const root = record(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")), configPath); + const root = record(materializeYamlComposition(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")) as unknown, { label: configPath }).value, configPath); if (root.kind !== "platform-infra-host-proxy") throw new Error(`${configPath}.kind must be platform-infra-host-proxy`); const defaultsRaw = record(root.defaults, `${configPath}.defaults`); const server = parseServer(record(root.server, `${configPath}.server`), `${configPath}.server`); @@ -172,7 +173,7 @@ function readHostProxyConfig(): HostProxyConfig { })); const targets: Record = {}; for (const [id, value] of Object.entries(record(root.targets, `${configPath}.targets`))) { - targets[id] = parseTarget(id, record(value, `${configPath}.targets.${id}`), sources); + targets[id] = parseTarget(id, record(value, `${configPath}.targets.${id}`), sources, root); } return { defaults: { targetId: stringField(defaultsRaw, "targetId", `${configPath}.defaults`) }, @@ -315,7 +316,7 @@ function podAccessSpec(raw: Record, label: string): HostProxyPo return { enabled, listenHost, listenPort, proxyUrl }; } -function parseTarget(id: string, raw: Record, sources: Record): HostProxyTarget { +function parseTarget(id: string, raw: Record, sources: Record, root: Record): HostProxyTarget { const sourceRef = stringField(raw, "sourceRef", `${configPath}.targets.${id}`); if (!sourceRef.startsWith("sources.")) throw new Error(`${configPath}.targets.${id}.sourceRef must use sources.`); const sourceId = sourceRef.slice("sources.".length); @@ -324,7 +325,9 @@ function parseTarget(id: string, raw: Record, sources: Record, sources: Record, root: Record, label: string): string[] { + const refs = record(envRaw.noProxyRefs, `${label}.noProxyRefs`); + const values: string[] = []; + for (const key of ["common", "extras"] as const) { + const ref = stringField(refs, key, `${label}.noProxyRefs`); + values.push(...stringArrayAtPath(root, ref, `${label}.noProxyRefs.${key}`)); + } + return [...new Set(values)]; +} + +function stringArrayAtPath(root: unknown, ref: string, label: string): string[] { + let current: unknown = root; + for (const segment of ref.split(".")) { + if (!/^[A-Za-z0-9_-]+$/u.test(segment) || typeof current !== "object" || current === null || Array.isArray(current)) { + throw new Error(`${label} references invalid path ${ref}`); + } + current = (current as Record)[segment]; + } + if (!Array.isArray(current) || !current.every((item) => typeof item === "string" && item.length > 0)) { + throw new Error(`${label} must reference a string array`); + } + return [...current]; +} + function resolveTarget(config: HostProxyConfig, targetId: string): HostProxyTarget { const target = config.targets[targetId]; if (target === undefined) throw new Error(`${configPath}.targets.${targetId} is not defined`); diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index c13adeda..1af93d2c 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -15,6 +15,7 @@ import { shQuote, sha256Fingerprint, } from "./platform-infra-ops-library"; +import { materializeYamlComposition } from "./yaml-composition"; const configFile = rootPath("config", "platform-infra", "pipelines-as-code.yaml"); const configLabel = "config/platform-infra/pipelines-as-code.yaml"; @@ -213,7 +214,7 @@ function help(): Record { } function readPacConfig(): PacConfig { - const root = readYamlRecord>(configFile, "platform-infra-pipelines-as-code"); + const root = materializeYamlComposition(readYamlRecord>(configFile, "platform-infra-pipelines-as-code"), { label: configLabel }).value; const release = y.objectField(root, "release", ""); const closeout = y.objectField(root, "closeout", ""); const gitOpsMirrorFlush = y.objectField(closeout, "gitOpsMirrorFlush", "closeout"); diff --git a/scripts/src/trans-host-proxy.ts b/scripts/src/trans-host-proxy.ts index 97ab97cf..e1f9fc32 100644 --- a/scripts/src/trans-host-proxy.ts +++ b/scripts/src/trans-host-proxy.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { repoRoot } from "./config"; +import { materializeYamlComposition } from "./yaml-composition"; export const transHostProxyConfigPath = "config/platform-infra/host-proxy.yaml"; @@ -19,7 +20,7 @@ export interface TransHostProxyEnvRule { export function readTransHostProxyEnvRule(targetId: string): TransHostProxyEnvRule | null { const absolutePath = join(repoRoot, transHostProxyConfigPath); if (!existsSync(absolutePath)) return null; - const root = record(Bun.YAML.parse(readFileSync(absolutePath, "utf8")), transHostProxyConfigPath); + const root = record(materializeYamlComposition(Bun.YAML.parse(readFileSync(absolutePath, "utf8")) as unknown, { label: transHostProxyConfigPath }).value, transHostProxyConfigPath); if (root.kind !== "platform-infra-host-proxy") throw new Error(`${transHostProxyConfigPath}.kind must be platform-infra-host-proxy`); const targets = record(root.targets, `${transHostProxyConfigPath}.targets`); const targetRaw = targets[targetId]; diff --git a/scripts/src/yaml-composition.ts b/scripts/src/yaml-composition.ts new file mode 100644 index 00000000..ed3f9abe --- /dev/null +++ b/scripts/src/yaml-composition.ts @@ -0,0 +1,166 @@ +// SPEC: pikasTech/unidesk#1579 YAML-first composition. +// Responsibility: materialize lightweight YAML templates before domain parsers validate shapes. + +export interface YamlCompositionResult { + readonly value: T; + readonly traces: readonly YamlCompositionTrace[]; +} + +export interface YamlCompositionTrace { + readonly path: string; + readonly extends: readonly string[]; + readonly overrideKeys: readonly string[]; + readonly variables: Record; +} + +interface CompositionOptions { + readonly label: string; + readonly stripTemplateKeys?: boolean; +} + +const COMPOSITION_KEYS = new Set(["extends", "overrides", "variables"]); +const TEMPLATE_ROOT_KEYS = new Set(["templates", "baselines"]); + +export function materializeYamlComposition(input: T, options: CompositionOptions): YamlCompositionResult { + const traces: YamlCompositionTrace[] = []; + const root = input as unknown; + const value = materializeValue(root, root, options.label, {}, traces, []); + if (options.stripTemplateKeys === true && isRecord(value)) { + const stripped = { ...value }; + for (const key of TEMPLATE_ROOT_KEYS) delete stripped[key]; + return { value: stripped as T, traces }; + } + return { value: value as T, traces }; +} + +export function renderYamlTemplateStrings(input: T, variables: Record): T { + return renderValue(input as unknown, deriveVariables(variables)) as T; +} + +function materializeValue( + value: unknown, + root: unknown, + path: string, + inheritedVariables: Record, + traces: YamlCompositionTrace[], + stack: readonly string[], +): unknown { + if (Array.isArray(value)) { + return value.map((item, index) => materializeValue(item, root, `${path}[${index}]`, inheritedVariables, traces, stack)); + } + if (!isRecord(value)) return renderValue(value, inheritedVariables); + const localVariables = deriveVariables({ ...inheritedVariables, ...stringRecord(value.variables) }); + const refs = extendsRefs(value.extends); + let merged: Record = {}; + if (refs.length > 0) { + for (const ref of refs) { + if (stack.includes(ref)) throw new Error(`${path}.extends has cyclic template reference ${ref}`); + const template = valueAtPath(root, ref); + if (!isRecord(template)) throw new Error(`${path}.extends references missing object ${ref}`); + const renderedTemplate = materializeValue(template, root, ref, localVariables, traces, [...stack, ref]); + merged = deepMerge(merged, asRecord(renderedTemplate, ref)); + } + } + const own = stripCompositionFields(value); + const overrides = value.overrides === undefined ? {} : asRecord(value.overrides, `${path}.overrides`); + merged = deepMerge(merged, own); + merged = deepMerge(merged, overrides); + const rendered = renderObjectScalars(merged, deriveVariables({ ...variablesFromRecord(merged), ...localVariables })); + const result = Object.fromEntries( + Object.entries(asRecord(rendered, path)).map(([key, item]) => [ + key, + materializeValue(item, root, `${path}.${key}`, deriveVariables({ ...localVariables, ...variablesFromRecord(rendered) }), traces, stack), + ]), + ); + if (refs.length > 0) { + traces.push({ + path, + extends: refs, + overrideKeys: Object.keys(overrides).sort(), + variables: deriveVariables({ ...localVariables, ...variablesFromRecord(result) }), + }); + } + return result; +} + +function stripCompositionFields(value: Record): Record { + return Object.fromEntries(Object.entries(value).filter(([key]) => !COMPOSITION_KEYS.has(key))); +} + +function extendsRefs(value: unknown): string[] { + if (value === undefined) return []; + if (typeof value === "string" && value.trim().length > 0) return [value.trim()]; + if (Array.isArray(value) && value.every((item) => typeof item === "string" && item.trim().length > 0)) { + return value.map((item) => item.trim()); + } + throw new Error("extends must be a non-empty string or string array"); +} + +function renderValue(value: unknown, variables: Record): unknown { + if (typeof value === "string") { + return value.replace(/\$\{([A-Za-z][A-Za-z0-9_]*)\}/gu, (match, name: string) => variables[name] ?? match); + } + if (Array.isArray(value)) return value.map((item) => renderValue(item, variables)); + if (!isRecord(value)) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, renderValue(item, variables)])); +} + +function renderObjectScalars(value: Record, variables: Record): Record { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [ + key, + typeof item === "string" ? renderValue(item, variables) : item, + ])); +} + +function variablesFromRecord(value: unknown): Record { + if (!isRecord(value)) return {}; + const result: Record = {}; + if (typeof value.node === "string" && value.node.length > 0) result.NODE = value.node; + if (typeof value.lane === "string" && value.lane.length > 0) result.LANE = value.lane; + if (typeof value.id === "string" && value.id.length > 0) result.targetId = value.id; + return deriveVariables(result); +} + +function deriveVariables(input: Record): Record { + const result = { ...input }; + if (result.NODE !== undefined) result.nodeLower = result.NODE.toLowerCase(); + if (result.LANE !== undefined) result.lane = result.LANE; + if (result.lane !== undefined && result.LANE === undefined) result.LANE = result.lane; + return result; +} + +function stringRecord(value: unknown): Record { + if (value === undefined) return {}; + const raw = asRecord(value, "variables"); + return Object.fromEntries(Object.entries(raw).map(([key, item]) => { + if (typeof item !== "string" || item.length === 0) throw new Error(`variables.${key} must be a non-empty string`); + return [key, item]; + })); +} + +function deepMerge(base: Record, override: Record): Record { + const result: Record = { ...base }; + for (const [key, value] of Object.entries(override)) { + const existing = result[key]; + result[key] = isRecord(existing) && isRecord(value) ? deepMerge(existing, value) : value; + } + return result; +} + +function valueAtPath(root: unknown, ref: string): unknown { + let current: unknown = root; + for (const segment of ref.split(".")) { + if (!/^[A-Za-z0-9_-]+$/u.test(segment) || !isRecord(current)) return undefined; + current = current[segment]; + } + return current; +} + +function asRecord(value: unknown, path: string): Record { + if (!isRecord(value)) throw new Error(`${path} must be an object`); + return value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +}