From 076c4b643dc9bff99de75894f440f0f05eabd9c2 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 29 Jun 2026 08:13:34 +0000 Subject: [PATCH] feat: add JD01 YAML-first deployment support --- .agents/skills/unidesk-ymalops/SKILL.md | 1 + config/agentrun.yaml | 262 ++ config/deploy-ssh-identities.yaml | 8 + config/hwlab-node-control-plane.yaml | 367 ++ config/hwlab-node-lanes.yaml | 258 +- config/hwlab-test-accounts.yaml | 76 + config/hwlab-web-probe-sentinel/profiles.yaml | 240 ++ .../report-views.multi-sentinel.yaml | 24 + .../scenarios.multi-sentinel.yaml | 126 + config/platform-infra/host-proxy.yaml | 112 + config/platform-infra/observability.yaml | 39 + config/platform-infra/sub2api.yaml | 66 + scripts/src/agentrun-lanes.ts | 23 + scripts/src/agentrun-manifests.ts | 10 +- scripts/src/agentrun/cleanup-scripts.ts | 188 +- scripts/src/agentrun/control-plane.ts | 12 +- scripts/src/agentrun/git-mirror.ts | 36 +- scripts/src/agentrun/secrets.ts | 277 +- scripts/src/agentrun/yaml-lane.ts | 430 ++- scripts/src/hwlab-node-control-plane-model.ts | 371 ++ .../src/hwlab-node-control-plane-runtime.ts | 2616 ++++++++++++++ scripts/src/hwlab-node-control-plane.ts | 2421 +++---------- scripts/src/hwlab-node-lanes.ts | 14 +- scripts/src/hwlab-node-web-sentinel-cicd.ts | 3040 +---------------- .../src/hwlab-node-web-sentinel-config-ref.ts | 209 ++ scripts/src/hwlab-node-web-sentinel-config.ts | 54 +- .../src/hwlab-node-web-sentinel-p5-observe.ts | 1484 ++++++++ scripts/src/hwlab-node-web-sentinel-p5.ts | 1491 ++++++++ .../src/hwlab-node-web-sentinel-resolver.ts | 10 +- .../src/hwlab-node-web-sentinel-service.ts | 24 +- scripts/src/hwlab-node/control-actions.ts | 34 +- scripts/src/hwlab-node/entry.ts | 24 + scripts/src/hwlab-node/observability.ts | 2 +- scripts/src/hwlab-node/plan.ts | 2 + scripts/src/hwlab-node/public-exposure.ts | 4 + scripts/src/hwlab-node/render.ts | 134 +- scripts/src/hwlab-node/runtime-common.ts | 42 +- scripts/src/hwlab-node/secret-scripts.ts | 58 +- scripts/src/hwlab-node/status.ts | 80 +- scripts/src/hwlab-node/web-observe-scripts.ts | 6 +- scripts/src/hwlab-node/web-probe-observe.ts | 12 +- scripts/src/hwlab-node/web-probe.ts | 187 +- scripts/src/platform-infra-egress-proxy.ts | 4 + scripts/src/platform-infra-host-proxy.ts | 1154 +++++++ scripts/src/platform-infra/manifest.ts | 19 +- scripts/src/provider-attach.ts | 30 +- scripts/src/ssh-file-transfer.ts | 41 +- scripts/web-probe-sentinel-scheduler.ts | 8 +- src/components/provider-gateway/Dockerfile | 2 + 49 files changed, 10909 insertions(+), 5223 deletions(-) create mode 100644 config/hwlab-web-probe-sentinel/profiles.yaml create mode 100644 config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml create mode 100644 config/hwlab-web-probe-sentinel/scenarios.multi-sentinel.yaml create mode 100644 config/platform-infra/host-proxy.yaml create mode 100644 scripts/src/hwlab-node-control-plane-model.ts create mode 100644 scripts/src/hwlab-node-control-plane-runtime.ts create mode 100644 scripts/src/hwlab-node-web-sentinel-config-ref.ts create mode 100644 scripts/src/hwlab-node-web-sentinel-p5-observe.ts create mode 100644 scripts/src/hwlab-node-web-sentinel-p5.ts create mode 100644 scripts/src/platform-infra-host-proxy.ts diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index 02dc5bb7..0adb9a31 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -26,6 +26,7 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ - 源码、配置、部署类正规化默认在独立 `.worktree/` 中做；轻量 skill/docs/reference 收敛可按项目规则直接在主 worktree 做。 - YAML 是 source of truth。不得新增隐藏代码默认值、schema 数值硬限制、合同测试或测试硬编码策略。 - 代码校验只保证字段能被正确读取和渲染：类型、必填、枚举键名、引用存在性。版本号、namespace、endpoint、容量、冷却时间、回退窗口等数值以 YAML 为准。 +- YAML 文件名、YAML 解析函数名和 YAML 渲染函数名不得携带具体 node/lane 名称或版本实例（例如 `JD01`、`D601`、`v03`、`jd01-v03`）。node/lane/version 只能作为 YAML 变量、selector、target key、template value 或 CLI 参数参与渲染；可复用文件和代码入口必须按职责命名。 - 避免“超级配置”。当一个能力同时涉及 target/lane、runtime、scenario、prompt、report、publicExposure、Secret、CI/CD 等不同职责时，按职责拆分到 owning YAML；root YAML 只保存归属和 `configRefs`/path 引用，不承载全部细节。 - 跨 YAML 引用应使用稳定的 `path/to/file.yaml#object.path` 或当前 domain parser 明确支持的等价语法。parser 只解析引用、校验存在性/类型/形状和冲突，不生成隐藏默认值，也不把合并后的大对象写成新的 source of truth。 - CLI `plan/status` 应输出 redacted 配置引用图：每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。 diff --git a/config/agentrun.yaml b/config/agentrun.yaml index b74e1eed..e4c1695c 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -64,6 +64,9 @@ controlPlane: D518: route: D518 kubeRoute: D518:k3s + JD01: + route: JD01 + kubeRoute: JD01:k3s lanes: v01: @@ -212,6 +215,10 @@ controlPlane: image: postgres:16-alpine storage: 5Gi port: 5432 + database: agentrun_v01 + user: agentrun_v01 + passwordSourceRef: agentrun/v01-local-postgres.env + passwordSourceKey: POSTGRES_PASSWORD gitMirror: namespace: devops-infra readService: git-mirror-http @@ -557,6 +564,261 @@ controlPlane: name: agentrun-v01-tool-unidesk-ssh key: UNIDESK_SSH_CLIENT_TOKEN + jd01-v02: + node: JD01 + version: v0.2 + source: + statusMode: k3s-git-mirror + repository: pikasTech/agentrun + branch: v0.2 + bootstrapFromBranch: v0.1 + bootstrapTimeoutSeconds: 900 + bootstrapPollSeconds: 15 + remote: git@github.com:pikasTech/agentrun.git + workspace: /root/workspace/agentrun-v02 + runtime: + namespace: agentrun-v02 + managerDeployment: agentrun-mgr + managerService: agentrun-mgr + managerPort: 8080 + internalBaseUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 + ci: + namespace: agentrun-ci + pipeline: agentrun-jd01-v02-ci-image-publish + pipelineRunPrefix: agentrun-jd01-v02-ci + serviceAccountName: agentrun-jd01-v02-tekton-runner + registryPrefix: 127.0.0.1:5000/agentrun + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless + gitops: + branch: jd01-v0.2-gitops + path: deploy/gitops/node/jd01/runtime-v02 + argoNamespace: argocd + argoApplication: agentrun-jd01-v02 + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + deployment: + format: unidesk-yaml-only + gitopsRoot: deploy/gitops/node/jd01 + runtimeRenderDir: runtime-v02 + artifactCatalogPath: deploy/artifact-catalog.jd01-v02.json + argocd: + project: agentrun-jd01-v02 + applicationFile: application-v02.yaml + manager: + serviceAccount: agentrun-jd01-v02-mgr + apiKeySecretRef: + name: agentrun-v02-api-key + key: HWLAB_API_KEY + env: + AGENTRUN_POSTGRES_POOL_MAX: "4" + AGENTRUN_MANAGER_RECONCILER_BATCH_SIZE: "20" + AGENTRUN_MANAGER_RECONCILER_ENABLED: "true" + AGENTRUN_MANAGER_RECONCILER_INTERVAL_MS: "30000" + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces + OTEL_SERVICE_NAME: agentrun-manager + UNIDESK_NODE_ID: JD01 + HWLAB_RUNTIME_LANE: v0.3 + unideskSshEndpointEnv: + name: UNIDESK_MAIN_SERVER_IP + value: 74.48.78.17 + bootRepoUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + imageBuild: + context: . + containerfile: deploy/container/Containerfile + repository: agentrun-mgr-env + network: host + buildArgs: + BUN_IMAGE: oven/bun:1-alpine + httpProxy: http://127.0.0.1:10808 + httpsProxy: http://127.0.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - hyueapi.com + - .hyueapi.com + buildContainerProxy: + httpProxy: http://127.0.0.1:10808 + httpsProxy: http://127.0.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - hyueapi.com + - .hyueapi.com + envIdentityFiles: + - deploy/container/Containerfile + - deploy/runtime/boot/agentrun-boot.sh + - deploy/runtime/boot/agentrun-mgr.sh + - deploy/runtime/boot/agentrun-runner.sh + - src + - scripts + - package.json + - bun.lock + - tsconfig.json + timeoutSeconds: 1800 + pollSeconds: 15 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 800m + memory: 1Gi + runner: + serviceAccount: agentrun-jd01-v02-runner + jobNamePrefix: agentrun-jd01-v02-runner + idleTimeoutMs: 600000 + backendRetry: + maxAttempts: 5 + initialBackoffMs: 1000 + maxBackoffMs: 30000 + apiKeySecretRef: + name: agentrun-v02-api-key + key: HWLAB_API_KEY + egressProxyUrl: http://127.0.0.1:10808 + noProxyExtra: + - localhost + - 127.0.0.1 + - ::1 + - .svc + - .svc.cluster.local + - .cluster.local + - hyueapi.com + - .hyueapi.com + retention: + maxRunners: 20 + cleanupOrder: oldest-inactive-last-active-first + activeHeartbeatMaxAgeMs: 900000 + selectors: + matchLabels: + app.kubernetes.io/part-of: agentrun + app.kubernetes.io/name: agentrun-runner + app.kubernetes.io/component: runner + jobNamePrefixes: + - agentrun-jd01-v02-runner + - agentrun-v02-runner + - agentrun-v01-runner + ageBasedCleanup: + enabled: false + maxAgeHours: 48 + cancelLifecycle: + deliveryMode: manager-epoch + gracefulAbortMs: 15000 + killEscalationMs: 30000 + staleHeartbeatFencingMs: 900000 + lateWriteFencing: + enabled: true + eventStages: + - accepted + - persisted + - delivered + - aborting + - terminalized + - fenced + - late-write-rejected + localPostgres: + enabled: true + serviceName: agentrun-v02-postgres + image: postgres:16-alpine + storage: 5Gi + port: 5432 + database: agentrun_v02 + user: agentrun_v02 + passwordSourceRef: agentrun/jd01-v02-local-postgres.env + passwordSourceKey: POSTGRES_PASSWORD + gitMirror: + namespace: devops-infra + readService: git-mirror-http + readDeployment: git-mirror-http + writeService: git-mirror-write + writeDeployment: git-mirror-write + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git + cachePvc: hwlab-git-mirror-cache + cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-jd01-v03-git-mirror-cache + sshSecretName: git-mirror-github-ssh + githubProxy: + host: 127.0.0.1 + port: 10808 + toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + syncJobPrefix: git-mirror-agentrun-jd01-v02-sync-manual + flushJobPrefix: git-mirror-agentrun-jd01-v02-flush-manual + repositories: + - key: agentrun + repository: pikasTech/agentrun + sourceBranch: v0.2 + gitopsBranch: jd01-v0.2-gitops + - key: unidesk + repository: pikasTech/unidesk + sourceBranch: master + - key: agent_skills + repository: pikasTech/agent_skills + sourceBranch: master + database: + mode: local-postgres + secretRef: + name: agentrun-v02-mgr-db + key: DATABASE_URL + localPostgresExpectedAbsent: false + secrets: + - id: manager-api-key + sourceRef: /root/.config/hwlab-v03/master-server-admin-api-key.env + sourceKey: HWLAB_API_KEY + targetRef: + namespace: agentrun-v02 + name: agentrun-v02-api-key + key: HWLAB_API_KEY + - id: runner-api-key-legacy-name + sourceRef: /root/.config/hwlab-v03/master-server-admin-api-key.env + sourceKey: HWLAB_API_KEY + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-api-key + key: HWLAB_API_KEY + - id: provider-codex-auth-json + sourceMode: file + sourceRef: /root/.codex/auth.json + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-provider-codex + key: auth.json + providerCredential: + profile: codex + - id: provider-codex-config + sourceMode: file + sourceRef: agentrun/jd01-v02-provider-codex-config.toml + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-provider-codex + key: config.toml + providerCredential: + profile: codex + - id: tool-github-pr-token + sourceRef: /root/.config/unidesk/github.env + sourceKey: GH_TOKEN + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-tool-github-pr + key: GH_TOKEN + - id: tool-unidesk-ssh-token + sourceRef: /root/unidesk/.state/docker-compose.env + sourceKey: UNIDESK_SSH_CLIENT_TOKEN + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-tool-unidesk-ssh + key: UNIDESK_SSH_CLIENT_TOKEN + d518-v02: node: D518 version: v0.2 diff --git a/config/deploy-ssh-identities.yaml b/config/deploy-ssh-identities.yaml index 41744297..7cb7667a 100644 --- a/config/deploy-ssh-identities.yaml +++ b/config/deploy-ssh-identities.yaml @@ -49,6 +49,14 @@ targets: mode: direct identities: - github.com + JD01: + providerId: JD01 + route: JD01 + homeDir: /root + egress: + mode: direct + identities: + - github.com G14: providerId: G14 route: G14 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 29a687d1..e9a5e822 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -8,6 +8,7 @@ metadata: - 1010 - 1119 - 1148 + - 1234 imagePolicy: requireReproducibleBuildSource: true forbidPrivateOrNodeLocalImagesAsInputs: true @@ -152,6 +153,104 @@ nodes: - 74.48.78.17 - hyueapi.com - .hyueapi.com + JD01: + route: JD01 + kubeRoute: JD01:k3s + k3s: + serviceName: k3s + dropInPath: /etc/systemd/system/k3s.service.d/20-unidesk-node-config.conf + nodeStatusName: jd01 + execStartPre: [] + install: + enabled: true + channel: stable + version: v1.36.2+k3s1 + installScriptUrl: https://get.k3s.io + binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s + sha256Url: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/sha256sum-amd64.txt + expectedSha256: 65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de + hostProxyConfigRef: config/platform-infra/host-proxy.yaml#targets.JD01 + proxyEnvPath: /etc/unidesk/proxy.env + registriesYamlPath: /etc/rancher/k3s/registries.yaml + localRegistry: + containerName: registry + image: docker.m.daocloud.io/library/registry:2 + canonicalImage: registry:2 + bind: 127.0.0.1:5000:5000 + state: + dir: /root/.unidesk/k3s-install + logPath: /root/.unidesk/k3s-install/install.log + statusPath: /root/.unidesk/k3s-install/status.json + downloads: + connectTimeoutSeconds: 15 + maxTimeSeconds: 1200 + retry: 8 + retryDelaySeconds: 5 + serverArgs: + - server + - --disable + - traefik + - --disable + - servicelb + - --disable + - metrics-server + - --node-name + - JD01 + - --node-label + - unidesk.ai/node-id=JD01 + - --node-label + - unidesk.ai/provider-id=JD01 + - --tls-san + - 127.0.0.1 + - --tls-san + - host.docker.internal + - --write-kubeconfig-mode + - "644" + - --kubelet-arg + - image-gc-high-threshold=95 + - --kubelet-arg + - image-gc-low-threshold=90 + - --kubelet-arg + - max-pods=500 + kubelet: + maxPods: 500 + registry: + endpoint: 127.0.0.1:5000 + egressProxy: + mode: host-route + clientName: jd01-host-proxy + hostProxyConfigRef: config/platform-infra/host-proxy.yaml#targets.JD01 + proxyEnvPath: /etc/unidesk/proxy.env + proxyUrl: http://10.42.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - argocd-repo-server + - argocd-repo-server.argocd + - argocd-redis + - argocd-redis.argocd + - git-mirror-http + - git-mirror-http.devops-infra + - git-mirror-write + - git-mirror-write.devops-infra + - 10.0.0.0/8 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + - hyueapi.com + - .hyueapi.com targets: - id: d601-v03 @@ -174,6 +273,8 @@ targets: cachePvcStorage: 20Gi cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d601-v03-git-mirror-cache servicePort: 8080 + readContainerPort: 8080 + writeContainerPort: 8080 deploymentReplicas: 1 secretName: git-mirror-github-ssh syncConfigMapName: git-mirror-sync-script @@ -184,6 +285,8 @@ targets: egressProxy: mode: node-global required: true + podHostNetwork: false + injectPodEnv: false githubTransport: mode: ssh privateKeySecretKey: ssh-privatekey @@ -195,6 +298,29 @@ targets: knownHostsSourceKey: GITHUB_KNOWN_HOSTS_B64 knownHostsSourceEncoding: base64 tekton: + install: + enabled: true + sourceKind: url + version: pipeline-v1.12.0-triggers-v0.34.0 + fieldManager: unidesk-hwlab-node-tekton + manifests: + - name: pipeline + url: https://infra.tekton.dev/tekton-releases/pipeline/previous/v1.12.0/release.yaml + - name: triggers + url: https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/release.yaml + - name: triggers-interceptors + url: https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/interceptors.yaml + requiredCrds: + - pipelines.tekton.dev + - pipelineruns.tekton.dev + - tasks.tekton.dev + - taskruns.tekton.dev + expectedDeploymentNamespaces: + - tekton-pipelines + - tekton-pipelines-resolvers + readinessTimeoutSeconds: 900 + runtimeProxy: + enabled: false pipelineName: hwlab-d601-v03-ci-image-publish serviceAccountName: hwlab-d601-v03-tekton-runner pipelineRunPrefix: hwlab-d601-v03-ci-poll @@ -319,6 +445,218 @@ targets: expectedStatefulSets: - argocd-application-controller readinessTimeoutSeconds: 600 + runtimeProxy: + enabled: false + - id: jd01-v03 + node: JD01 + lane: v03 + enabled: true + ciNamespace: hwlab-ci + runtimeNamespace: hwlab-v03 + source: + repository: pikasTech/HWLAB + branch: v0.3 + gitops: + branch: v0.3-gitops + path: deploy/gitops/node/jd01/runtime-v03 + gitMirror: + namespace: devops-infra + serviceReadName: git-mirror-http + serviceWriteName: git-mirror-write + cachePvcName: hwlab-git-mirror-cache + cachePvcStorage: 20Gi + cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-jd01-v03-git-mirror-cache + servicePort: 8080 + readContainerPort: 8080 + writeContainerPort: 8081 + deploymentReplicas: 1 + secretName: git-mirror-github-ssh + syncConfigMapName: git-mirror-sync-script + syncJobPrefix: git-mirror-hwlab-jd01-v03-sync-manual + flushJobPrefix: git-mirror-hwlab-jd01-v03-flush-manual + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + egressProxy: + mode: host-route + required: true + podHostNetwork: false + injectPodEnv: true + githubTransport: + mode: ssh + privateKeySecretKey: ssh-privatekey + privateKeySourceRef: github/hwlab-git-mirror-ssh.env + privateKeySourceKey: GITHUB_SSH_PRIVATE_KEY_B64 + privateKeySourceEncoding: base64 + knownHostsSecretKey: known_hosts + knownHostsSourceRef: github/hwlab-git-mirror-ssh.env + knownHostsSourceKey: GITHUB_KNOWN_HOSTS_B64 + knownHostsSourceEncoding: base64 + tekton: + install: + enabled: true + sourceKind: url + version: pipeline-v1.12.0-triggers-v0.34.0 + fieldManager: unidesk-hwlab-node-tekton + manifests: + - name: pipeline + url: https://infra.tekton.dev/tekton-releases/pipeline/previous/v1.12.0/release.yaml + - name: triggers + url: https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/release.yaml + - name: triggers-interceptors + url: https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/interceptors.yaml + requiredCrds: + - pipelines.tekton.dev + - pipelineruns.tekton.dev + - tasks.tekton.dev + - taskruns.tekton.dev + expectedDeploymentNamespaces: + - tekton-pipelines + - tekton-pipelines-resolvers + readinessTimeoutSeconds: 900 + runtimeProxy: + enabled: false + pipelineName: hwlab-jd01-v03-ci-image-publish + serviceAccountName: hwlab-jd01-v03-tekton-runner + pipelineRunPrefix: hwlab-jd01-v03-ci-poll + gitWorkspaceSecret: + name: hwlab-git-ssh + namespace: hwlab-ci + sourceRefFrom: gitMirror.githubTransport + privateKeySecretKey: ssh-privatekey + knownHostsSecretKey: known_hosts + runtimeObserverRbac: + namespace: hwlab-v03 + roleName: hwlab-jd01-v03-runtime-observer + roleBindingName: hwlab-jd01-v03-runtime-observer + argoObserverRbac: + namespace: argocd + roleName: hwlab-jd01-v03-argo-observer + roleBindingName: hwlab-jd01-v03-argo-observer + toolsImage: + output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: Always + sourceKind: dockerfile + context: . + dockerfileInline: + filename: hwlab-ci-node-tools.public.Dockerfile + lines: + - FROM docker.io/oven/bun:1.3.13 AS bun-runtime + - FROM docker.io/docker:29-cli AS docker-cli + - FROM docker.io/library/golang:1.24-bookworm AS golang-toolchain + - FROM docker.io/library/node:22-bookworm-slim AS node-runtime + - FROM docker.io/library/python:3.12-bookworm + - ARG HTTP_PROXY + - ARG HTTPS_PROXY + - ARG ALL_PROXY + - ARG NO_PROXY + - ARG http_proxy + - ARG https_proxy + - ARG all_proxy + - ARG no_proxy + - COPY --from=golang-toolchain /usr/local/go /usr/local/go + - COPY --from=node-runtime /usr/local/bin/node /usr/local/bin/node + - COPY --from=node-runtime /usr/local/lib/node_modules /usr/local/lib/node_modules + - COPY --from=bun-runtime /usr/local/bin/bun /usr/local/bin/bun + - COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker + - ENV PATH=/usr/local/go/bin:$PATH + - RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx + - RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx + - ENV HWLAB_CI_NODE_DEPS=/opt/hwlab-ci-node-deps/node_modules + - RUN set -eu; export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}"; export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$HTTP_PROXY}}"; export ALL_PROXY="${ALL_PROXY:-${all_proxy:-}}"; export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"; export http_proxy="$HTTP_PROXY"; export https_proxy="$HTTPS_PROXY"; export all_proxy="$ALL_PROXY"; export no_proxy="$NO_PROXY"; export npm_config_registry="https://registry.npmmirror.com/"; export BUN_CONFIG_REGISTRY="https://registry.npmmirror.com/"; export npm_config_noproxy="$NO_PROXY"; if [ -n "$HTTP_PROXY" ]; then export npm_config_proxy="$HTTP_PROXY"; fi; if [ -n "$HTTPS_PROXY" ]; then export npm_config_https_proxy="$HTTPS_PROXY"; fi; export npm_config_fetch_retries=2; export npm_config_fetch_retry_mintimeout=2000; export npm_config_fetch_retry_maxtimeout=16000; export npm_config_fetch_timeout=120000; proxy_label="${HTTP_PROXY:+HTTP_PROXY}"; proxy_label="${proxy_label:-none}"; mkdir -p /opt/hwlab-ci-node-deps; cd /opt/hwlab-ci-node-deps; printf '{"private":true,"dependencies":{}}\n' > package.json; ok=0; delay=2; for attempt in 1 2 3 4 5; do echo "{\"event\":\"tools-yaml-node-npm-install\",\"attempt\":\"$attempt/5\",\"registry\":\"$npm_config_registry\",\"proxy\":\"$proxy_label\"}" >&2; if timeout 180s npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3; then ok=1; break; fi; if [ "$attempt" = 5 ]; then break; fi; echo "{\"event\":\"tools-yaml-node-npm-install\",\"status\":\"retrying\",\"attempt\":\"$attempt/5\",\"sleepSeconds\":$delay}" >&2; sleep "$delay"; delay=$((delay * 2)); done; test "$ok" = 1; node --input-type=module -e 'import("/opt/hwlab-ci-node-deps/node_modules/yaml/browser/dist/index.js").then((yaml)=>console.log("yaml-ok", typeof yaml.parse))' + - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V && go version + buildArgs: {} + buildNetwork: host + publicBaseImages: + - docker.io/library/node:22-bookworm-slim + - docker.io/library/golang:1.24-bookworm + - docker.io/oven/bun:1.3.13 + - docker.io/buildpack-deps:bookworm-scm + - docker.io/library/python:3.12-bookworm + - docker.io/docker:29-cli + buildOwner: JD01 + buildMode: node-local + ciBuildBenchmarks: + - profile: no-mirror-full + runtimeLaneConfigRef: config/hwlab-node-lanes.yaml#lanes.v03.targets.JD01 + pipelineRunPrefix: hwlab-jd01-v03-ci-bench + catalogPathTemplate: .unidesk/ci-build-benchmark/{profile}/{pipelineRun}/artifact-catalog.json + imageTagMode: full + pipelineTimeoutSeconds: 7200 + cachePolicy: + noPipelineRunReuse: true + forceFullBuild: true + forbidGitopsCatalogReuse: true + forbidDependencyCache: true + forbidBuildkitCache: true + forbidRegistryMirror: true + forbidLocalPreheatedImages: true + timings: + requiredStages: + - source-fetch + - dependency-install + - base-image-pull + - service-image-build + - registry-push + - pipeline-total + failureFamilies: + - dns + - proxy-connect + - tls-timeout + - rate-limit + - auth + - cache-hit-forbidden + - image-policy + - build-script + - registry-push + argo: + namespace: argocd + projectName: hwlab-jd01 + applicationName: hwlab-node-v03 + applicationFile: application-v03.yaml + install: + enabled: true + sourceKind: url + version: v3.4.2 + manifestUrl: https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.2/manifests/install.yaml + fieldManager: unidesk-hwlab-node-argocd + imagePullPolicy: IfNotPresent + preloadImages: + - 127.0.0.1:5000/hwlab/argocd:v3.4.2 + - 127.0.0.1:5000/hwlab/dex:v2.45.0 + - 127.0.0.1:5000/hwlab/redis:8.2.3-alpine + imageRewrites: + - source: quay.io/argoproj/argocd:v3.4.2 + pullImage: quay.m.daocloud.io/argoproj/argocd:v3.4.2 + target: 127.0.0.1:5000/hwlab/argocd:v3.4.2 + - source: ghcr.io/dexidp/dex:v2.45.0 + pullImage: ghcr.m.daocloud.io/dexidp/dex:v2.45.0 + target: 127.0.0.1:5000/hwlab/dex:v2.45.0 + - source: public.ecr.aws/docker/library/redis:8.2.3-alpine + pullImage: docker.m.daocloud.io/library/redis:8.2.3-alpine + target: 127.0.0.1:5000/hwlab/redis:8.2.3-alpine + requiredCrds: + - applications.argoproj.io + - appprojects.argoproj.io + expectedDeployments: + - argocd-applicationset-controller + - argocd-dex-server + - argocd-notifications-controller + - argocd-redis + - argocd-repo-server + - argocd-server + expectedStatefulSets: + - argocd-application-controller + readinessTimeoutSeconds: 600 + runtimeProxy: + enabled: true + mode: host-route + configRef: nodes.JD01.egressProxy + hostNetwork: false + injectEnv: true + deployments: + - argocd-repo-server + statefulSets: + - argocd-application-controller - id: d518-v03 node: D518 lane: v03 @@ -339,6 +677,8 @@ targets: cachePvcStorage: 20Gi cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d518-v03-git-mirror-cache servicePort: 8080 + readContainerPort: 8080 + writeContainerPort: 8080 deploymentReplicas: 1 secretName: git-mirror-github-ssh syncConfigMapName: git-mirror-sync-script @@ -349,6 +689,8 @@ targets: egressProxy: mode: node-global required: true + podHostNetwork: false + injectPodEnv: false githubTransport: mode: ssh privateKeySecretKey: ssh-privatekey @@ -360,6 +702,29 @@ targets: knownHostsSourceKey: GITHUB_KNOWN_HOSTS_B64 knownHostsSourceEncoding: base64 tekton: + install: + enabled: true + sourceKind: url + version: pipeline-v1.12.0-triggers-v0.34.0 + fieldManager: unidesk-hwlab-node-tekton + manifests: + - name: pipeline + url: https://infra.tekton.dev/tekton-releases/pipeline/previous/v1.12.0/release.yaml + - name: triggers + url: https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/release.yaml + - name: triggers-interceptors + url: https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/interceptors.yaml + requiredCrds: + - pipelines.tekton.dev + - pipelineruns.tekton.dev + - tasks.tekton.dev + - taskruns.tekton.dev + expectedDeploymentNamespaces: + - tekton-pipelines + - tekton-pipelines-resolvers + readinessTimeoutSeconds: 900 + runtimeProxy: + enabled: false pipelineName: hwlab-d518-v03-ci-image-publish serviceAccountName: hwlab-d518-v03-tekton-runner pipelineRunPrefix: hwlab-d518-v03-ci-poll @@ -492,3 +857,5 @@ targets: expectedStatefulSets: - argocd-application-controller readinessTimeoutSeconds: 600 + runtimeProxy: + enabled: false diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index ef669e7e..4a0cc7d8 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -34,6 +34,13 @@ nodes: gitopsRoot: deploy/gitops/node networkProfile: d518-node-ci-egress downloadProfile: d518-node-default + JD01: + route: JD01 + kubeRoute: JD01:k3s + sourceWorkspace: /root/workspace/hwlab-v03 + gitopsRoot: deploy/gitops/node + networkProfile: jd01-node-ci-egress + downloadProfile: jd01-node-default lanes: v02: @@ -76,7 +83,7 @@ lanes: apiUrl: http://74.48.78.17:19667 v03: node: G14 - activeTarget: D518 + activeTarget: JD01 minor: 3 version: v0.3 sourceBranch: v0.3 @@ -773,6 +780,198 @@ lanes: envKey: DATASTORE_URI authnKey: authn-preshared-key role: hwlab_d518_v03_app + JD01: + node: JD01 + workspace: /root/workspace/hwlab-v03 + sourceWorkspace: + requiredCommands: + - git + - node + - npm + - npx + requiredFiles: + - AGENTS.md + - package.json + - package-lock.json + - scripts/src/browser-launcher.mjs + - scripts/web-live-dom-probe.mjs + install: + dependencyCommand: npm ci + browserCommand: PLAYWRIGHT_BROWSERS_PATH=0 npx playwright install chromium + timeoutSeconds: 900 + cicdRepo: /root/workspace/hwlab-v03-cicd.git + cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock + app: hwlab-node-v03 + pipeline: hwlab-jd01-v03-ci-image-publish + pipelineRunPrefix: hwlab-jd01-v03-ci-poll + serviceAccountName: hwlab-jd01-v03-tekton-runner + controlPlaneFieldManager: unidesk-hwlab-jd01-v03-control-plane + git: + url: git@github.com:pikasTech/HWLAB.git + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + argo: + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + gitopsBranch: v0.3-gitops + catalogPath: deploy/artifact-catalog.jd01-v03.json + runtime: + path: deploy/gitops/node/jd01/runtime-v03 + namespace: hwlab-v03 + renderDir: runtime-v03 + runtimeStore: + postgres: + mode: local-k3s + secretName: hwlab-v03-postgres + statefulSet: hwlab-v03-postgres + serviceName: hwlab-v03-postgres + adminUser: hwlab_v03 + adminPasswordSourceRef: hwlab/jd01-v03-postgres.env + adminPasswordSourceKey: HWLAB_V03_POSTGRES_PASSWORD + cloudApi: + secretName: hwlab-cloud-api-v03-db + secretKey: database-url + database: hwlab_v03 + role: hwlab_v03 + openfga: + secretName: hwlab-v03-openfga + secretKey: datastore-uri + authnKey: authn-preshared-key + postgresPasswordKey: postgres-password + database: hwlab_openfga + role: hwlab_openfga + poolMax: 16 + connectionTimeoutMs: 5000 + queryRetryMaxAttempts: 5 + queryRetryInitialDelayMs: 250 + queryRetryMaxDelayMs: 5000 + webProbe: + browserProxyMode: direct + defaultOrigin: + mode: public + baseUrl: https://hwlab.pikapython.com + authLogin: + maxAttempts: 6 + requestTimeoutMs: 30000 + initialDelayMs: 500 + maxDelayMs: 10000 + alertThresholds: + sameOriginApiSlowMs: 10000 + partialApiSlowMs: 10000 + longLivedStreamOpenSlowMs: 10000 + visibleLoadingSlowMs: 10000 + turnTimingSampleSlackSeconds: 3 + turnElapsedSevereTimeoutSeconds: 120 + uncommandedStateChangeCommandWindowMs: 10000 + scrollJumpCommandWindowMs: 8000 + scrollJumpFromY: 250 + scrollJumpToY: 40 + sessionRailFallbackRatio: 0.5 + tektonDir: tekton-v03 + argoApplicationFile: application-v03.yaml + registryPrefix: 127.0.0.1:5000/hwlab + baseImage: 127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim + baseImageSource: node:20-bookworm-slim + serviceIds: + - hwlab-cloud-api + - hwlab-workbench-runtime + - hwlab-user-billing + - hwlab-project-management + - hwlab-cloud-web + - hwlab-gateway + - hwlab-edge-proxy + - hwlab-agent-skills + buildkit: + sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless + sourceImage: docker.io/moby/buildkit:rootless + stepEnv: + HOME: /tekton/home + XDG_CONFIG_HOME: /tekton/home/.config + observability: + prometheusOperator: false + webProbe: + sentinels: + - id: jd01-web-probe-sentinel + enabled: true + configRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.sentinel + runtimeImageRewrites: + - source: postgres:16-alpine + target: 127.0.0.1:5000/hwlab/postgres:16-alpine + - source: fatedier/frpc:v0.68.1 + target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 + - source: openfga/openfga:v1.17.0 + target: 127.0.0.1:5000/hwlab/openfga:v1.17.0 + runtimeImageBuilds: + - id: moonbridge + kind: moonbridge + target: 127.0.0.1:5000/hwlab/moonbridge:1b99888d3dae + sourceRepo: https://github.com/ZhiYi-R/moon-bridge.git + sourceRef: 1b99888d3dae889b79ee602cb875c7907f7e76f2 + builderImage: golang:1.25-bookworm + goProxy: https://goproxy.cn,direct + dockerNetworkMode: host + public: + webUrl: https://hwlab.pikapython.com + apiUrl: https://hwlab.pikapython.com + bootstrapAdmin: + username: admin + displayName: HWLAB v0.3 Admin + usernameSourceRef: .env/HWLAB_admin.txt + usernameSourceLine: 1 + passwordSourceRef: .env/HWLAB_admin.txt + passwordSourceKey: HWLAB_ADMIN_PASSWORD + passwordSourceLine: 2 + passwordHashTransform: hwlab-sha256 + secretName: hwlab-v03-bootstrap-admin + secretKey: password-hash + rollout: + deployment: hwlab-cloud-api + codeAgentProvider: + secretName: hwlab-v03-code-agent-provider + sourceRef: hwlab/jd01-v03-code-agent-provider.env + openaiSourceKey: OPENAI_API_KEY + opencodeSourceKey: OPENCODE_API_KEY + codeAgentRuntime: + enabled: true + adapter: agentrun-v02 + managerUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 + apiKeySecretName: hwlab-v03-master-server-admin-api-key + apiKeySecretKey: api-key + runnerNamespace: agentrun-v02 + secretNamespace: agentrun-v02 + repoUrlFrom: runtimeGitReadUrl + providerIdFrom: runtimeNodeId + defaultProviderProfile: codex + codexStdioSupervisor: repo-owned + publicExposure: + mode: pk01-caddy-frp + publicBaseUrl: https://hwlab.pikapython.com + hostname: hwlab.pikapython.com + expectedA: 82.156.23.220 + frpc: + serverAddr: 82.156.23.220 + serverPort: 22000 + tokenSourceRef: platform-infra/pk01-frp.env + tokenSourceKey: FRP_TOKEN + secretName: hwlab-v03-frpc-secrets + secretKey: frpc.toml + tokenKey: token + webProxy: + name: hwlab-jd01-v03-cloud-web + remotePort: 22090 + localIP: hwlab-cloud-web.hwlab-v03.svc.cluster.local + localPort: 8080 + apiProxy: + name: hwlab-jd01-v03-edge-proxy + remotePort: 22089 + localIP: hwlab-edge-proxy.hwlab-v03.svc.cluster.local + localPort: 6667 + caddy: + route: PK01 + configPath: /etc/caddy/Caddyfile + serviceName: caddy + email: ops@pikapython.com + tls: auto + responseHeaderTimeoutSeconds: 600 networkProfiles: node-ci-egress: @@ -916,6 +1115,43 @@ networkProfiles: - .svc - .svc.cluster.local - .cluster.local + jd01-node-ci-egress: + proxy: + http: http://127.0.0.1:10808 + https: http://127.0.0.1:10808 + all: http://127.0.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 10.0.0.0/8 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + dockerBuildProxy: + http: http://127.0.0.1:10808 + https: http://127.0.0.1:10808 + all: http://127.0.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local downloadProfiles: node-default: @@ -978,6 +1214,26 @@ downloadProfiles: retries: 3 connectTimeoutSeconds: 10 maxTimeSeconds: 120 + jd01-node-default: + git: + proxyMode: inherit + retries: 3 + timeoutSeconds: 60 + npm: + registry: https://registry.npmmirror.com/ + retries: 3 + fetchTimeoutSeconds: 120 + pip: + indexUrl: https://pypi.org/simple + retries: 3 + timeoutSeconds: 120 + docker: + registryMirrors: [] + pullRetries: 3 + curl: + retries: 3 + connectTimeoutSeconds: 10 + maxTimeSeconds: 120 d601-node-no-mirror-benchmark: git: proxyMode: inherit diff --git a/config/hwlab-test-accounts.yaml b/config/hwlab-test-accounts.yaml index a2c74882..bf8c7e68 100644 --- a/config/hwlab-test-accounts.yaml +++ b/config/hwlab-test-accounts.yaml @@ -87,3 +87,79 @@ targets: targetKey: api-key scopes: - api + + - id: jd01-v03 + node: JD01 + lane: v03 + namespace: hwlab-v03 + publicUrl: https://hwlab.pikapython.com + userBilling: + serviceId: hwlab-user-billing + databaseUrlSource: + sourceRef: hwlab/jd01-v03-postgres.env + sourceKey: DATABASE_URL + accounts: + - logicalId: jd01-v03-admin + kind: bootstrap-admin-api-key + userId: usr_v03_admin + username: admin + displayName: HWLAB v0.3 Admin + role: admin + status: active + permissions: + - admin + - system:hwlab + - "tool:*" + sourceRef: hwlab/jd01-v03-admin.env + sourceKey: HWLAB_API_KEY + createIfMissing: + enabled: true + randomBase64Url: + bytes: 32 + prefix: hwl_live_ + hostEnvTargets: + - path: /root/.config/hwlab-v03/master-server-admin-api-key.env + envKey: HWLAB_API_KEY + mode: "0600" + target: + kind: kubernetes-secret + namespace: hwlab-v03 + secretName: hwlab-v03-master-server-admin-api-key + targetKey: api-key + rolloutDeployment: hwlab-cloud-api + - logicalId: jd01-v03-inner-test + kind: user-billing-api-key + userId: usr_jd01_v03_inner_test + username: inner-test + email: inner-test+jd01@hwlab.local + displayName: HWLAB v0.3 JD01 Test User + role: user + status: active + planId: default + initialCredits: 100 + permissions: + - code_agent + - hwpod + - aipod + workbench: + projectId: prj_hwpod_workbench + lane: v03 + sourceRef: hwlab/jd01-v03-inner-test.env + sourceKey: HWLAB_API_KEY + createIfMissing: + enabled: true + randomBase64Url: + bytes: 32 + prefix: hwl_live_ + hostEnvTargets: + - path: /root/.config/hwlab-v03/inner-test-api-key.env + envKey: HWLAB_API_KEY + mode: "0600" + target: + kind: user-billing-api-key + serviceId: hwlab-user-billing + keyId: key_jd01_v03_inner_test + keyName: JD01 v0.3 inner test fixed key + targetKey: api-key + scopes: + - api diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml new file mode 100644 index 00000000..19aad256 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -0,0 +1,240 @@ +version: 1 +kind: HwlabWebProbeSentinelProfiles +metadata: + id: hwlab-web-probe-sentinel-profiles + owner: UniDesk + specRef: PJ2026-01060508 + composition: + mode: yaml-anchors-and-merge + intent: node overlays inherit common web-probe sentinel baselines and render node/lane identity from variables. + +baselines: + sentinel: &sentinel-base + enabled: true + mode: web-probe-observe-wrapper + + runtime: + common: &runtime-common + namespace: hwlab-${lane} + listenHost: 0.0.0.0 + servicePort: 8080 + pvcStorage: 10Gi + replicas: 1 + healthPath: /api/health + metricsPath: /metrics + scheduler: &scheduler-10m + intervalMs: 600000 + heartbeatStaleSeconds: 900 + maxConcurrentRuns: 1 + scheduler15m: &scheduler-15m + intervalMs: 900000 + heartbeatStaleSeconds: 900 + maxConcurrentRuns: 1 + sqlite: &sqlite-common + busyTimeoutMs: 2000 + + cicd: + source: &cicd-source + repository: pikasTech/unidesk + branch: master + gitSshUrl: ssh://git@ssh.github.com:443/pikasTech/unidesk.git + gitMirrorReadUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/unidesk.git + buildContext: . + entrypoint: scripts/web-probe-sentinel-service.ts + checkoutPaths: + - scripts + - config + - config.json + - src + - package.json + - bun.lock + - bun.lockb + builder: &cicd-builder + namespace: devops-infra + sourceMode: sparse-git-checkout + gitSshSecretName: git-mirror-github-ssh + dockerSocketPath: /var/run/docker.sock + activeDeadlineSeconds: 900 + ttlSecondsAfterFinished: 3600 + monitorWeb: &monitor-web + frontendStack: vue3-vendored-browser-build + runtimeMode: runner-served-bridge + assetRoot: scripts/assets/web-probe-sentinel-monitor-web + envReuse: + mode: docker-layer-and-ci-node-deps + nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules + gitMirror: + source: source.gitMirrorReadUrl + preSync: required + postFlush: required + ciBudget: + maxSeconds: 120 + maintenance: &maintenance + startCommand: sentinel maintenance start + stopCommand: sentinel maintenance stop + confirmWait: &confirm-wait + maxSeconds: 120 + + publicExposure: + common: &public-exposure-common + enabled: true + mode: pk01-caddy-frp-path + hostname: monitor.pikapython.com + expectedA: 82.156.23.220 + frpc: &frpc-common + image: 127.0.0.1:5000/hwlab/frpc:v0.68.1 + serverAddr: 82.156.23.220 + serverPort: 22000 + tokenSourceRef: platform-infra/pk01-frp.env + tokenSourceKey: FRP_TOKEN + secretKey: frpc.toml + tokenKey: token + caddy: &caddy-common + route: PK01 + configPath: /etc/caddy/Caddyfile + serviceName: caddy + email: ops@pikapython.com + tls: auto + responseHeaderTimeoutSeconds: 600 + + secrets: + jd01BootstrapSource: &jd01-bootstrap-source + purpose: bootstrap-admin + sourceRef: .env/HWLAB_admin.txt + sourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD + sourceLine: 2 + dsflashPromptSource: &dsflash-prompt-source + purpose: prompt-set + sourceRef: hwlab/web-probe-sentinel-dsflash-go.env + sourceKey: DSFLASH_GO_TOOL_CALL_10X_PROMPTS_JSON + frpTokenSource: &frp-token-source + purpose: frp-token + sourceRef: platform-infra/pk01-frp.env + sourceKey: FRP_TOKEN + +nodes: + JD01: + target: &jd01-target + node: ${NODE} + lane: ${LANE} + publicOriginRef: config/hwlab-node-lanes.yaml#lanes.${LANE}.targets.${NODE}.public.webUrl + + cicdCommon: &jd01-cicd-common + controlPlaneConfigRef: config/hwlab-node-control-plane.yaml#targets[1] + source: + <<: *cicd-source + argo: &jd01-argo + namespace: argocd + projectName: hwlab-jd01 + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + targetRevision: v0.3-gitops + maintenance: + <<: *maintenance + monitorWeb: + <<: *monitor-web + confirmWait: + <<: *confirm-wait + + sentinels: + jd01-web-probe-sentinel: + sentinel: + <<: *sentinel-base + id: jd01-web-probe-sentinel + configRefs: + runtime: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.runtime + scenarios: config/hwlab-web-probe-sentinel/scenarios.multi-sentinel.yaml#sentinel.scenarios + promptSet: config/hwlab-web-probe-sentinel/prompt-set.dsflash-go.yaml#sentinel.promptSet + reportViews: config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml#sentinel.reportViews + publicExposure: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.publicExposure + cicd: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.cicd + secrets: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.secrets + runtime: + <<: *runtime-common + target: + <<: *jd01-target + observeWrapperRef: config/hwlab-node-lanes.yaml#lanes.${LANE}.targets.${NODE}.observability.webProbe.sentinels[0] + serviceAccountName: hwlab-web-probe-sentinel-${nodeLower} + deploymentName: hwlab-web-probe-sentinel-${nodeLower} + serviceName: hwlab-web-probe-sentinel-${nodeLower} + pvcName: hwlab-web-probe-sentinel-${nodeLower}-state + stateRoot: /var/lib/web-probe-sentinel-${nodeLower} + imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel-${nodeLower}:source-commit + scheduler: + <<: *scheduler-10m + sqlite: + <<: *sqlite-common + path: /var/lib/web-probe-sentinel-${nodeLower}/index.sqlite + publicExposure: + <<: *public-exposure-common + publicBaseUrl: https://monitor.pikapython.com/sentinels/${nodeLower}-web-probe-sentinel + routePrefix: /sentinels/${nodeLower}-web-probe-sentinel + frpc: + <<: *frpc-common + deploymentName: hwlab-web-probe-sentinel-${nodeLower}-frpc + secretName: hwlab-web-probe-sentinel-${nodeLower}-frpc + httpProxy: + name: hwlab-${nodeLower}-${lane}-web-probe-sentinel + remotePort: 22094 + localIP: hwlab-web-probe-sentinel-${nodeLower}.hwlab-${lane}.svc.cluster.local + localPort: 8080 + caddy: + <<: *caddy-common + managedBlockOwner: hwlab-web-probe-sentinel-${nodeLower}-${lane} + cicd: + <<: *jd01-cicd-common + builder: + <<: *cicd-builder + jobPrefix: web-probe-sentinel-${nodeLower}-publish + gitopsPath: deploy/gitops/node/${nodeLower}/web-probe-sentinel + argo: + <<: *jd01-argo + applicationName: hwlab-web-probe-sentinel-${nodeLower} + image: + repository: 127.0.0.1:5000/hwlab/web-probe-sentinel-${nodeLower} + tagSource: source-commit + baseImageRef: config/hwlab-node-control-plane.yaml#targets[1].tekton.toolsImage.output + envRecipeRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.runtime + targetValidation: + scenarioId: workbench-dsflash-go-tool-call-10x + maxSeconds: 300 + serviceUnavailablePolicy: structured-failure + secrets: + sources: + - <<: *jd01-bootstrap-source + - <<: *dsflash-prompt-source + - purpose: account-a + sourceRef: .env/HWLAB_admin.txt + sourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD + sourceLine: 2 + format: web-account-json + usernameSourceRef: .env/HWLAB_admin.txt + usernameSourceLine: 1 + - purpose: account-b + sourceRef: hwlab/${nodeLower}-${lane}-preset-users.env + sourceKey: ${NODE}_SECOND_USER_PASSWORD + format: web-account-json + username: ${nodeLower}-sentinel@hwlab.local + - <<: *frp-token-source + runtimeSecrets: + - name: hwlab-web-probe-sentinel-${nodeLower}-bootstrap + namespace: hwlab-${lane} + data: + - sourcePurpose: bootstrap-admin + targetKey: bootstrap-admin-password + - name: hwlab-web-probe-sentinel-${nodeLower}-prompt-set + namespace: hwlab-${lane} + data: + - sourcePurpose: prompt-set + targetKey: prompts.json + - name: hwlab-web-probe-sentinel-${nodeLower}-accounts + namespace: hwlab-${lane} + data: + - sourcePurpose: account-a + targetKey: account-a.json + - sourcePurpose: account-b + targetKey: account-b.json + - name: hwlab-web-probe-sentinel-${nodeLower}-frpc + namespace: hwlab-${lane} + data: + - sourcePurpose: frp-token + targetKey: token diff --git a/config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml b/config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml new file mode 100644 index 00000000..88be6945 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml @@ -0,0 +1,24 @@ +version: 1 +kind: HwlabWebProbeSentinelReportViews +metadata: + id: web-probe-sentinel-multi-scenario-report-views + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + reportViews: + defaultView: summary + views: + - summary + - turn-summary + - auth-session-switch-summary + - findings + - trace-frame + pageSize: 20 + maxPageSize: 100 + rawAccess: explicit-only + checkCatalogRef: config/hwlab-web-probe-sentinel/check-catalog.yaml#sentinel.checkCatalog + redaction: + prompt: hash-and-byte-count + assistantFinal: summary-and-hash + providerPayload: denied + secrets: denied diff --git a/config/hwlab-web-probe-sentinel/scenarios.multi-sentinel.yaml b/config/hwlab-web-probe-sentinel/scenarios.multi-sentinel.yaml new file mode 100644 index 00000000..4c9d9e60 --- /dev/null +++ b/config/hwlab-web-probe-sentinel/scenarios.multi-sentinel.yaml @@ -0,0 +1,126 @@ +version: 1 +kind: HwlabWebProbeSentinelScenarios +metadata: + id: web-probe-sentinel-multi-scenario-suite + owner: UniDesk + specRef: PJ2026-01060508 +sentinel: + scenarios: + - id: workbench-dsflash-go-tool-call-10x + enabled: true + cadence: 10m + observeTargetPath: /workbench + sampleIntervalMs: 1000 + screenshotIntervalMs: 60000 + maxRunSeconds: 1200 + providerProfile: dsflash-go + providerProfileMode: exact + promptSetRef: config/hwlab-web-probe-sentinel/prompt-set.dsflash-go.yaml#sentinel.promptSet + reportViewRef: config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml#sentinel.reportViews + commandSequence: + - type: newSession + - type: selectProvider + provider: dsflash-go + - type: sendPrompt + promptSource: promptSet + repeat: 10 + sessionInvarianceChecks: + - id: after-round-1-navigation-invariance + afterRound: 1 + refreshCurrent: true + switchAwayAndBack: true + alternateSessionStrategy: existing-or-create + assertSessionInvariant: true + expectedSentinelRange: sentinel-01..sentinel-01 + findingId: workbench-message-order-user-clustered-after-navigation + severity: amber + blocking: false + - id: after-round-5-navigation-invariance + afterRound: 5 + refreshCurrent: true + switchAwayAndBack: true + alternateSessionStrategy: existing-or-create + assertSessionInvariant: true + requireComposerReady: true + expectedSentinelRange: sentinel-01..sentinel-05 + findingId: workbench-message-order-user-clustered-after-navigation + severity: amber + blocking: false + - id: after-round-10-refresh-invariance + afterRound: 10 + refreshCurrent: true + switchAwayAndBack: false + assertSessionInvariant: true + expectedSentinelRange: sentinel-01..sentinel-10 + findingId: workbench-message-order-user-clustered-after-navigation + severity: amber + blocking: false + + - id: workbench-auth-session-switch-2users + enabled: true + cadence: 10m + observeTargetPath: /workbench + sampleIntervalMs: 1000 + screenshotIntervalMs: 60000 + maxRunSeconds: 900 + providerProfile: session-switch-sentinel + providerProfileMode: exact + promptSetRef: config/hwlab-web-probe-sentinel/prompt-set.auth-session-switch.yaml#sentinel.promptSet + reportViewRef: config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml#sentinel.reportViews + accounts: + - id: account-a + sourcePurpose: account-a + usernameKey: username + passwordKey: password + - id: account-b + sourcePurpose: account-b + usernameKey: username + passwordKey: password + commandSequence: + - type: loginAccount + accountId: account-a + - type: listSessions + - type: logout + - type: loginAccount + accountId: account-b + - type: listSessions + - type: switchSessions + fromAccountId: account-b + toAccountId: account-a + - type: listSessions + - type: logout + + - id: mdtodo-visual-regression + enabled: true + cadence: 15m + observeTargetPath: /projects/mdtodo + viewport: 1440x900 + sampleIntervalMs: 1000 + screenshotIntervalMs: 60000 + maxRunSeconds: 360 + providerProfile: dsflash-go + providerProfileMode: exact + promptSetRef: config/hwlab-web-probe-sentinel/prompt-set.dsflash-go.yaml#sentinel.promptSet + reportViewRef: config/hwlab-web-probe-sentinel/report-views.multi-sentinel.yaml#sentinel.reportViews + commandSequence: + - type: goto + path: /projects/mdtodo/sources/constart-71freq-mdtodo/files/file_0db4dc4e46adf188/tasks/R1 + - type: screenshot + label: mdtodo-desktop-few-task-gap + waitProjectManagementReady: true + - type: goto + path: /projects/mdtodo/sources/constart-71freq-mdtodo/files/file_5f9645ffe8774b92/tasks/R14 + - type: screenshot + label: mdtodo-r14-selected + waitProjectManagementReady: true + - type: openMdtodoReportPreview + task: R14 + link: R14 + - type: screenshot + label: mdtodo-r14-report-preview + waitProjectManagementReady: true + - type: toggleMdtodoReportFullscreen + text: toggle + - type: screenshot + label: mdtodo-r14-report-fullscreen + waitProjectManagementReady: true diff --git a/config/platform-infra/host-proxy.yaml b/config/platform-infra/host-proxy.yaml new file mode 100644 index 00000000..82f697cb --- /dev/null +++ b/config/platform-infra/host-proxy.yaml @@ -0,0 +1,112 @@ +version: 1 +kind: platform-infra-host-proxy +metadata: + owner: unidesk + relatedIssues: + - 1148 + - 1110 + - 1117 + +defaults: + targetId: JD01 + +server: + id: benchmark-validated-master-shadowsocks-server + enabled: true + benchmarkRef: pikasTech/unidesk#1110 + implementationRef: pikasTech/unidesk#1117 + sourceConfigRef: config/platform-infra/egress-proxy-sources.yaml#sources.master-shadowsocks + composeFile: config/platform-infra/sub2api-master-egress-proxy.compose.yaml + serviceName: sub2api-master-egress-proxy + containerName: unidesk-sub2api-master-egress-proxy + image: ghcr.io/shadowsocks/ssserver-rust:latest + configPath: /root/unidesk/.state/secrets/platform-infra/sub2api-master-egress-proxy.config.json + listenHost: 0.0.0.0 + listenPort: 18792 + health: + mode: tcp + host: 127.0.0.1 + port: 18792 + +sources: + jd01-real-deps-master-shadowsocks: + sourceType: benchmark-validated-master-shadowsocks + serverRef: server.benchmark-validated-master-shadowsocks-server + benchmarkRef: pikasTech/unidesk#1110 + implementationRef: pikasTech/unidesk#1117 + sourceConfigRef: config/platform-infra/egress-proxy-sources.yaml#sources.master-shadowsocks + client: + mode: trans-static-binary + upstreamUrl: https://github.com/SagerNet/sing-box/releases/download/v1.13.14/sing-box-1.13.14-linux-amd64.tar.gz + version: v1.13.14 + archiveCachePath: .state/artifacts/platform-infra/sing-box-1.13.14-linux-amd64.tar.gz + archiveSha256: f48703461a15476951ac4967cdad339d986f4b8096b4eb3ff0829a500502d697 + archiveInstallPath: /var/cache/unidesk/host-egress-proxy/sing-box-1.13.14-linux-amd64.tar.gz + binaryMember: sing-box-1.13.14-linux-amd64/sing-box + binaryCachePath: .state/artifacts/platform-infra/sing-box-1.13.14-linux-amd64 + binarySha256: 68aeab83cc4ab2659a5b92232261a20746ccdafc3b3d1e19b2d63247eec3bbf7 + installPath: /usr/local/bin/sing-box + configPath: /etc/unidesk/host-egress-proxy/sing-box.json + unitPath: /etc/systemd/system/unidesk-host-egress-proxy.service + serviceName: unidesk-host-egress-proxy + listenHost: 127.0.0.1 + listenPort: 10808 + podAccess: + enabled: true + listenHost: 10.42.0.1 + listenPort: 10808 + proxyUrl: http://10.42.0.1:10808 + clashApiListen: 127.0.0.1:19090 + healthUrl: http://127.0.0.1:19090/connections + proxyUrl: http://127.0.0.1:10808 + externalProbeUrl: https://www.gstatic.com/generate_204 + +targets: + JD01: + route: JD01 + enabled: true + sourceRef: sources.jd01-real-deps-master-shadowsocks + env: + httpProxy: http://127.0.0.1:10808 + httpsProxy: http://127.0.0.1:10808 + allProxy: http://127.0.0.1:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - host.docker.internal + - 74.48.78.17 + - 82.156.23.220 + - 10.0.0.0/8 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - argocd-repo-server + - argocd-repo-server.argocd + - argocd-redis + - argocd-redis.argocd + - git-mirror-http + - git-mirror-http.devops-infra + - git-mirror-write + - git-mirror-write.devops-infra + - 127.0.0.1:5000 + - localhost:5000 + - hyueapi.com + - .hyueapi.com + files: + envFile: /etc/unidesk/proxy.env + profile: /etc/profile.d/unidesk-proxy.sh + apt: /etc/apt/apt.conf.d/90unidesk-proxy + dockerSystemdDropIn: /etc/systemd/system/docker.service.d/10-unidesk-proxy.conf + k3sSystemdDropIn: /etc/systemd/system/k3s.service.d/10-unidesk-proxy.conf + apply: + reloadSystemd: true + restartDocker: true + restartK3s: false diff --git a/config/platform-infra/observability.yaml b/config/platform-infra/observability.yaml index 0108860b..8f4f17ba 100644 --- a/config/platform-infra/observability.yaml +++ b/config/platform-infra/observability.yaml @@ -35,6 +35,12 @@ targets: role: active enabled: true createNamespace: true + - id: JD01 + route: JD01:k3s + namespace: platform-infra + role: active + enabled: true + createNamespace: true collector: deploymentName: otel-collector @@ -97,6 +103,20 @@ instrumentation: - projection_write - trace_events_read - turn_status_read + - serviceName: hwlab-cloud-api + owningRepo: pikasTech/HWLAB + configRefs: + targetNode: config/hwlab-node-lanes.yaml#lanes.v03.targets.JD01.node + lane: config/hwlab-node-lanes.yaml#lanes.v03.version + namespace: config/hwlab-node-lanes.yaml#lanes.v03.targets.JD01.runtime.namespace + requiredSpans: + - POST /v1/agent/chat + - durable_admission + - billing_preflight + - agentrun_dispatch + - projection_write + - trace_events_read + - turn_status_read - serviceName: user-billing owningRepo: pikasTech/HWLAB configRefs: @@ -113,6 +133,14 @@ instrumentation: namespace: config/hwlab-node-lanes.yaml#lanes.v03.targets.D518.runtime.namespace requiredSpans: - billing_preflight + - serviceName: user-billing + owningRepo: pikasTech/HWLAB + configRefs: + targetNode: config/hwlab-node-lanes.yaml#lanes.v03.targets.JD01.node + lane: config/hwlab-node-lanes.yaml#lanes.v03.version + namespace: config/hwlab-node-lanes.yaml#lanes.v03.targets.JD01.runtime.namespace + requiredSpans: + - billing_preflight - serviceName: agentrun-manager owningRepo: pikasTech/agentrun configRefs: @@ -124,6 +152,17 @@ instrumentation: - run_created - command_result - projection_sync + - serviceName: agentrun-manager + owningRepo: pikasTech/agentrun + configRefs: + targetNode: config/agentrun.yaml#controlPlane.lanes.jd01-v02.node + lane: config/agentrun.yaml#controlPlane.lanes.jd01-v02.version + namespace: config/agentrun.yaml#controlPlane.lanes.jd01-v02.runtime.namespace + requiredSpans: + - agentrun_dispatch + - run_created + - command_result + - projection_sync resourceAttributes: required: diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index 7293a364..1b51fbaf 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -233,6 +233,72 @@ targets: - 74.48.78.17 - hyueapi.com - .hyueapi.com + - id: JD01 + route: JD01:k3s + namespace: platform-infra + role: active + enabled: true + databaseMode: bundled + redisMode: bundled-persistent + appReplicas: 1 + redisReplicas: 1 + image: + repository: weishaw/sub2api + tag: 0.1.138 + pullPolicy: IfNotPresent + dependencyImages: + postgres: docker.m.daocloud.io/library/postgres:18-alpine + redis: docker.m.daocloud.io/library/redis:8-alpine + codexPool: + sentinelImageBuild: + baseImageCachePolicy: pull + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - host.docker.internal + - 74.48.78.17 + - 192.168.0.0/16 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 10.42.0.0/16 + - 10.43.0.0/16 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 127.0.0.1:5000 + - localhost:5000 + egressProxy: + enabled: false + deploymentName: sub2api-egress-proxy + serviceName: sub2api-egress-proxy + secretName: sub2api-egress-proxy-config + secretKey: config.json + image: ghcr.io/sagernet/sing-box:latest + imagePullPolicy: IfNotPresent + listenPort: 10808 + hostNetwork: true + hostProxyConfigRef: config/platform-infra/host-proxy.yaml#targets.JD01 + proxyEnvPath: /etc/unidesk/proxy.env + sourceConfigRef: config/platform-infra/egress-proxy-sources.yaml#sources.master-shadowsocks + applyToSub2Api: false + applyToSentinel: false + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - .svc + - .cluster.local + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + - hyueapi.com + - .hyueapi.com runtime: database: mode: external diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 00345ead..61b3dfac 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -54,6 +54,7 @@ export interface AgentRunLaneSpec { readonly nodeKubeRoute: string; readonly version: string; readonly source: { + readonly statusMode: "host-worktree" | "k3s-git-mirror"; readonly repository: string; readonly branch: string; readonly bootstrapFromBranch: string | null; @@ -76,6 +77,7 @@ export interface AgentRunLaneSpec { readonly serviceAccountName: string; readonly registryPrefix: string; readonly toolsImage: string; + readonly buildkitImage: string | null; }; readonly gitops: { readonly branch: string; @@ -123,6 +125,10 @@ export interface AgentRunLaneSpec { readonly image: string | null; readonly storage: string | null; readonly port: number | null; + readonly database: string | null; + readonly user: string | null; + readonly passwordSourceRef: string | null; + readonly passwordSourceKey: string | null; }; }; readonly gitMirror: { @@ -271,6 +277,7 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record, path: string): AgentRunLaneSpec["deployment"] { const argocd = recordField(input, "argocd", path); const manager = recordField(input, "manager", path); @@ -660,6 +675,10 @@ function parseLocalPostgres(input: Record, path: string): Agent image: optionalStringField(input, "image", path) ?? null, storage: optionalStringField(input, "storage", path) ?? null, port: optionalIntegerField(input, "port", path) ?? null, + database: optionalStringField(input, "database", path) ?? null, + user: optionalStringField(input, "user", path) ?? null, + passwordSourceRef: optionalStringField(input, "passwordSourceRef", path) ?? null, + passwordSourceKey: optionalStringField(input, "passwordSourceKey", path) ?? null, }; } return { @@ -668,6 +687,10 @@ function parseLocalPostgres(input: Record, path: string): Agent image: stringField(input, "image", path), storage: stringField(input, "storage", path), port: integerField(input, "port", path), + database: stringField(input, "database", path), + user: stringField(input, "user", path), + passwordSourceRef: secretSourceRefField(input, "passwordSourceRef", path), + passwordSourceKey: stringField(input, "passwordSourceKey", path), }; } diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts index 84edbd58..e23c8864 100644 --- a/scripts/src/agentrun-manifests.ts +++ b/scripts/src/agentrun-manifests.ts @@ -300,10 +300,11 @@ function agentRunRuntimeNamespaceManifest(spec: AgentRunLaneSpec): Record { const localPostgres = spec.deployment.localPostgres; - if (!localPostgres.enabled || localPostgres.serviceName === null || localPostgres.image === null || localPostgres.storage === null || localPostgres.port === null) { + if (!localPostgres.enabled || localPostgres.serviceName === null || localPostgres.image === null || localPostgres.storage === null || localPostgres.port === null || localPostgres.database === null || localPostgres.user === null) { throw new Error(`localPostgres is enabled for ${spec.version} without renderable YAML fields`); } const name = localPostgres.serviceName; + const secretName = spec.database.secretRef.name; return { apiVersion: "v1", kind: "List", @@ -330,6 +331,13 @@ function agentRunPostgresManifest(spec: AgentRunLaneSpec): Record> { const result = await triggerCurrentYamlLaneConfirmedSteps(config, spec, configPath, waited); + if (spec.source.statusMode === "k3s-git-mirror") { + return { + ...result, + sourceWorkspaceRestore: { ok: true, status: "skipped", statusMode: spec.source.statusMode, reason: "k3s-git-mirror-does-not-use-host-worktree", valuesPrinted: false }, + valuesPrinted: false, + }; + } const restore = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceRestoreScript(spec)]); const restorePayload = captureJsonPayload(restore); const restoreOk = restore.exitCode === 0 && restorePayload.ok === true; @@ -53,63 +60,27 @@ export async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spe } export async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { - progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { - node: spec.nodeId, - lane: spec.lane, - status: "submitting", - }); - const bootstrapSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapSubmitScript(spec)]); - const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit); - if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) { + const source = await resolveTriggerCurrentSource(config, spec, configPath, waited); + if (source.ok !== true) return source; + const sourceCommit = stringOrNull(source.sourceCommit); + const bootstrapPayload = source.sourcePayload; + if (sourceCommit === null || !isGitSha(sourceCommit)) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), - phase: "source-bootstrap-submit", - degradedReason: "yaml-lane-source-bootstrap-submit-failed", - result: bootstrapSubmitPayload, - capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), - valuesPrinted: false, - }; - } - const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId)); - const bootstrapPayload = bootstrap.payload; - const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit); - if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) { - return { - ok: false, - command: "agentrun control-plane trigger-current", - mode: waited ? "confirmed-waited" : "confirmed-trigger", - configPath, - target: agentRunLaneSummary(spec), - phase: "source-bootstrap", - degradedReason: "yaml-lane-source-bootstrap-failed", + phase: "source-resolve", + degradedReason: "yaml-lane-source-commit-invalid", result: bootstrapPayload, - bootstrapStatus: bootstrap, valuesPrinted: false, }; } - const buildSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageSubmitScript(spec, sourceCommit)]); - const buildSubmitPayload = captureJsonPayload(buildSubmit); - if (buildSubmit.exitCode !== 0 || buildSubmitPayload.ok === false) { - return { - ok: false, - command: "agentrun control-plane trigger-current", - mode: waited ? "confirmed-waited" : "confirmed-trigger", - configPath, - target: agentRunLaneSummary(spec), - phase: "image-build-submit", - sourceCommit, - degradedReason: "yaml-lane-image-build-submit-failed", - sourceBootstrap: bootstrapPayload, - result: buildSubmitPayload, - capture: compactCapture(buildSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), - valuesPrinted: false, - }; - } - const build = await waitForYamlLaneBuildImage(config, spec, sourceCommit, stringOrNull(buildSubmitPayload.jobId)); + const buildResult = await buildTriggerCurrentImage(config, spec, sourceCommit, bootstrapPayload, configPath, waited); + if (buildResult.ok !== true) return buildResult; + const build = buildResult.buildStatus; + const buildSubmitPayload = buildResult.buildSubmitPayload; const buildPayload = build.payload; const digest = stringOrNull(buildPayload.digest); const envIdentity = stringOrNull(buildPayload.envIdentity); @@ -227,3 +198,124 @@ export async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig valuesPrinted: false, }; } + +async function resolveTriggerCurrentSource(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise & { ok: boolean; sourceCommit?: string | null; sourcePayload?: Record }> { + if (spec.source.statusMode === "k3s-git-mirror") { + progressEvent("agentrun.yaml-lane.source-status.progress", { + node: spec.nodeId, + lane: spec.lane, + statusMode: spec.source.statusMode, + status: "probing", + }); + const sourceStatus = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)]); + const sourcePayload = captureJsonPayload(sourceStatus); + const sourceCommit = stringOrNull(sourcePayload.sourceCommit) ?? stringOrNull(sourcePayload.remoteBranchCommit); + if (sourceStatus.exitCode !== 0 || sourcePayload.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-status", + degradedReason: "yaml-lane-k3s-source-status-failed", + result: sourcePayload, + capture: compactCapture(sourceStatus, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + return { ok: true, sourceCommit, sourcePayload, valuesPrinted: false }; + } + + progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { + node: spec.nodeId, + lane: spec.lane, + status: "submitting", + }); + const bootstrapSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapSubmitScript(spec)]); + const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit); + if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-bootstrap-submit", + degradedReason: "yaml-lane-source-bootstrap-submit-failed", + result: bootstrapSubmitPayload, + capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId)); + const sourcePayload = bootstrap.payload; + const sourceCommit = stringOrNull(sourcePayload.sourceCommit); + if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-bootstrap", + degradedReason: "yaml-lane-source-bootstrap-failed", + result: sourcePayload, + bootstrapStatus: bootstrap, + valuesPrinted: false, + }; + } + return { ok: true, sourceCommit, sourcePayload, sourceBootstrapSubmit: bootstrapSubmitPayload, valuesPrinted: false }; +} + +async function buildTriggerCurrentImage(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, sourcePayload: Record, configPath: string, waited: boolean): Promise & { ok: boolean; buildStatus?: Record & { ok: boolean; payload: Record }; buildSubmitPayload?: Record }> { + if (spec.source.statusMode === "k3s-git-mirror") { + const build = await runYamlLaneK3sBuildImageJob(config, spec, sourceCommit); + const buildSubmitPayload = { + ok: build.ok !== false, + status: "submitted", + mode: "k3s-buildkit-job", + jobName: stringOrNull(build.jobName) ?? null, + valuesPrinted: false, + }; + if (build.ok !== true) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "image-build", + sourceCommit, + degradedReason: "yaml-lane-k3s-image-build-failed", + sourceBootstrap: sourcePayload, + buildSubmit: buildSubmitPayload, + result: build.payload, + buildStatus: build, + valuesPrinted: false, + }; + } + return { ok: true, buildStatus: build, buildSubmitPayload, valuesPrinted: false }; + } + + const buildSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneBuildImageSubmitScript(spec, sourceCommit)]); + const buildSubmitPayload = captureJsonPayload(buildSubmit); + if (buildSubmit.exitCode !== 0 || buildSubmitPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "image-build-submit", + sourceCommit, + degradedReason: "yaml-lane-image-build-submit-failed", + sourceBootstrap: sourcePayload, + result: buildSubmitPayload, + capture: compactCapture(buildSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const build = await waitForYamlLaneBuildImage(config, spec, sourceCommit, stringOrNull(buildSubmitPayload.jobId)); + return { ok: true, buildStatus: build, buildSubmitPayload, valuesPrinted: false }; +} diff --git a/scripts/src/agentrun/control-plane.ts b/scripts/src/agentrun/control-plane.ts index b5eb9fc8..d5fc8f67 100644 --- a/scripts/src/agentrun/control-plane.ts +++ b/scripts/src/agentrun/control-plane.ts @@ -38,7 +38,7 @@ import { agentRunControlPlaneStatusCommand } from "./public-exposure"; import { applyYamlScript, manifestObjectRef, yamlLaneGitMirrorStatusScript } from "./secrets"; import { compactAgentRunLaneStatusTarget, compactLaneSecretsStatus } from "./trigger"; import { capture, captureJsonPayload, compactCapture, record, stringOrNull, timedStatusStage } from "./utils"; -import { yamlLaneRuntimeStatusScript, yamlLaneSourceStatusScript } from "./yaml-lane"; +import { yamlLaneK3sSourceStatusScript, yamlLaneRuntimeStatusScript, yamlLaneSourceStatusScript } from "./yaml-lane"; export function parseSecretSyncOptions(args: string[]): SecretSyncOptions { const base = parseConfirmOptions(args); @@ -315,7 +315,9 @@ export async function status(config: UniDeskConfig, options: StatusOptions): Pro export async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { const spec = target.spec; - const sourceProbe = await timedStatusStage("source", () => capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)])); + const sourceProbe = await timedStatusStage("source", () => spec.source.statusMode === "k3s-git-mirror" + ? capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)]) + : capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)])); const sourcePayload = captureJsonPayload(sourceProbe.value); const branchTipCommit = stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead); const initialSourceCommit = options.sourceCommit @@ -362,14 +364,15 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio : null, valuesPrinted: false, }; + const sourceWorkspaceRequired = spec.source.statusMode === "host-worktree"; const warnings = [ ...(sourceWorktreeDetached ? ["source-worktree-detached"] : []), ...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []), ]; const blockers = [ - ...(sourcePayload.workspaceExists === true ? [] : ["source-worktree-missing"]), + ...(sourceWorkspaceRequired && sourcePayload.workspaceExists !== true ? ["source-worktree-missing"] : []), ...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]), - ...(sourcePayload.workspaceClean === true || sourcePayload.workspaceExists !== true ? [] : ["source-worktree-dirty"]), + ...(sourceWorkspaceRequired && sourcePayload.workspaceClean !== true && sourcePayload.workspaceExists === true ? ["source-worktree-dirty"] : []), ...(mirrorPayload.readReady === true ? [] : ["git-mirror-read-not-ready"]), ...(mirrorPayload.writeReady === true ? [] : ["git-mirror-write-not-ready"]), ...(mirrorPayload.cacheReady === true ? [] : ["git-mirror-cache-not-ready"]), @@ -422,6 +425,7 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio ? { code: "runtime-blocked", summary: `runtime alignment is blocked: ${runtimeBlockers.join(", ")}`, command: statusFullCommand } : { code: "inspect-full-status", summary: "alignment is inconclusive; inspect full status details", command: statusFullCommand }; const compactSourceStatus = { + statusMode: spec.source.statusMode, workspaceExists: sourcePayload.workspaceExists ?? false, workspaceClean: sourcePayload.workspaceClean ?? null, branch: sourcePayload.branch ?? null, diff --git a/scripts/src/agentrun/git-mirror.ts b/scripts/src/agentrun/git-mirror.ts index 3632b9c1..a3f9d298 100644 --- a/scripts/src/agentrun/git-mirror.ts +++ b/scripts/src/agentrun/git-mirror.ts @@ -446,21 +446,27 @@ export function refreshYamlLaneScript(spec: AgentRunLaneSpec): string { `application=${shQuote(spec.gitops.argoApplication)}`, "kubectl -n \"$namespace\" annotate application \"$application\" argocd.argoproj.io/refresh=hard --overwrite >/tmp/agentrun-refresh-output.txt", "kubectl -n \"$namespace\" get application \"$application\" -o json >/tmp/agentrun-refresh-app.json", - "NAMESPACE=\"$namespace\" APPLICATION=\"$application\" node <<'NODE'", - "const fs = require('node:fs');", - "let app = {};", - "try { app = JSON.parse(fs.readFileSync('/tmp/agentrun-refresh-app.json', 'utf8')); } catch {}", - "console.log(JSON.stringify({", - " ok: true,", - " namespace: process.env.NAMESPACE,", - " application: process.env.APPLICATION,", - " revision: app.status?.sync?.revision ?? null,", - " syncStatus: app.status?.sync?.status ?? null,", - " healthStatus: app.status?.health?.status ?? null,", - " observedAt: new Date().toISOString(),", - " valuesPrinted: false", - "}));", - "NODE", + "NAMESPACE=\"$namespace\" APPLICATION=\"$application\" python3 - <<'PY'", + "import datetime, json, os", + "try:", + " with open('/tmp/agentrun-refresh-app.json', 'r', encoding='utf-8') as fh:", + " app = json.load(fh)", + "except Exception:", + " app = {}", + "status = app.get('status') or {}", + "sync = status.get('sync') or {}", + "health = status.get('health') or {}", + "print(json.dumps({", + " 'ok': True,", + " 'namespace': os.environ.get('NAMESPACE'),", + " 'application': os.environ.get('APPLICATION'),", + " 'revision': sync.get('revision'),", + " 'syncStatus': sync.get('status'),", + " 'healthStatus': health.get('status'),", + " 'observedAt': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'),", + " 'valuesPrinted': False,", + "}, ensure_ascii=False))", + "PY", ].join("\n"); } diff --git a/scripts/src/agentrun/secrets.ts b/scripts/src/agentrun/secrets.ts index 28706fef..fcb1f01d 100644 --- a/scripts/src/agentrun/secrets.ts +++ b/scripts/src/agentrun/secrets.ts @@ -343,9 +343,10 @@ export function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCo "if [ \"$existing_status\" = False ]; then kubectl -n \"$namespace\" delete pipelinerun \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true; fi", "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then created=false; else kubectl create -f \"$tmp\"; created=true; fi", "status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", - "CREATED=\"$created\" STATUS=\"$status\" PIPELINE_RUN=\"$pipeline_run\" node <<'NODE'", - "console.log(JSON.stringify({ ok: true, pipelineRun: process.env.PIPELINE_RUN, created: process.env.CREATED === 'true', status: process.env.STATUS || null, valuesPrinted: false }));", - "NODE", + "CREATED=\"$created\" STATUS=\"$status\" PIPELINE_RUN=\"$pipeline_run\" python3 - <<'PY'", + "import json, os", + "print(json.dumps({'ok': True, 'pipelineRun': os.environ.get('PIPELINE_RUN'), 'created': os.environ.get('CREATED') == 'true', 'status': os.environ.get('STATUS') or None, 'valuesPrinted': False}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -361,9 +362,10 @@ export function createYamlLaneJobScript(namespace: string, jobName: string, mani "printf '%s' \"$manifest_b64\" | base64 -d > \"$tmp\"", "kubectl -n \"$namespace\" delete job \"$job\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "kubectl create -f \"$tmp\"", - "JOB=\"$job\" node <<'NODE'", - "console.log(JSON.stringify({ ok: true, jobName: process.env.JOB, valuesPrinted: false }));", - "NODE", + "JOB=\"$job\" python3 - <<'PY'", + "import json, os", + "print(json.dumps({'ok': True, 'jobName': os.environ.get('JOB'), 'valuesPrinted': False}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -375,14 +377,27 @@ export function yamlLaneJobProbeScript(namespace: string, jobName: string): stri "kubectl -n \"$namespace\" get job \"$job\" -o json > /tmp/agentrun-job.json 2>/dev/null", "job_exit=$?", "kubectl -n \"$namespace\" logs \"job/$job\" --tail=120 > /tmp/agentrun-job.log 2>/dev/null", - "JOB_EXIT=\"$job_exit\" JOB=\"$job\" node <<'NODE'", - "const fs = require('node:fs');", - "let job = null; try { job = JSON.parse(fs.readFileSync('/tmp/agentrun-job.json', 'utf8')); } catch {}", - "let log = ''; try { log = fs.readFileSync('/tmp/agentrun-job.log', 'utf8'); } catch {}", - "const succeeded = Number(job?.status?.succeeded || 0) > 0;", - "const failed = Number(job?.status?.failed || 0) > 0;", - "console.log(JSON.stringify({ ok: process.env.JOB_EXIT === '0', jobName: process.env.JOB, succeeded, failed, active: job?.status?.active || 0, logsTail: log.slice(-4000), valuesPrinted: false }));", - "NODE", + "JOB_EXIT=\"$job_exit\" JOB=\"$job\" python3 - <<'PY'", + "import json, os", + "def read_json(path):", + " try:", + " with open(path, 'r', encoding='utf-8') as fh:", + " return json.load(fh)", + " except Exception:", + " return None", + "def read_text(path):", + " try:", + " with open(path, 'r', encoding='utf-8') as fh:", + " return fh.read()", + " except Exception:", + " return ''", + "job = read_json('/tmp/agentrun-job.json') or {}", + "status = job.get('status') or {}", + "log = read_text('/tmp/agentrun-job.log')", + "succeeded = int(status.get('succeeded') or 0) > 0", + "failed = int(status.get('failed') or 0) > 0", + "print(json.dumps({'ok': os.environ.get('JOB_EXIT') == '0', 'jobName': os.environ.get('JOB'), 'succeeded': succeeded, 'failed': failed, 'active': status.get('active') or 0, 'logsTail': log[-4000:], 'valuesPrinted': False}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -619,38 +634,53 @@ export function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { "repo_path=\"/cache/${repository}.git\"", "rm -f /tmp/agentrun-gitmirror-refs.txt", "if [ \"$read_exit\" -eq 0 ]; then", - " kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; gitops_branch=\"$3\"; printf \"sourceCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$source_branch\" 2>/dev/null || true; printf \"gitopsCommit=\"; git --git-dir=\"$repo_path\" rev-parse \"refs/heads/$gitops_branch\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" \"$gitops_branch\" > /tmp/agentrun-gitmirror-refs.txt 2>/dev/null", + " kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; gitops_branch=\"$3\"; source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$source_branch^{commit}\" 2>/dev/null || true); gitops_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$gitops_branch^{commit}\" 2>/dev/null || true); printf \"sourceCommit=%s\\n\" \"$source_commit\"; printf \"gitopsCommit=%s\\n\" \"$gitops_commit\"' sh \"$repo_path\" \"$source_branch\" \"$gitops_branch\" > /tmp/agentrun-gitmirror-refs.txt 2>/dev/null", "fi", - "NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" CACHE_MODE=\"$cache_mode\" CACHE_HOST_PATH=\"$cache_host_path\" REPOSITORY=\"$repository\" SOURCE_BRANCH=\"$source_branch\" GITOPS_BRANCH=\"$gitops_branch\" REPOSITORIES_JSON=\"$repositories_json\" node <<'NODE'", - "const fs = require('node:fs');", - "const repositories = JSON.parse(process.env.REPOSITORIES_JSON || '[]');", - "let names = ''; try { names = fs.readFileSync('/tmp/agentrun-gitmirror-names.txt', 'utf8'); } catch {}", - "let refs = ''; try { refs = fs.readFileSync('/tmp/agentrun-gitmirror-refs.txt', 'utf8'); } catch {}", - "const refValue = (key) => refs.split(/\\r?\\n/).find((line) => line.startsWith(`${key}=`))?.slice(key.length + 1).trim() || null;", - "const readReady = process.env.READ_EXIT === '0';", - "const writeReady = process.env.WRITE_EXIT === '0';", - "const cachePvcExists = process.env.CACHE_EXIT === '0';", - "const cacheMode = process.env.CACHE_MODE || 'pvc';", - "const cacheReady = cacheMode === 'hostPath' ? true : cachePvcExists;", - "console.log(JSON.stringify({", - " ok: readReady && writeReady && cacheReady,", - " namespace: process.env.NAMESPACE,", - " readReady,", - " writeReady,", - " cacheMode,", - " cacheReady,", - " cachePvcExists,", - " cacheHostPath: process.env.CACHE_HOST_PATH || null,", - " resources: names.split(/\\r?\\n/).filter(Boolean).slice(0, 40),", - " repository: process.env.REPOSITORY,", - " sourceBranch: process.env.SOURCE_BRANCH,", - " gitopsBranch: process.env.GITOPS_BRANCH,", - " sourceCommit: refValue('sourceCommit'),", - " gitopsCommit: refValue('gitopsCommit'),", - " repositories,", - " valuesPrinted: false", - "}));", - "NODE", + "NAMESPACE=\"$namespace\" READ_EXIT=\"$read_exit\" WRITE_EXIT=\"$write_exit\" CACHE_EXIT=\"$cache_exit\" CACHE_MODE=\"$cache_mode\" CACHE_HOST_PATH=\"$cache_host_path\" REPOSITORY=\"$repository\" SOURCE_BRANCH=\"$source_branch\" GITOPS_BRANCH=\"$gitops_branch\" REPOSITORIES_JSON=\"$repositories_json\" python3 - <<'PY'", + "import json, os, re", + "def read_text(path):", + " try:", + " with open(path, 'r', encoding='utf-8') as fh:", + " return fh.read()", + " except Exception:", + " return ''", + "def ref_value(text, key):", + " prefix = key + '='", + " for line in re.split(r'\\r?\\n', text):", + " if line.startswith(prefix):", + " value = line[len(prefix):].strip()", + " return value or None", + " return None", + "try:", + " repositories = json.loads(os.environ.get('REPOSITORIES_JSON') or '[]')", + "except Exception:", + " repositories = []", + "names = read_text('/tmp/agentrun-gitmirror-names.txt')", + "refs = read_text('/tmp/agentrun-gitmirror-refs.txt')", + "read_ready = os.environ.get('READ_EXIT') == '0'", + "write_ready = os.environ.get('WRITE_EXIT') == '0'", + "cache_pvc_exists = os.environ.get('CACHE_EXIT') == '0'", + "cache_mode = os.environ.get('CACHE_MODE') or 'pvc'", + "cache_ready = True if cache_mode == 'hostPath' else cache_pvc_exists", + "print(json.dumps({", + " 'ok': read_ready and write_ready and cache_ready,", + " 'namespace': os.environ.get('NAMESPACE'),", + " 'readReady': read_ready,", + " 'writeReady': write_ready,", + " 'cacheMode': cache_mode,", + " 'cacheReady': cache_ready,", + " 'cachePvcExists': cache_pvc_exists,", + " 'cacheHostPath': os.environ.get('CACHE_HOST_PATH') or None,", + " 'resources': [line for line in re.split(r'\\r?\\n', names) if line][:40],", + " 'repository': os.environ.get('REPOSITORY'),", + " 'sourceBranch': os.environ.get('SOURCE_BRANCH'),", + " 'gitopsBranch': os.environ.get('GITOPS_BRANCH'),", + " 'sourceCommit': ref_value(refs, 'sourceCommit'),", + " 'gitopsCommit': ref_value(refs, 'gitopsCommit'),", + " 'repositories': repositories,", + " 'valuesPrinted': False", + "}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -660,6 +690,7 @@ export type LaneSecretSource = { sourceMode: "env" | "file"; sourceKey: string | null; targetRef: { namespace: string; name: string; key: string }; + transform?: "local-postgres-database" | "local-postgres-user" | "local-postgres-database-url"; }; export function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource): { redactedPath: string; value: string; valueBytes: number; fingerprint: string } { @@ -680,6 +711,7 @@ export function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecret value = envValue; } if (value.length === 0) throw new Error(`secret source ${sourceRef} is empty`); + value = transformSecretSourceValue(spec, source, value); return { redactedPath: sourceRef.startsWith("/") ? redactAbsoluteSecretPath(sourceRef) : `.state/secrets/${sourceRef}`, value, @@ -688,6 +720,20 @@ export function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecret }; } +function transformSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource, rawValue: string): string { + if (source.transform === undefined) return rawValue; + const localPostgres = spec.deployment.localPostgres; + if (!localPostgres.enabled || localPostgres.serviceName === null || localPostgres.database === null || localPostgres.user === null || localPostgres.port === null) { + throw new Error(`secret source ${source.id} requires enabled localPostgres with database/user/port`); + } + if (source.transform === "local-postgres-database") return localPostgres.database; + if (source.transform === "local-postgres-user") return localPostgres.user; + const user = encodeURIComponent(localPostgres.user); + const password = encodeURIComponent(rawValue); + const database = encodeURIComponent(localPostgres.database); + return `postgresql://${user}:${password}@${localPostgres.serviceName}:${localPostgres.port}/${database}`; +} + export function redactAbsoluteSecretPath(sourceRef: string): string { const parts = sourceRef.split("/").filter(Boolean); return parts.length === 0 ? "/" : `/${parts.slice(0, -1).join("/")}/`; @@ -732,6 +778,45 @@ export function collectLaneSecretSources(spec: AgentRunLaneSpec): LaneSecretSour targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: spec.database.secretRef.key }, }); } + const localPostgres = spec.deployment.localPostgres; + if (spec.database.mode === "local-postgres" && localPostgres.enabled) { + if (localPostgres.passwordSourceRef === null || localPostgres.passwordSourceKey === null) { + throw new Error(`config/agentrun.yaml controlPlane.lanes.${spec.lane}.deployment.localPostgres must declare passwordSourceRef/passwordSourceKey`); + } + result.push( + { + id: "local-postgres-password", + sourceRef: localPostgres.passwordSourceRef, + sourceMode: "env", + sourceKey: localPostgres.passwordSourceKey, + targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: "POSTGRES_PASSWORD" }, + }, + { + id: "local-postgres-user", + sourceRef: localPostgres.passwordSourceRef, + sourceMode: "env", + sourceKey: localPostgres.passwordSourceKey, + targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: "POSTGRES_USER" }, + transform: "local-postgres-user", + }, + { + id: "local-postgres-database", + sourceRef: localPostgres.passwordSourceRef, + sourceMode: "env", + sourceKey: localPostgres.passwordSourceKey, + targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: "POSTGRES_DB" }, + transform: "local-postgres-database", + }, + { + id: "database", + sourceRef: localPostgres.passwordSourceRef, + sourceMode: "env", + sourceKey: localPostgres.passwordSourceKey, + targetRef: { namespace: spec.runtime.namespace, name: spec.database.secretRef.name, key: spec.database.secretRef.key }, + transform: "local-postgres-database-url", + }, + ); + } for (const secret of spec.secrets) { result.push({ id: secret.id, @@ -762,47 +847,44 @@ export function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ target "trap 'rm -rf \"$tmp_dir\"' EXIT", "payload_json=\"$tmp_dir/payload.json\"", "printf '%s' \"$payload_b64\" | base64 -d > \"$payload_json\"", - "PAYLOAD_JSON=\"$payload_json\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", - "const fs = require('node:fs');", - "const cp = require('node:child_process');", - "const crypto = require('node:crypto');", - "const items = JSON.parse(fs.readFileSync(process.env.PAYLOAD_JSON, 'utf8'));", - "const results = [];", - "const groups = new Map();", - "function run(argv, input) {", - " const out = cp.spawnSync(argv[0], argv.slice(1), { input, encoding: 'utf8' });", - " if (out.status !== 0) throw new Error(`${argv.join(' ')} failed: ${out.stderr || out.stdout}`);", - " return out;", - "}", - "function tryRun(argv) {", - " return cp.spawnSync(argv[0], argv.slice(1), { encoding: 'utf8' });", - "}", - "for (let index = 0; index < items.length; index += 1) {", - " const item = items[index];", - " const ref = item.targetRef;", - " const groupKey = `${ref.namespace}\\u0000${ref.name}`;", - " if (!groups.has(groupKey)) groups.set(groupKey, { namespace: ref.namespace, name: ref.name, items: [] });", - " groups.get(groupKey).items.push({ key: ref.key, valueBase64: item.valueBase64 });", - "}", - "for (const group of groups.values()) {", - " const ns = run(['kubectl', 'create', 'namespace', group.namespace, '--dry-run=client', '-o', 'yaml']).stdout;", - " run(['kubectl', 'apply', '--server-side', '--field-manager=unidesk-agentrun-secret-sync', '-f', '-'], ns);", - " const existing = tryRun(['kubectl', '-n', group.namespace, 'get', 'secret', group.name]);", - " if (existing.status !== 0) run(['kubectl', '-n', group.namespace, 'create', 'secret', 'generic', group.name]);", - " const patch = { data: {} };", - " for (const entry of group.items) patch.data[entry.key] = entry.valueBase64;", - " const patchFile = `${process.env.TMP_DIR}/${group.namespace}-${group.name}-patch.json`;", - " fs.writeFileSync(patchFile, JSON.stringify(patch));", - " run(['kubectl', '-n', group.namespace, 'patch', 'secret', group.name, '--type=merge', `--patch-file=${patchFile}`]);", - " const fetched = JSON.parse(run(['kubectl', '-n', group.namespace, 'get', 'secret', group.name, '-o', 'json']).stdout);", - " for (const entry of group.items) {", - " const raw = fetched.data?.[entry.key] || '';", - " const decoded = raw ? Buffer.from(raw, 'base64') : Buffer.alloc(0);", - " results.push({ namespace: group.namespace, secret: group.name, key: entry.key, ok: raw.length > 0, valueBytes: decoded.length, fingerprint: raw ? 'sha256:' + crypto.createHash('sha256').update(decoded).digest('hex') : null, valuesPrinted: false });", - " }", - "}", - "console.log(JSON.stringify({ ok: results.every((item) => item.ok), secretCount: results.length, items: results, valuesPrinted: false }));", - "NODE", + "PAYLOAD_JSON=\"$payload_json\" TMP_DIR=\"$tmp_dir\" python3 - <<'PY'", + "import base64, hashlib, json, os, subprocess", + "def run(argv, input_text=None):", + " out = subprocess.run(argv, input=input_text, text=True, capture_output=True)", + " if out.returncode != 0:", + " raise RuntimeError('%s failed: %s' % (' '.join(argv), (out.stderr or out.stdout).strip()))", + " return out", + "def try_run(argv):", + " return subprocess.run(argv, text=True, capture_output=True)", + "with open(os.environ['PAYLOAD_JSON'], 'r', encoding='utf-8') as fh:", + " items = json.load(fh)", + "groups = {}", + "for item in items:", + " ref = item.get('targetRef') or {}", + " key = (ref.get('namespace'), ref.get('name'))", + " groups.setdefault(key, {'namespace': ref.get('namespace'), 'name': ref.get('name'), 'items': []})['items'].append({'key': ref.get('key'), 'valueBase64': item.get('valueBase64') or ''})", + "results = []", + "for group in groups.values():", + " namespace = group['namespace']", + " name = group['name']", + " ns_yaml = run(['kubectl', 'create', 'namespace', namespace, '--dry-run=client', '-o', 'yaml']).stdout", + " run(['kubectl', 'apply', '--server-side', '--field-manager=unidesk-agentrun-secret-sync', '-f', '-'], ns_yaml)", + " existing = try_run(['kubectl', '-n', namespace, 'get', 'secret', name])", + " if existing.returncode != 0:", + " run(['kubectl', '-n', namespace, 'create', 'secret', 'generic', name])", + " patch = {'data': {entry['key']: entry['valueBase64'] for entry in group['items']}}", + " patch_file = os.path.join(os.environ['TMP_DIR'], '%s-%s-patch.json' % (namespace, name))", + " with open(patch_file, 'w', encoding='utf-8') as fh:", + " json.dump(patch, fh)", + " run(['kubectl', '-n', namespace, 'patch', 'secret', name, '--type=merge', '--patch-file=%s' % patch_file])", + " fetched = json.loads(run(['kubectl', '-n', namespace, 'get', 'secret', name, '-o', 'json']).stdout)", + " data = fetched.get('data') or {}", + " for entry in group['items']:", + " raw = data.get(entry['key']) or ''", + " decoded = base64.b64decode(raw.encode('utf-8')) if raw else b''", + " results.append({'namespace': namespace, 'secret': name, 'key': entry['key'], 'ok': bool(raw), 'valueBytes': len(decoded), 'fingerprint': ('sha256:' + hashlib.sha256(decoded).hexdigest()) if raw else None, 'valuesPrinted': False})", + "print(json.dumps({'ok': all(item['ok'] for item in results), 'secretCount': len(results), 'items': results, 'valuesPrinted': False}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -869,15 +951,20 @@ export function applyYamlScript(yaml: string, fieldManager: string, dryRun: bool "kubectl apply $args -f \"$manifest\" > \"$tmp_dir/apply.out\" 2> \"$tmp_dir/apply.err\"", "apply_exit=$?", "set -e", - "APPLY_EXIT=\"$apply_exit\" APPLY_OUT=\"$tmp_dir/apply.out\" APPLY_ERR=\"$tmp_dir/apply.err\" MANIFEST=\"$manifest\" node <<'NODE'", - "const fs = require('node:fs');", - "const crypto = require('node:crypto');", - "const out = fs.readFileSync(process.env.APPLY_OUT, 'utf8');", - "const err = fs.readFileSync(process.env.APPLY_ERR, 'utf8');", - "const manifest = fs.readFileSync(process.env.MANIFEST, 'utf8');", - "const resources = out.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean).slice(0, 80);", - "console.log(JSON.stringify({ ok: process.env.APPLY_EXIT === '0', exitCode: Number(process.env.APPLY_EXIT), resourceCount: resources.length, resources, manifestBytes: Buffer.byteLength(manifest, 'utf8'), manifestDigest: 'sha256:' + crypto.createHash('sha256').update(manifest).digest('hex'), stderrTail: err.slice(-3000), valuesPrinted: false }));", - "NODE", + "APPLY_EXIT=\"$apply_exit\" APPLY_OUT=\"$tmp_dir/apply.out\" APPLY_ERR=\"$tmp_dir/apply.err\" MANIFEST=\"$manifest\" python3 - <<'PY'", + "import hashlib, json, os, re", + "def read_bytes(path):", + " try:", + " with open(path, 'rb') as fh:", + " return fh.read()", + " except Exception:", + " return b''", + "out = read_bytes(os.environ['APPLY_OUT']).decode('utf-8', errors='replace')", + "err = read_bytes(os.environ['APPLY_ERR']).decode('utf-8', errors='replace')", + "manifest = read_bytes(os.environ['MANIFEST'])", + "resources = [line.strip() for line in re.split(r'\\r?\\n', out) if line.strip()][:80]", + "print(json.dumps({'ok': os.environ.get('APPLY_EXIT') == '0', 'exitCode': int(os.environ.get('APPLY_EXIT') or '1'), 'resourceCount': len(resources), 'resources': resources, 'manifestBytes': len(manifest), 'manifestDigest': 'sha256:' + hashlib.sha256(manifest).hexdigest(), 'stderrTail': err[-3000:], 'valuesPrinted': False}, ensure_ascii=False))", + "PY", "exit \"$apply_exit\"", ].join("\n"); } diff --git a/scripts/src/agentrun/yaml-lane.ts b/scripts/src/agentrun/yaml-lane.ts index c658f9f9..38be1fbd 100644 --- a/scripts/src/agentrun/yaml-lane.ts +++ b/scripts/src/agentrun/yaml-lane.ts @@ -354,26 +354,78 @@ export function yamlLaneSourceStatusScript(spec: AgentRunLaneSpec): string { " actual_workspace=$(pwd)", "fi", "export expected_workspace source_branch workspace_exists workspace_clean local_head branch remote_url remote_branch_exists remote_branch_commit status_short actual_workspace", - "node <<'NODE'", - "function nullable(value) { return value && value !== 'null' ? value : null; }", - "function booleanValue(value) { if (value === 'true') return true; if (value === 'false') return false; return null; }", - "const env = process.env;", - "console.log(JSON.stringify({", - " ok: env.workspace_exists === 'true',", - " expectedWorkspace: env.expected_workspace,", - " actualWorkspace: env.actual_workspace || null,", - " workspaceExists: env.workspace_exists === 'true',", - " workspaceClean: booleanValue(env.workspace_clean),", - " branch: nullable(env.branch),", - " remoteUrl: nullable(env.remote_url),", - " localHead: nullable(env.local_head),", - " remoteBranch: env.source_branch,", - " remoteBranchExists: env.remote_branch_exists === 'true',", - " remoteBranchCommit: nullable(env.remote_branch_commit),", - " statusShort: nullable(env.status_short),", - " valuesPrinted: false", - "}));", - "NODE", + "python3 - <<'PY'", + "import json, os", + "def nullable(value):", + " return value if value and value != 'null' else None", + "def boolean_value(value):", + " if value == 'true': return True", + " if value == 'false': return False", + " return None", + "env = os.environ", + "print(json.dumps({", + " 'ok': env.get('workspace_exists') == 'true',", + " 'statusMode': 'host-worktree',", + " 'expectedWorkspace': env.get('expected_workspace'),", + " 'actualWorkspace': env.get('actual_workspace') or None,", + " 'workspaceExists': env.get('workspace_exists') == 'true',", + " 'workspaceClean': boolean_value(env.get('workspace_clean')),", + " 'branch': nullable(env.get('branch')),", + " 'remoteUrl': nullable(env.get('remote_url')),", + " 'localHead': nullable(env.get('local_head')),", + " 'remoteBranch': env.get('source_branch'),", + " 'remoteBranchExists': env.get('remote_branch_exists') == 'true',", + " 'remoteBranchCommit': nullable(env.get('remote_branch_commit')),", + " 'statusShort': nullable(env.get('status_short')),", + " 'valuesPrinted': False,", + "}, ensure_ascii=False))", + "PY", + ].join("\n"); +} + +export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string { + return [ + "set +e", + `namespace=${shQuote(spec.gitMirror.namespace)}`, + `read_deployment=${shQuote(spec.gitMirror.readDeployment)}`, + `repository=${shQuote(spec.source.repository)}`, + `source_branch=${shQuote(spec.source.branch)}`, + "repo_path=\"/cache/${repository}.git\"", + "kubectl -n \"$namespace\" get deploy \"$read_deployment\" >/tmp/agentrun-source-read-deploy.txt 2>/dev/null", + "read_exit=$?", + "source_commit=''", + "if [ \"$read_exit\" -eq 0 ]; then", + " source_commit=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$source_branch^{commit}\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" 2>/dev/null | tail -n 1 | tr -d '\\r')", + "fi", + "export namespace read_deployment repository source_branch read_exit source_commit", + "python3 - <<'PY'", + "import json, os", + "source_commit = os.environ.get('source_commit') or None", + "read_ready = os.environ.get('read_exit') == '0'", + "print(json.dumps({", + " 'ok': read_ready and source_commit is not None,", + " 'statusMode': 'k3s-git-mirror',", + " 'expectedWorkspace': None,", + " 'actualWorkspace': None,", + " 'workspaceExists': None,", + " 'workspaceClean': None,", + " 'branch': os.environ.get('source_branch'),", + " 'remoteUrl': None,", + " 'localHead': source_commit,", + " 'remoteBranch': os.environ.get('source_branch'),", + " 'remoteBranchExists': source_commit is not None,", + " 'remoteBranchCommit': source_commit,", + " 'sourceCommit': source_commit,", + " 'gitMirror': {", + " 'namespace': os.environ.get('namespace'),", + " 'readDeployment': os.environ.get('read_deployment'),", + " 'readReady': read_ready,", + " 'repository': os.environ.get('repository'),", + " },", + " 'statusShort': None,", + " 'valuesPrinted': False,", + "}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -412,54 +464,70 @@ export function yamlLaneRuntimeStatusScript(spec: AgentRunLaneSpec, pipelineRun: "manager_svc_exit=$?", "kubectl -n \"$runtime_namespace\" get secret \"$database_secret\" -o json > \"$tmp_dir/db-secret.json\" 2>/dev/null", "db_secret_exit=$?", - "SECRET_REFS_JSON=\"$secrets_json\" NODE_TMP=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/secrets.json\"", - "const fs = require('node:fs');", - "const cp = require('node:child_process');", - "const refs = JSON.parse(process.env.SECRET_REFS_JSON || '[]');", - "const result = [];", - "for (const ref of refs) {", - " const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });", - " let keyPresent = false;", - " if (out.status === 0) {", - " try { keyPresent = Object.prototype.hasOwnProperty.call((JSON.parse(out.stdout).data || {}), ref.key); } catch {}", - " }", - " result.push({ namespace: ref.namespace, name: ref.name, key: ref.key, present: out.status === 0, keyPresent, valuesPrinted: false });", - "}", - "console.log(JSON.stringify({ ready: result.every((item) => item.present && item.keyPresent), count: result.length, items: result, valuesPrinted: false }));", - "NODE", + "SECRET_REFS_JSON=\"$secrets_json\" python3 - <<'PY' > \"$tmp_dir/secrets.json\"", + "import json, os, subprocess", + "refs = json.loads(os.environ.get('SECRET_REFS_JSON') or '[]')", + "result = []", + "for ref in refs:", + " out = subprocess.run(['kubectl', '-n', ref.get('namespace', ''), 'get', 'secret', ref.get('name', ''), '-o', 'json'], text=True, capture_output=True)", + " key_present = False", + " if out.returncode == 0:", + " try:", + " key_present = ref.get('key') in (json.loads(out.stdout).get('data') or {})", + " except Exception:", + " key_present = False", + " result.append({'namespace': ref.get('namespace'), 'name': ref.get('name'), 'key': ref.get('key'), 'present': out.returncode == 0, 'keyPresent': key_present, 'valuesPrinted': False})", + "print(json.dumps({'ready': all(item['present'] and item['keyPresent'] for item in result), 'count': len(result), 'items': result, 'valuesPrinted': False}, ensure_ascii=False))", + "PY", "kubectl -n \"$runtime_namespace\" get deploy,sts,svc,secret -o name > \"$tmp_dir/runtime-names.txt\" 2>/dev/null", - "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" node <<'NODE'", - "const fs = require('node:fs');", - "const path = require('node:path');", - "const dir = process.env.NODE_TMP;", - "function readJson(name) { try { return JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); } catch { return null; } }", - "function exists(exitName) { return process.env[exitName] === '0'; }", - "function condition(obj) { return obj?.status?.conditions?.[0] || null; }", - "function envValue(deploy, name) { return deploy?.spec?.template?.spec?.containers?.[0]?.env?.find((entry) => entry.name === name)?.value || null; }", - "function pipelineRunParam(obj, name) { return (obj?.spec?.params || []).find((entry) => entry.name === name)?.value || null; }", - "const pipelineRun = readJson('pipelinerun.json');", - "const argo = readJson('argo.json');", - "const managerDeploy = readJson('manager-deploy.json');", - "const managerSvc = readJson('manager-svc.json');", - "const dbSecret = readJson('db-secret.json');", - "const secrets = readJson('secrets.json') || { ready: true, count: 0, items: [], valuesPrinted: false };", - "let names = ''; try { names = fs.readFileSync(path.join(dir, 'runtime-names.txt'), 'utf8'); } catch {}", - "const c = condition(pipelineRun);", - "console.log(JSON.stringify({", - " ok: exists('RUNTIME_NS_EXIT') && exists('CI_NS_EXIT'),", - " runtimeNamespaceExists: exists('RUNTIME_NS_EXIT'),", - " ciNamespaceExists: exists('CI_NS_EXIT'),", - " serviceAccountExists: exists('SERVICE_ACCOUNT_EXIT'),", - " pipeline: { exists: exists('PIPELINE_EXIT'), name: process.env.pipeline_name },", - " pipelineRun: { exists: exists('PIPELINERUN_EXIT'), name: process.env.pipeline_run || null, status: c?.status || null, reason: c?.reason || null, sourceCommit: pipelineRunParam(pipelineRun, 'revision'), startTime: pipelineRun?.status?.startTime || null, completionTime: pipelineRun?.status?.completionTime || null },", - " argo: { exists: exists('ARGO_EXIT'), namespace: process.env.argo_namespace, application: process.env.argo_application, revision: argo?.status?.sync?.revision || null, syncStatus: argo?.status?.sync?.status || null, healthStatus: argo?.status?.health?.status || null },", - " manager: { deploymentExists: exists('MANAGER_DEPLOY_EXIT'), serviceExists: exists('MANAGER_SVC_EXIT'), deployment: process.env.manager_deployment, service: process.env.manager_service, image: managerDeploy?.spec?.template?.spec?.containers?.[0]?.image || null, sourceCommit: envValue(managerDeploy, 'AGENTRUN_SOURCE_COMMIT'), servicePorts: Array.isArray(managerSvc?.spec?.ports) ? managerSvc.spec.ports.map((port) => ({ name: port.name || null, port: port.port || null, targetPort: port.targetPort || null })) : [] },", - " database: { secretPresent: exists('DB_SECRET_EXIT'), secretName: process.env.database_secret, key: process.env.database_key, keyPresent: Boolean(dbSecret?.data && Object.prototype.hasOwnProperty.call(dbSecret.data, process.env.database_key || '')) , valuesPrinted: false },", - " secrets,", - " localPostgres: { absent: !/postgres/i.test(names), matchingObjects: names.split(/\\r?\\n/).filter((line) => /postgres/i.test(line)).slice(0, 20) },", - " valuesPrinted: false", - "}));", - "NODE", + "NODE_TMP=\"$tmp_dir\" RUNTIME_NS_EXIT=\"$runtime_ns_exit\" CI_NS_EXIT=\"$ci_ns_exit\" PIPELINE_EXIT=\"$pipeline_exit\" SERVICE_ACCOUNT_EXIT=\"$sa_exit\" PIPELINERUN_EXIT=\"$pr_exit\" ARGO_EXIT=\"$argo_exit\" MANAGER_DEPLOY_EXIT=\"$manager_deploy_exit\" MANAGER_SVC_EXIT=\"$manager_svc_exit\" DB_SECRET_EXIT=\"$db_secret_exit\" python3 - <<'PY'", + "import json, os, pathlib, re", + "base = pathlib.Path(os.environ['NODE_TMP'])", + "def read_json(name):", + " try: return json.loads((base / name).read_text())", + " except Exception: return None", + "def exists(name): return os.environ.get(name) == '0'", + "def condition(obj):", + " conds = (((obj or {}).get('status') or {}).get('conditions') or [])", + " return conds[0] if conds else {}", + "def env_value(deploy, name):", + " containers = (((((deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [])", + " envs = (containers[0].get('env') or []) if containers else []", + " for entry in envs:", + " if entry.get('name') == name: return entry.get('value')", + " return None", + "def pipeline_run_param(obj, name):", + " for entry in (((obj or {}).get('spec') or {}).get('params') or []):", + " if entry.get('name') == name: return entry.get('value')", + " return None", + "pipeline_run = read_json('pipelinerun.json')", + "argo = read_json('argo.json')", + "manager_deploy = read_json('manager-deploy.json')", + "manager_svc = read_json('manager-svc.json')", + "db_secret = read_json('db-secret.json')", + "secrets = read_json('secrets.json') or {'ready': True, 'count': 0, 'items': [], 'valuesPrinted': False}", + "try: names = (base / 'runtime-names.txt').read_text()", + "except Exception: names = ''", + "c = condition(pipeline_run)", + "ports = []", + "for port in ((((manager_svc or {}).get('spec') or {}).get('ports')) or []):", + " ports.append({'name': port.get('name'), 'port': port.get('port'), 'targetPort': port.get('targetPort')})", + "matching_postgres = [line for line in re.split(r'\\r?\\n', names) if re.search('postgres', line, re.I)][:20]", + "print(json.dumps({", + " 'ok': exists('RUNTIME_NS_EXIT') and exists('CI_NS_EXIT'),", + " 'runtimeNamespaceExists': exists('RUNTIME_NS_EXIT'),", + " 'ciNamespaceExists': exists('CI_NS_EXIT'),", + " 'serviceAccountExists': exists('SERVICE_ACCOUNT_EXIT'),", + " 'pipeline': {'exists': exists('PIPELINE_EXIT'), 'name': os.environ.get('pipeline_name')},", + " 'pipelineRun': {'exists': exists('PIPELINERUN_EXIT'), 'name': os.environ.get('pipeline_run') or None, 'status': c.get('status'), 'reason': c.get('reason'), 'sourceCommit': pipeline_run_param(pipeline_run, 'revision'), 'startTime': ((pipeline_run or {}).get('status') or {}).get('startTime'), 'completionTime': ((pipeline_run or {}).get('status') or {}).get('completionTime')},", + " 'argo': {'exists': exists('ARGO_EXIT'), 'namespace': os.environ.get('argo_namespace'), 'application': os.environ.get('argo_application'), 'revision': (((argo or {}).get('status') or {}).get('sync') or {}).get('revision'), 'syncStatus': (((argo or {}).get('status') or {}).get('sync') or {}).get('status'), 'healthStatus': (((argo or {}).get('status') or {}).get('health') or {}).get('status')},", + " 'manager': {'deploymentExists': exists('MANAGER_DEPLOY_EXIT'), 'serviceExists': exists('MANAGER_SVC_EXIT'), 'deployment': os.environ.get('manager_deployment'), 'service': os.environ.get('manager_service'), 'image': ((((((manager_deploy or {}).get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [{}])[0]).get('image'), 'sourceCommit': env_value(manager_deploy, 'AGENTRUN_SOURCE_COMMIT'), 'servicePorts': ports},", + " 'database': {'secretPresent': exists('DB_SECRET_EXIT'), 'secretName': os.environ.get('database_secret'), 'key': os.environ.get('database_key'), 'keyPresent': os.environ.get('database_key') in (((db_secret or {}).get('data') or {})), 'valuesPrinted': False},", + " 'secrets': secrets,", + " 'localPostgres': {'absent': len(matching_postgres) == 0, 'matchingObjects': matching_postgres},", + " 'valuesPrinted': False,", + "}, ensure_ascii=False))", + "PY", ].join("\n"); } @@ -930,6 +998,234 @@ export function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: st ].join("\n"); } +export function yamlLaneK3sBuildImageJobManifest(spec: AgentRunLaneSpec, sourceCommit: string, jobName: string): Record { + const build = spec.deployment.manager.imageBuild; + if (spec.ci.buildkitImage === null) throw new Error(`config/agentrun.yaml controlPlane.lanes.${spec.lane}.ci.buildkitImage is required when source.statusMode=k3s-git-mirror`); + const imageRepository = `${spec.ci.registryPrefix}/${build.repository}`; + const envIdentity = `source-${sourceCommit.slice(0, 12)}`; + const image = `${imageRepository}:${envIdentity}`; + const proxyEnv = yamlLaneK3sBuildProxyEnv(spec); + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace: spec.ci.namespace, + labels: { + "app.kubernetes.io/name": "agentrun-manager-image-build", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/source-commit": sourceCommit, + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: Math.max(60, build.timeoutSeconds), + ttlSecondsAfterFinished: 3600, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "agentrun-manager-image-build", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/source-commit": sourceCommit, + }, + }, + spec: { + restartPolicy: "Never", + serviceAccountName: spec.ci.serviceAccountName, + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 }, + volumes: [ + { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, + { name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } }, + { name: "tmp", emptyDir: {} }, + ], + initContainers: [{ + name: "source", + image: spec.ci.toolsImage, + imagePullPolicy: "IfNotPresent", + env: proxyEnv, + command: ["/bin/sh", "-ec", yamlLaneK3sBuildSourceShell(spec, sourceCommit)], + volumeMounts: [{ name: "workspace", mountPath: "/workspace" }], + }], + containers: [{ + name: "buildkit", + image: spec.ci.buildkitImage, + imagePullPolicy: "IfNotPresent", + env: [ + ...proxyEnv, + { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }, + ], + command: ["/bin/sh", "-ec", yamlLaneK3sBuildImageShell(spec, sourceCommit, image, envIdentity)], + securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + { name: "tmp", mountPath: "/tmp" }, + ], + }], + }, + }, + }, + }; +} + +export async function runYamlLaneK3sBuildImageJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string): Promise & { ok: boolean; payload: Record }> { + const jobName = `agentrun-build-${spec.nodeId.toLowerCase()}-${spec.lane}-${sourceCommit.slice(0, 12)}`.slice(0, 63); + const manifest = yamlLaneK3sBuildImageJobManifest(spec, sourceCommit, jobName); + const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.ci.namespace, jobName, manifest)]); + const createPayload = captureJsonPayload(created); + if (created.exitCode !== 0 || createPayload.ok === false) { + return { + ok: false, + payload: { ok: false, status: "create-failed", jobName, degradedReason: "k3s-buildkit-job-create-failed", valuesPrinted: false }, + create: createPayload, + capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const startedAt = Date.now(); + const timeoutMs = Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds) * 1000; + const pollMs = Math.max(1, spec.deployment.manager.imageBuild.pollSeconds) * 1000; + let polls = 0; + let lastPayload: Record = {}; + let lastProbe: SshCaptureResult | null = null; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.ci.namespace, jobName)]); + const probePayload = captureJsonPayload(lastProbe); + const buildPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload); + if (Object.keys(buildPayload).length > 0) lastPayload = buildPayload; + progressEvent("agentrun.yaml-lane.k3s-image-build.progress", { + node: spec.nodeId, + lane: spec.lane, + sourceCommit, + jobName, + polls, + status: stringOrNull(buildPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"), + digest: stringOrNull(buildPayload.digest), + elapsedMs: Date.now() - startedAt, + }); + if (probePayload.succeeded === true) { + if (buildPayload.ok === true && stringOrNull(buildPayload.digest) !== null && stringOrNull(buildPayload.envIdentity) !== null) { + return { ok: true, payload: buildPayload, jobName, create: createPayload, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false }; + } + return { + ok: false, + payload: { ...buildPayload, ok: false, status: stringOrNull(buildPayload.status) ?? "succeeded-without-result", degradedReason: "k3s-buildkit-result-missing", jobName, valuesPrinted: false }, + jobName, + create: createPayload, + polls, + elapsedMs: Date.now() - startedAt, + probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + if (probePayload.failed === true) { + const payload = Object.keys(buildPayload).length > 0 ? buildPayload : { ok: false, status: "failed", degradedReason: "k3s-buildkit-job-failed", jobName, valuesPrinted: false }; + return { + ok: false, + payload, + jobName, + create: createPayload, + polls, + elapsedMs: Date.now() - startedAt, + probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + await sleep(pollMs); + } + return { + ok: false, + payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "k3s-buildkit-job-timeout", jobName, valuesPrinted: false }, + jobName, + create: createPayload, + polls, + elapsedMs: Date.now() - startedAt, + probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; +} + +function yamlLaneK3sBuildProxyEnv(spec: AgentRunLaneSpec): Array<{ name: string; value: string }> { + const build = spec.deployment.manager.imageBuild; + const result: Array<{ name: string; value: string }> = []; + const noProxy = build.noProxy.join(","); + const allProxy = build.httpsProxy ?? build.httpProxy; + if (build.httpProxy !== null) result.push({ name: "HTTP_PROXY", value: build.httpProxy }, { name: "http_proxy", value: build.httpProxy }); + if (build.httpsProxy !== null) result.push({ name: "HTTPS_PROXY", value: build.httpsProxy }, { name: "https_proxy", value: build.httpsProxy }); + if (allProxy !== null) result.push({ name: "ALL_PROXY", value: allProxy }, { name: "all_proxy", value: allProxy }); + result.push({ name: "NO_PROXY", value: noProxy }, { name: "no_proxy", value: noProxy }); + return result; +} + +function yamlLaneK3sBuildSourceShell(spec: AgentRunLaneSpec, sourceCommit: string): string { + return [ + "set -eu", + `read_url=${shQuote(spec.gitMirror.readUrl)}`, + `source_branch=${shQuote(spec.source.branch)}`, + `source_commit=${shQuote(sourceCommit)}`, + "rm -rf /workspace/repo", + "git clone --no-checkout \"$read_url\" /workspace/repo", + "cd /workspace/repo", + "git fetch origin \"refs/heads/$source_branch:refs/remotes/origin/$source_branch\"", + "git checkout --detach \"$source_commit\"", + "actual=$(git rev-parse HEAD)", + "test \"$actual\" = \"$source_commit\"", + "chmod -R a+rwX /workspace", + ].join("\n"); +} + +function yamlLaneK3sBuildImageShell(spec: AgentRunLaneSpec, sourceCommit: string, image: string, envIdentity: string): string { + const build = spec.deployment.manager.imageBuild; + const contextPath = build.context === "." ? "/workspace/repo" : `/workspace/repo/${build.context.replace(/^\.\//u, "")}`; + const buildContainerNoProxy = build.buildContainerProxy.noProxy.join(","); + const buildArgs = [ + ...Object.entries(build.buildArgs).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `build-arg:${key}=${value}`), + ...yamlLaneK3sBuildProxyBuildArgs(build.buildContainerProxy.httpProxy, build.buildContainerProxy.httpsProxy, buildContainerNoProxy), + ]; + const buildctlArgs = [ + "build", + "--allow", "network.host", + "--frontend", "dockerfile.v0", + "--local", `context=${contextPath}`, + "--local", "dockerfile=/workspace/repo", + "--opt", `filename=${build.containerfile}`, + "--opt", `network=${build.network}`, + ...buildArgs.flatMap((arg) => ["--opt", arg]), + "--metadata-file", "/workspace/build-metadata.json", + "--output", `type=image,name=${image},push=true,registry.insecure=true`, + ]; + const repositoryDigestBase = image.slice(0, image.lastIndexOf(":")); + return [ + "set -eu", + "cd /workspace/repo", + `buildctl-daemonless.sh ${buildctlArgs.map((arg) => shQuote(arg)).join(" ")}`, + "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)", + "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)", + "if [ -z \"$digest\" ]; then", + ` printf '{"ok":false,"status":"failed","failureKind":"image-digest-missing","sourceCommit":"%s","envIdentity":"%s","image":"%s","valuesPrinted":false}\\n' ${shQuote(sourceCommit)} ${shQuote(envIdentity)} ${shQuote(image)}`, + " exit 1", + "fi", + `printf '{"ok":true,"status":"built","sourceCommit":"%s","envIdentity":"%s","image":"%s","digest":"%s","repositoryDigest":"%s@%s","valuesPrinted":false}\\n' ${shQuote(sourceCommit)} ${shQuote(envIdentity)} ${shQuote(image)} "$digest" ${shQuote(repositoryDigestBase)} "$digest"`, + ].join("\n"); +} + +function yamlLaneK3sBuildProxyBuildArgs(httpProxy: string | null, httpsProxy: string | null, noProxy: string): string[] { + const result: string[] = []; + const allProxy = httpsProxy ?? httpProxy; + if (httpProxy !== null) result.push(`build-arg:HTTP_PROXY=${httpProxy}`, `build-arg:http_proxy=${httpProxy}`); + if (httpsProxy !== null) result.push(`build-arg:HTTPS_PROXY=${httpsProxy}`, `build-arg:https_proxy=${httpsProxy}`); + if (allProxy !== null) result.push(`build-arg:ALL_PROXY=${allProxy}`, `build-arg:all_proxy=${allProxy}`); + if (noProxy.length > 0) result.push(`build-arg:NO_PROXY=${noProxy}`, `build-arg:no_proxy=${noProxy}`); + return result; +} + export async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise & { ok: boolean; payload: Record }> { const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63); const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName); diff --git a/scripts/src/hwlab-node-control-plane-model.ts b/scripts/src/hwlab-node-control-plane-model.ts new file mode 100644 index 00000000..3a2412ce --- /dev/null +++ b/scripts/src/hwlab-node-control-plane-model.ts @@ -0,0 +1,371 @@ +export const HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH = "config/hwlab-node-control-plane.yaml"; + +export type InfraAction = "plan" | "status" | "apply"; +export type ToolsImageAction = "status" | "build" | "logs"; +export type TektonInstallAction = "status" | "apply" | "logs"; +export type ArgoAction = "status" | "apply" | "logs"; +export type EgressBenchmarkAction = "benchmark" | "status" | "logs"; +export type CiBuildBenchmarkAction = "benchmark" | "status" | "logs"; +export type K3sInstallAction = "plan" | "install" | "status"; + +export interface InfraOptions { + action: InfraAction; + node: string; + lane: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; +} + +export interface ToolsImageOptions { + action: ToolsImageAction; + node: string; + lane: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + tailLines: number; +} + +export interface TektonInstallOptions { + action: TektonInstallAction; + node: string; + lane: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + tailLines: number; +} + +export interface ArgoOptions { + action: ArgoAction; + node: string; + lane: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + tailLines: number; +} + +export interface EgressBenchmarkOptions { + action: EgressBenchmarkAction; + node: string; + lane: string; + profile: "no-mirror"; + dryRun: boolean; + confirm: boolean; + samples: number; + sampleTimeoutSeconds: number; + timeoutSeconds: number; + tailLines: number; +} + +export interface CiBuildBenchmarkOptions { + action: CiBuildBenchmarkAction; + node: string; + lane: string; + profile: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + tailLines: number; +} + +export interface K3sInstallOptions { + action: K3sInstallAction; + node: string; + lane: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + tailLines: number; +} + +export interface CiBuildBenchmarkCachePolicy { + noPipelineRunReuse: boolean; + forceFullBuild: boolean; + forbidGitopsCatalogReuse: boolean; + forbidDependencyCache: boolean; + forbidBuildkitCache: boolean; + forbidRegistryMirror: boolean; + forbidLocalPreheatedImages: boolean; +} + +export interface CiBuildBenchmarkProfileSpec { + profile: string; + runtimeLaneConfigRef: string; + pipelineRunPrefix: string; + catalogPathTemplate: string; + imageTagMode: "full"; + pipelineTimeoutSeconds: number; + cachePolicy: CiBuildBenchmarkCachePolicy; + requiredTimings: readonly string[]; + failureFamilies: readonly string[]; +} + +export type ControlPlaneEgressProxySpec = ControlPlaneK8sServiceEgressProxySpec | ControlPlaneHostRouteEgressProxySpec; + +export interface ControlPlaneK8sServiceEgressProxySpec { + mode: "k8s-service-cluster-ip"; + clientName: string; + namespace: string; + serviceName: string; + port: number; + sourceConfigRef: string | null; + sourceFingerprint: string | null; + sourceRef: string; + sourceKey: string; + sourceType: "subscription-url" | "master-shadowsocks"; + preferredOutbound: "vless-reality" | "hysteria2" | null; + noProxy: readonly string[]; +} + +export interface ControlPlaneHostRouteEgressProxySpec { + mode: "host-route"; + clientName: string; + hostProxyConfigRef: string; + proxyEnvPath: string; + proxyUrl: string; + noProxy: readonly string[]; +} + +export interface ControlPlaneGitMirrorEgressProxySpec { + mode: "node-global" | "host-route" | "direct"; + required: boolean; + podHostNetwork: boolean; + injectPodEnv: boolean; +} + +export interface ControlPlaneRuntimeProxySpec { + enabled: boolean; + mode: "host-route"; + configRef: string | null; + hostNetwork: boolean; + injectEnv: boolean; + deployments: readonly string[]; + statefulSets: readonly string[]; +} + +export type ControlPlaneGitMirrorGithubTransportSpec = + | { + mode: "ssh"; + privateKeySecretKey: string; + privateKeySourceRef: string; + privateKeySourceKey: string; + privateKeySourceEncoding: "plain" | "base64"; + knownHostsSecretKey: string | null; + knownHostsSourceRef: string | null; + knownHostsSourceKey: string | null; + knownHostsSourceEncoding: "plain" | "base64" | null; + } + | { + mode: "https"; + username: string; + tokenSecretName: string; + tokenSecretKey: string; + tokenSourceRef: string; + tokenSourceKey: string; + }; + +export interface ControlPlaneTektonGitWorkspaceSecretSpec { + name: string; + namespace: string; + sourceRefFrom: "gitMirror.githubTransport"; + privateKeySecretKey: string; + knownHostsSecretKey: string; +} + +export interface ControlPlaneTektonRuntimeObserverRbacSpec { + namespace: string; + roleName: string; + roleBindingName: string; +} + +export interface ControlPlaneTektonArgoObserverRbacSpec { + namespace: string; + roleName: string; + roleBindingName: string; +} + +export interface ControlPlaneTektonInstallManifestSpec { + name: string; + url: string; +} + +export interface ControlPlaneTektonInstallSpec { + enabled: boolean; + sourceKind: "url"; + version: string; + fieldManager: string; + manifests: readonly ControlPlaneTektonInstallManifestSpec[]; + requiredCrds: readonly string[]; + expectedDeploymentNamespaces: readonly string[]; + readinessTimeoutSeconds: number; + runtimeProxy: ControlPlaneRuntimeProxySpec; +} + +export interface ControlPlaneNodeSpec { + id: string; + route: string; + kubeRoute: string; + k3s: ControlPlaneK3sNodeSpec | null; + registry: { endpoint: string }; + egressProxy: ControlPlaneEgressProxySpec | null; +} + +export interface ControlPlaneK3sNodeSpec { + serviceName: string; + dropInPath: string; + nodeStatusName: string; + execStartPre: readonly (readonly string[])[]; + install: ControlPlaneK3sInstallSpec | null; + serverArgs: readonly string[]; + kubelet: { maxPods: number }; +} + +export interface ControlPlaneK3sInstallSpec { + enabled: boolean; + channel: string; + version: string; + installScriptUrl: string; + binaryUrl: string; + sha256Url: string; + expectedSha256: string; + hostProxyConfigRef: string; + proxyEnvPath: string; + registriesYamlPath: string; + localRegistry: { + containerName: string; + image: string; + canonicalImage: string; + bind: string; + }; + state: { + dir: string; + logPath: string; + statusPath: string; + }; + downloads: { + connectTimeoutSeconds: number; + maxTimeSeconds: number; + retry: number; + retryDelaySeconds: number; + }; +} + +export interface DockerfileInlineSpec { + filename: string; + lines: readonly string[]; +} + +export interface ImageRewriteSpec { + source: string; + pullImage: string; + target: string; +} + +export interface ControlPlaneTargetSpec { + id: string; + node: string; + lane: string; + enabled: boolean; + ciNamespace: string; + runtimeNamespace: string; + source: { repository: string; branch: string }; + gitops: { branch: string; path: string }; + gitMirror: { + namespace: string; + serviceReadName: string; + serviceWriteName: string; + cachePvcName: string; + cachePvcStorage: string; + cacheHostPath: string | null; + servicePort: number; + readContainerPort: number; + writeContainerPort: number; + deploymentReplicas: number; + secretName: string; + syncConfigMapName: string; + syncJobPrefix: string; + flushJobPrefix: string; + readUrl: string; + writeUrl: string; + egressProxy: ControlPlaneGitMirrorEgressProxySpec | null; + githubTransport: ControlPlaneGitMirrorGithubTransportSpec; + }; + tekton: { + install: ControlPlaneTektonInstallSpec; + pipelineName: string; + serviceAccountName: string; + pipelineRunPrefix: string; + gitWorkspaceSecret: ControlPlaneTektonGitWorkspaceSecretSpec; + runtimeObserverRbac: ControlPlaneTektonRuntimeObserverRbacSpec; + argoObserverRbac: ControlPlaneTektonArgoObserverRbacSpec; + toolsImage: { + output: string; + imagePullPolicy: "Always" | "IfNotPresent" | "Never"; + sourceKind: "dockerfile" | "docker-compose"; + context: string; + dockerfile?: string; + dockerfileInline?: DockerfileInlineSpec; + composeFile?: string; + buildArgs: Readonly>; + buildNetwork: string | null; + publicBaseImages: readonly string[]; + buildOwner: string; + buildMode: string; + }; + }; + ciBuildBenchmarks: readonly CiBuildBenchmarkProfileSpec[]; + argo: { + namespace: string; + projectName: string; + applicationName: string; + applicationFile: string; + install: { + enabled: boolean; + sourceKind: "url"; + version: string; + manifestUrl: string; + fieldManager: string; + imagePullPolicy: "Always" | "IfNotPresent" | "Never"; + preloadImages: readonly string[]; + imageRewrites: readonly ImageRewriteSpec[]; + requiredCrds: readonly string[]; + expectedDeployments: readonly string[]; + expectedStatefulSets: readonly string[]; + readinessTimeoutSeconds: number; + runtimeProxy: ControlPlaneRuntimeProxySpec; + }; + }; +} + +export interface ControlPlaneImagePolicy { + requireReproducibleBuildSource: boolean; + forbidPrivateOrNodeLocalImagesAsInputs: boolean; + allowNodeLocalRegistryAsBuildOutput: boolean; + requiredSourceKinds: readonly ("dockerfile" | "docker-compose")[]; +} + +export interface ControlPlaneConfig { + version: number; + kind: string; + metadata: { owner: string; relatedIssues: readonly number[] }; + imagePolicy: ControlPlaneImagePolicy; + nodes: Record; + targets: readonly ControlPlaneTargetSpec[]; +} + +export interface HwlabNodeControlPlaneSourceWorkspaceBootstrapSpec { + readonly configPath: string; + readonly targetId: string; + readonly node: string; + readonly lane: string; + readonly ciNamespace: string; + readonly serviceAccountName: string; + readonly toolsImage: string; + readonly imagePullPolicy: "Always" | "IfNotPresent" | "Never"; + readonly gitReadUrl: string; + readonly gitWriteUrl: string; + readonly gitMirrorNamespace: string; +} diff --git a/scripts/src/hwlab-node-control-plane-runtime.ts b/scripts/src/hwlab-node-control-plane-runtime.ts new file mode 100644 index 00000000..0e3413f5 --- /dev/null +++ b/scripts/src/hwlab-node-control-plane-runtime.ts @@ -0,0 +1,2616 @@ +import { createHash } from "node:crypto"; +import { rootPath } from "./config"; +import { runCommand, type CommandResult } from "./command"; +import { + HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + type ArgoOptions, + type CiBuildBenchmarkCachePolicy, + type CiBuildBenchmarkProfileSpec, + type ControlPlaneEgressProxySpec, + type ControlPlaneGitMirrorGithubTransportSpec, + type ControlPlaneHostRouteEgressProxySpec, + type ControlPlaneK3sInstallSpec, + type ControlPlaneK3sNodeSpec, + type ControlPlaneNodeSpec, + type ControlPlaneRuntimeProxySpec, + type ControlPlaneTargetSpec, + type TektonInstallOptions, + type ToolsImageOptions, +} from "./hwlab-node-control-plane-model"; + +export function argoDesiredManifest(target: ControlPlaneTargetSpec): Record[] { + return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; +} + +export function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record { + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "AppProject", + metadata: { name: target.argo.projectName, namespace: target.argo.namespace }, + spec: { + sourceRepos: [target.gitMirror.readUrl], + destinations: [{ server: "https://kubernetes.default.svc", namespace: target.runtimeNamespace }], + clusterResourceWhitelist: [{ group: "*", kind: "*" }], + namespaceResourceWhitelist: [{ group: "*", kind: "*" }], + }, + }; +} + +export function argoApplicationSkeleton(target: ControlPlaneTargetSpec): Record { + return { + apiVersion: "argoproj.io/v1alpha1", + kind: "Application", + metadata: { name: target.argo.applicationName, namespace: target.argo.namespace }, + spec: { + project: target.argo.projectName, + source: { repoURL: target.gitMirror.readUrl, targetRevision: target.gitops.branch, path: target.gitops.path }, + destination: { server: "https://kubernetes.default.svc", namespace: target.runtimeNamespace }, + syncPolicy: { automated: { prune: true, selfHeal: true } }, + }, + }; +} + +export function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { + return { + id: target.id, + node: node.id, + kubeRoute: node.kubeRoute, + lane: target.lane, + enabled: target.enabled, + ciNamespace: target.ciNamespace, + runtimeNamespace: target.runtimeNamespace, + k3sNodeConfig: k3sNodeConfigPlan(node), + registry: node.registry.endpoint, + egressProxy: controlPlaneEgressProxySummary(node.egressProxy), + sourceBranch: target.source.branch, + gitopsBranch: target.gitops.branch, + gitopsPath: target.gitops.path, + gitMirrorNamespace: target.gitMirror.namespace, + readUrl: target.gitMirror.readUrl, + writeUrl: target.gitMirror.writeUrl, + pipeline: target.tekton.pipelineName, + pipelineRunPrefix: target.tekton.pipelineRunPrefix, + serviceAccount: target.tekton.serviceAccountName, + gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), + runtimeObserverRbac: target.tekton.runtimeObserverRbac, + argoObserverRbac: target.tekton.argoObserverRbac, + tektonInstall: { + enabled: target.tekton.install.enabled, + version: target.tekton.install.version, + manifests: target.tekton.install.manifests, + requiredCrds: target.tekton.install.requiredCrds, + expectedDeploymentNamespaces: target.tekton.install.expectedDeploymentNamespaces, + }, + toolsImage: target.tekton.toolsImage, + argoApplication: target.argo.applicationName, + argoInstall: { + enabled: target.argo.install.enabled, + version: target.argo.install.version, + manifestUrl: target.argo.install.manifestUrl, + imageRewrites: target.argo.install.imageRewrites, + }, + }; +} + +export function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { + return { + sourceRepo: target.source.repository, + branch: target.source.branch, + gitopsBranch: target.gitops.branch, + runtimePath: target.gitops.path, + runtimeNamespace: target.runtimeNamespace, + namespace: target.ciNamespace, + k3sNodeConfig: k3sNodeConfigPlan(node), + gitMirror: { + namespace: target.gitMirror.namespace, + readUrl: target.gitMirror.readUrl, + writeUrl: target.gitMirror.writeUrl, + cachePvc: target.gitMirror.cachePvcName, + cachePvcStorage: target.gitMirror.cachePvcStorage, + cacheHostPath: target.gitMirror.cacheHostPath, + servicePort: target.gitMirror.servicePort, + readContainerPort: target.gitMirror.readContainerPort, + writeContainerPort: target.gitMirror.writeContainerPort, + deploymentReplicas: target.gitMirror.deploymentReplicas, + syncConfigMap: target.gitMirror.syncConfigMapName, + egressProxy: target.gitMirror.egressProxy, + effectiveEgressProxy: gitMirrorEffectiveEgressProxySummary(node, target), + githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport), + statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"], + }, + pipeline: target.tekton.pipelineName, + pipelineRunPrefix: target.tekton.pipelineRunPrefix, + serviceAccount: target.tekton.serviceAccountName, + gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), + runtimeObserverRbac: target.tekton.runtimeObserverRbac, + argoObserverRbac: target.tekton.argoObserverRbac, + tektonInstall: { + enabled: target.tekton.install.enabled, + sourceKind: target.tekton.install.sourceKind, + version: target.tekton.install.version, + manifests: target.tekton.install.manifests, + requiredCrds: target.tekton.install.requiredCrds, + expectedDeploymentNamespaces: target.tekton.install.expectedDeploymentNamespaces, + readinessTimeoutSeconds: target.tekton.install.readinessTimeoutSeconds, + }, + toolsImage: target.tekton.toolsImage, + argoNamespace: target.argo.namespace, + argoApplication: target.argo.applicationName, + argoInstall: { + enabled: target.argo.install.enabled, + sourceKind: target.argo.install.sourceKind, + version: target.argo.install.version, + manifestUrl: target.argo.install.manifestUrl, + preloadImages: target.argo.install.preloadImages, + imageRewrites: target.argo.install.imageRewrites, + requiredCrds: target.argo.install.requiredCrds, + expectedDeployments: target.argo.install.expectedDeployments, + expectedStatefulSets: target.argo.install.expectedStatefulSets, + }, + registry: node.registry.endpoint, + imagePolicy: { + noPrivateInputImages: true, + buildInput: { sourceKind: target.tekton.toolsImage.sourceKind, context: target.tekton.toolsImage.context, dockerfile: target.tekton.toolsImage.dockerfile ?? null, dockerfileInline: target.tekton.toolsImage.dockerfileInline ?? null, composeFile: target.tekton.toolsImage.composeFile ?? null, publicBaseImages: target.tekton.toolsImage.publicBaseImages }, + outputImage: target.tekton.toolsImage.output, + }, + }; +} + +export function k3sNodeConfigPlan(node: ControlPlaneNodeSpec): Record { + if (node.k3s === null) return { managed: false }; + const dropIn = k3sDropInContent(node.k3s); + return { + managed: true, + serviceName: node.k3s.serviceName, + dropInPath: node.k3s.dropInPath, + nodeStatusName: node.k3s.nodeStatusName, + desiredMaxPods: node.k3s.kubelet.maxPods, + dropInSha256: sha256Short(dropIn), + execStartPreCount: node.k3s.execStartPre.length, + serverArgCount: node.k3s.serverArgs.length, + }; +} + +export function k3sInstallPlan(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, spec: ControlPlaneK3sInstallSpec): Record { + return { + node: node.id, + route: node.route, + kubeRoute: node.kubeRoute, + lane: target.lane, + version: spec.version, + channel: spec.channel, + installScriptUrl: spec.installScriptUrl, + binaryUrl: spec.binaryUrl, + sha256Url: spec.sha256Url, + expectedSha256: `sha256:${spec.expectedSha256}`, + hostProxyConfigRef: spec.hostProxyConfigRef, + proxyEnvPath: spec.proxyEnvPath, + registriesYamlPath: spec.registriesYamlPath, + state: spec.state, + localRegistry: spec.localRegistry, + serviceName: node.k3s?.serviceName ?? null, + nodeStatusName: node.k3s?.nodeStatusName ?? null, + serverArgCount: node.k3s?.serverArgs.length ?? 0, + valuesPrinted: false, + }; +} + +export function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string { + return [ + "# Managed by UniDesk. Source: config/hwlab-node-control-plane.yaml nodes..k3s", + "[Service]", + ...spec.execStartPre.map((command) => `ExecStartPre=${command.map(systemdExecArg).join(" ")}`), + "ExecStart=", + `ExecStart=${["/usr/local/bin/k3s", ...spec.serverArgs].map(systemdExecArg).join(" ")}`, + "", + ].join("\n"); +} + +export function k3sServiceUnitContent(spec: ControlPlaneK3sNodeSpec, install: ControlPlaneK3sInstallSpec): string { + return [ + "[Unit]", + "Description=Lightweight Kubernetes", + "Documentation=https://k3s.io", + "Wants=network-online.target", + "After=network-online.target", + "", + "[Install]", + "WantedBy=multi-user.target", + "", + "[Service]", + "Type=notify", + `EnvironmentFile=-${install.proxyEnvPath}`, + "KillMode=process", + "Delegate=yes", + "LimitNOFILE=1048576", + "LimitNPROC=infinity", + "LimitCORE=infinity", + "TasksMax=infinity", + "TimeoutStartSec=0", + "Restart=always", + "RestartSec=5s", + "ExecStartPre=-/sbin/modprobe br_netfilter", + "ExecStartPre=-/sbin/modprobe overlay", + `ExecStart=${["/usr/local/bin/k3s", ...spec.serverArgs].map(systemdExecArg).join(" ")}`, + "", + ].join("\n"); +} + +export function k3sRegistriesYaml(install: ControlPlaneK3sInstallSpec): string { + const endpoint = install.localRegistry.bind.split(":").slice(0, 2).join(":"); + return [ + "mirrors:", + ` "${endpoint}":`, + " endpoint:", + ` - "http://${endpoint}"`, + "configs:", + ` "${endpoint}":`, + " tls:", + " insecure_skip_verify: true", + "", + ].join("\n"); +} + +export function controlPlaneEgressProxySummary(proxy: ControlPlaneEgressProxySpec | null): Record | null { + if (proxy === null) return null; + if (proxy.mode === "host-route") { + return { + mode: proxy.mode, + clientName: proxy.clientName, + hostProxyConfigRef: proxy.hostProxyConfigRef, + proxyEnvPath: proxy.proxyEnvPath, + proxyUrl: proxy.proxyUrl, + noProxyCount: proxy.noProxy.length, + valuesPrinted: false, + }; + } + return { + mode: proxy.mode, + clientName: proxy.clientName, + namespace: proxy.namespace, + serviceName: proxy.serviceName, + port: proxy.port, + sourceConfigRef: proxy.sourceConfigRef, + sourceType: proxy.sourceType, + sourceRef: proxy.sourceRef, + sourceKey: proxy.sourceKey, + sourceFingerprint: proxy.sourceFingerprint, + preferredOutbound: proxy.preferredOutbound, + valuesPrinted: false, + }; +} + +export function hostRouteNoProxy(proxy: ControlPlaneHostRouteEgressProxySpec): readonly string[] { + return [...new Set(["localhost", "127.0.0.1", "::1", "127.0.0.1:5000", "localhost:5000", ...proxy.noProxy])]; +} + +export function runtimeHostProxyConfig(node: ControlPlaneNodeSpec, spec: ControlPlaneRuntimeProxySpec): Record { + if (!spec.enabled) { + return { + enabled: false, + mode: "host-route", + configRef: spec.configRef, + hostNetwork: false, + injectEnv: false, + deployments: [], + statefulSets: [], + valuesPrinted: false, + }; + } + if (node.egressProxy?.mode !== "host-route") { + throw new Error(`runtimeProxy enabled for ${node.id} requires nodes.${node.id}.egressProxy.mode=host-route`); + } + return { + enabled: true, + mode: spec.mode, + configRef: spec.configRef, + hostNetwork: spec.hostNetwork, + injectEnv: spec.injectEnv, + deployments: spec.deployments, + statefulSets: spec.statefulSets, + proxyUrl: node.egressProxy.proxyUrl, + noProxy: hostRouteNoProxy(node.egressProxy), + valuesPrinted: false, + }; +} + +export function runtimeHostProxyEnv(node: ControlPlaneNodeSpec, spec: ControlPlaneRuntimeProxySpec): readonly Record[] { + if (!spec.enabled || !spec.injectEnv) return []; + if (node.egressProxy?.mode !== "host-route") { + throw new Error(`runtimeProxy env for ${node.id} requires nodes.${node.id}.egressProxy.mode=host-route`); + } + const proxyUrl = node.egressProxy.proxyUrl; + const noProxy = hostRouteNoProxy(node.egressProxy).join(","); + return [ + { name: "HTTP_PROXY", value: proxyUrl }, + { name: "HTTPS_PROXY", value: proxyUrl }, + { name: "ALL_PROXY", value: proxyUrl }, + { name: "NO_PROXY", value: noProxy }, + { name: "http_proxy", value: proxyUrl }, + { name: "https_proxy", value: proxyUrl }, + { name: "all_proxy", value: proxyUrl }, + { name: "no_proxy", value: noProxy }, + ]; +} + +export function runtimeProxyReady(status: Record): boolean { + const proxy = record(status.runtimeProxy); + return proxy.enabled !== true || boolField(proxy, "ready"); +} + +export function gitMirrorRuntimeProxySpec(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): ControlPlaneRuntimeProxySpec { + const config = target.gitMirror.egressProxy; + if (config === null || config.mode !== "host-route") { + return { enabled: false, mode: "host-route", configRef: null, hostNetwork: false, injectEnv: false, deployments: [], statefulSets: [] }; + } + return { + enabled: config.podHostNetwork || config.injectPodEnv, + mode: "host-route", + configRef: `nodes.${node.id}.egressProxy`, + hostNetwork: config.podHostNetwork, + injectEnv: config.injectPodEnv, + deployments: [], + statefulSets: [], + }; +} + +export function gitMirrorEffectiveEgressProxySummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { + const config = target.gitMirror.egressProxy; + if (config === null || config.mode === "direct") { + return { + mode: "direct", + required: false, + transport: target.gitMirror.githubTransport.mode, + valuesPrinted: false, + }; + } + const proxy = node.egressProxy; + if (config.mode === "host-route") { + return { + mode: config.mode, + required: config.required, + transport: target.gitMirror.githubTransport.mode, + podHostNetwork: config.podHostNetwork, + injectPodEnv: config.injectPodEnv, + ready: proxy?.mode === "host-route", + nodeProxy: controlPlaneEgressProxySummary(proxy), + valuesPrinted: false, + }; + } + return { + mode: config.mode, + required: config.required, + transport: target.gitMirror.githubTransport.mode, + podHostNetwork: config.podHostNetwork, + injectPodEnv: config.injectPodEnv, + ready: proxy !== null, + nodeProxy: controlPlaneEgressProxySummary(proxy), + valuesPrinted: false, + }; +} + +export function gitMirrorGithubTransportSummary(transport: ControlPlaneGitMirrorGithubTransportSpec): Record { + if (transport.mode === "ssh") { + return { + mode: "ssh", + privateKeySecretKey: transport.privateKeySecretKey, + privateKeySourceRef: transport.privateKeySourceRef, + privateKeySourceKey: transport.privateKeySourceKey, + privateKeySourceEncoding: transport.privateKeySourceEncoding, + knownHostsSecretKey: transport.knownHostsSecretKey, + knownHostsSourceRef: transport.knownHostsSourceRef, + knownHostsSourceKey: transport.knownHostsSourceKey, + knownHostsSourceEncoding: transport.knownHostsSourceEncoding, + valuesPrinted: false, + }; + } + return { + mode: "https", + username: transport.username, + tokenSecretName: transport.tokenSecretName, + tokenSecretKey: transport.tokenSecretKey, + tokenSourceRef: transport.tokenSourceRef, + tokenSourceKey: transport.tokenSourceKey, + valuesPrinted: false, + }; +} + +export function tektonGitWorkspaceSecretSummary(target: ControlPlaneTargetSpec): Record { + const transport = target.gitMirror.githubTransport; + const secret = target.tekton.gitWorkspaceSecret; + return { + name: secret.name, + namespace: secret.namespace, + sourceRefFrom: secret.sourceRefFrom, + privateKeySecretKey: secret.privateKeySecretKey, + privateKeySourceRef: transport.mode === "ssh" ? transport.privateKeySourceRef : null, + privateKeySourceKey: transport.mode === "ssh" ? transport.privateKeySourceKey : null, + knownHostsSecretKey: secret.knownHostsSecretKey, + knownHostsSourceRef: transport.mode === "ssh" ? transport.knownHostsSourceRef : null, + knownHostsSourceKey: transport.mode === "ssh" ? transport.knownHostsSourceKey : null, + valuesPrinted: false, + }; +} + +export function systemdExecArg(value: string): string { + if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value; + return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`; +} + +export function k3sInstallSubmitScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, install: ControlPlaneK3sInstallSpec): string { + if (node.k3s === null) throw new Error("k3sInstallSubmitScript requires node.k3s"); + const runner = k3sInstallRunnerScript(node, target, install); + const encoded = Buffer.from(runner, "utf8").toString("base64"); + return ` +set -eu +state_dir=${shQuote(install.state.dir)} +runner="$state_dir/install-runner.sh" +pid_file="$state_dir/pid" +status_file=${shQuote(install.state.statusPath)} +mkdir -p "$state_dir" +if [ -s "$pid_file" ]; then + pid="$(cat "$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" >/dev/null 2>&1; then + python3 - "$pid" "$status_file" <<'PY' +import json, pathlib, sys +status = None +path = pathlib.Path(sys.argv[2]) +if path.exists(): + try: + status = json.loads(path.read_text()) + except Exception as exc: + status = {"parseError": str(exc)} +print(json.dumps({"ok": True, "alreadyRunning": True, "pid": sys.argv[1], "status": status, "valuesPrinted": False}, ensure_ascii=False)) +PY + exit 0 + fi +fi +printf %s ${shQuote(encoded)} | base64 -d >"$runner" +chmod 0700 "$runner" +cat >"$status_file" <${shQuote(install.state.logPath)} 2>&1 & +pid=$! +printf '%s\\n' "$pid" >"$pid_file" +python3 - "$pid" "$state_dir" "$status_file" <<'PY' +import json, sys +print(json.dumps({"ok": True, "submitted": True, "pid": sys.argv[1], "stateDir": sys.argv[2], "statusPath": sys.argv[3], "valuesPrinted": False}, ensure_ascii=False)) +PY +`; +} + +export function k3sInstallRunnerScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, install: ControlPlaneK3sInstallSpec): string { + if (node.k3s === null) throw new Error("k3sInstallRunnerScript requires node.k3s"); + const unit = Buffer.from(k3sServiceUnitContent(node.k3s, install), "utf8").toString("base64"); + const dropIn = Buffer.from(k3sDropInContent(node.k3s), "utf8").toString("base64"); + const registries = Buffer.from(k3sRegistriesYaml(install), "utf8").toString("base64"); + return `#!/bin/sh +set -eu +state_dir=${shQuote(install.state.dir)} +status_file=${shQuote(install.state.statusPath)} +proxy_env=${shQuote(install.proxyEnvPath)} +registries_yaml=${shQuote(install.registriesYamlPath)} +binary_url=${shQuote(install.binaryUrl)} +sha256_url=${shQuote(install.sha256Url)} +expected_sha=${shQuote(install.expectedSha256)} +version=${shQuote(install.version)} +node_name=${shQuote(node.k3s.nodeStatusName)} +service_name=${shQuote(node.k3s.serviceName)} +registry_name=${shQuote(install.localRegistry.containerName)} +registry_image=${shQuote(install.localRegistry.image)} +registry_bind=${shQuote(install.localRegistry.bind)} +log_status() { + phase="$1"; ok="$2"; message="$3" + python3 - "$status_file" "$phase" "$ok" "$message" <<'PY' +import json, pathlib, sys, datetime +path = pathlib.Path(sys.argv[1]) +payload = { + "ok": sys.argv[3] == "true", + "phase": sys.argv[2], + "message": sys.argv[4], + "updatedAt": datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z", + "node": "${node.id}", + "lane": "${target.lane}", + "valuesPrinted": False, +} +path.parent.mkdir(parents=True, exist_ok=True) +path.write_text(json.dumps(payload, ensure_ascii=False) + "\\n") +PY +} +log_status starting false "k3s install started" +mkdir -p "$state_dir" /usr/local/bin /etc/rancher/k3s /etc/systemd/system/k3s.service.d +if [ ! -s "$proxy_env" ]; then + log_status failed false "proxy env missing" + exit 41 +fi +. "$proxy_env" +export HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy +curl -fsS --max-time 20 --proxy "$HTTPS_PROXY" -o /dev/null https://www.gstatic.com/generate_204 +log_status proxy-ready false "host proxy probe passed" +if command -v docker >/dev/null 2>&1; then + if docker ps --format '{{.Names}}' | grep -Fx "$registry_name" >/dev/null 2>&1; then + : + elif docker ps -a --format '{{.Names}}' | grep -Fx "$registry_name" >/dev/null 2>&1; then + docker start "$registry_name" >/dev/null 2>&1 || true + else + docker run -d --restart unless-stopped --name "$registry_name" -p "$registry_bind" "$registry_image" >/dev/null 2>&1 || true + fi +fi +log_status downloading false "downloading k3s binary through host proxy" +current_sha="$(sha256sum /usr/local/bin/k3s 2>/dev/null | cut -d' ' -f1 || true)" +if [ "$current_sha" != "$expected_sha" ]; then + sha_file="$state_dir/sha256sum-amd64.txt" + curl -fL --connect-timeout ${install.downloads.connectTimeoutSeconds} --max-time ${install.downloads.maxTimeSeconds} --retry ${install.downloads.retry} --retry-delay ${install.downloads.retryDelaySeconds} -o "$sha_file" "$sha256_url" + if ! grep -F "$expected_sha" "$sha_file" >/dev/null 2>&1; then + log_status failed false "expected k3s sha256 missing from upstream sha256 file" + exit 44 + fi + tmp_bin=/usr/local/bin/k3s.tmp.$$ + curl -fL --connect-timeout ${install.downloads.connectTimeoutSeconds} --max-time ${install.downloads.maxTimeSeconds} --retry ${install.downloads.retry} --retry-delay ${install.downloads.retryDelaySeconds} -o "$tmp_bin" "$binary_url" + actual_sha=$(sha256sum "$tmp_bin" | cut -d' ' -f1) + if [ "$actual_sha" != "$expected_sha" ]; then + rm -f "$tmp_bin" + log_status failed false "k3s binary sha256 mismatch" + exit 42 + fi + chmod 0755 "$tmp_bin" + mv -f "$tmp_bin" /usr/local/bin/k3s +fi +ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl +ln -sf /usr/local/bin/k3s /usr/local/bin/crictl +ln -sf /usr/local/bin/k3s /usr/local/bin/ctr +printf %s ${shQuote(registries)} | base64 -d >"$registries_yaml" +printf %s ${shQuote(unit)} | base64 -d >/etc/systemd/system/k3s.service +printf %s ${shQuote(dropIn)} | base64 -d >${shQuote(node.k3s.dropInPath)} +log_status systemd false "starting k3s service" +systemctl daemon-reload +systemctl enable --now "$service_name" +for _ in $(seq 1 120); do + if /usr/local/bin/kubectl get node "$node_name" >/tmp/unidesk-k3s-node.out 2>/tmp/unidesk-k3s-node.err; then + if /usr/local/bin/kubectl get node "$node_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -Fx True >/dev/null 2>&1; then + log_status succeeded true "k3s node ready" + exit 0 + fi + fi + sleep 5 +done +log_status failed false "timed out waiting for k3s node ready" +exit 43 +`; +} + +export function k3sInstallStatusScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, install: ControlPlaneK3sInstallSpec, tailLines: number): string { + if (node.k3s === null) throw new Error("k3sInstallStatusScript requires node.k3s"); + return ` +set +e +status_path=${shQuote(install.state.statusPath)} +log_path=${shQuote(install.state.logPath)} +pid_file=${shQuote(`${install.state.dir}/pid`)} +service_name=${shQuote(node.k3s.serviceName)} +node_name=${shQuote(node.k3s.nodeStatusName)} +expected_sha=${shQuote(install.expectedSha256)} +proxy_env=${shQuote(install.proxyEnvPath)} +registry_name=${shQuote(install.localRegistry.containerName)} +running=false +pid= +if [ -s "$pid_file" ]; then + pid="$(cat "$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" >/dev/null 2>&1; then running=true; fi +fi +service_active="$(systemctl is-active "$service_name" 2>/dev/null || true)" +binary_sha="$(sha256sum /usr/local/bin/k3s 2>/dev/null | cut -d' ' -f1 || true)" +proxy_probe=false +if [ -s "$proxy_env" ]; then + . "$proxy_env" + curl -fsS --max-time 20 --proxy "$HTTPS_PROXY" -o /dev/null https://www.gstatic.com/generate_204 >/dev/null 2>&1 && proxy_probe=true +fi +registry_running=false +if command -v docker >/dev/null 2>&1; then + docker ps --format '{{.Names}}' 2>/dev/null | grep -Fx "$registry_name" >/dev/null 2>&1 && registry_running=true +fi +node_ready=false +node_capacity= +node_allocatable= +if command -v kubectl >/dev/null 2>&1; then + ready="$(kubectl get node "$node_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + [ "$ready" = "True" ] && node_ready=true + node_capacity="$(kubectl get node "$node_name" -o jsonpath='{.status.capacity.pods}' 2>/dev/null || true)" + node_allocatable="$(kubectl get node "$node_name" -o jsonpath='{.status.allocatable.pods}' 2>/dev/null || true)" +fi +python3 - "$status_path" "$log_path" "$running" "$pid" "$service_active" "$binary_sha" "$expected_sha" "$proxy_probe" "$registry_running" "$node_ready" "$node_capacity" "$node_allocatable" ${shQuote(String(tailLines))} <<'PY' +import json, pathlib, sys +status_path = pathlib.Path(sys.argv[1]) +log_path = pathlib.Path(sys.argv[2]) +status = None +if status_path.exists(): + try: + status = json.loads(status_path.read_text()) + except Exception as exc: + status = {"parseError": str(exc), "rawTail": status_path.read_text(errors="replace")[-1000:]} +tail_lines = int(sys.argv[13]) +log_tail = "" +if log_path.exists(): + log_tail = "\\n".join(log_path.read_text(errors="replace").splitlines()[-tail_lines:]) +binary_sha = sys.argv[6] +payload = { + "node": "${node.id}", + "lane": "${target.lane}", + "running": sys.argv[3] == "true", + "pid": sys.argv[4] or None, + "status": status, + "checks": { + "serviceActive": sys.argv[5] == "active", + "binarySha256Ok": binary_sha == sys.argv[7], + "binarySha256Prefix": binary_sha[:12] if binary_sha else "", + "proxyProbe": sys.argv[8] == "true", + "registryRunning": sys.argv[9] == "true", + "nodeReady": sys.argv[10] == "true", + "capacityPods": int(sys.argv[11]) if sys.argv[11].isdigit() else None, + "allocatablePods": int(sys.argv[12]) if sys.argv[12].isdigit() else None, + }, + "logBytes": log_path.stat().st_size if log_path.exists() else 0, + "logTail": log_tail, + "valuesPrinted": False, +} +print(json.dumps(payload, ensure_ascii=False)) +PY +`; +} + +export function statusScript(nodeSpec: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { + const tektonRequiredCrds = shellJsonArray(target.tekton.install.requiredCrds); + const tektonDeploymentNamespaces = shellJsonArray(target.tekton.install.expectedDeploymentNamespaces); + const requiredCrds = shellJsonArray(target.argo.install.requiredCrds); + const argoDeployments = shellJsonArray(target.argo.install.expectedDeployments); + const argoStatefulSets = shellJsonArray(target.argo.install.expectedStatefulSets); + const tektonRuntimeProxy = JSON.stringify(runtimeHostProxyConfig(nodeSpec, target.tekton.install.runtimeProxy)); + const argoRuntimeProxy = JSON.stringify(runtimeHostProxyConfig(nodeSpec, target.argo.install.runtimeProxy)); + const k3s = nodeSpec.k3s; + const k3sDropIn = k3s === null ? "" : k3sDropInContent(k3s); + const gitMirrorEgressProxyJson = JSON.stringify(gitMirrorEffectiveEgressProxySummary(nodeSpec, target)); + return ` +set +e +node=${shQuote(target.node)} +lane=${shQuote(target.lane)} +ci_ns=${shQuote(target.ciNamespace)} +runtime_ns=${shQuote(target.runtimeNamespace)} +gitmirror_ns=${shQuote(target.gitMirror.namespace)} +read_deploy=${shQuote(target.gitMirror.serviceReadName)} +write_deploy=${shQuote(target.gitMirror.serviceWriteName)} +read_svc=${shQuote(target.gitMirror.serviceReadName)} +write_svc=${shQuote(target.gitMirror.serviceWriteName)} +cache_pvc=${shQuote(target.gitMirror.cachePvcName)} +cache_host_path=${shQuote(target.gitMirror.cacheHostPath ?? "")} +github_transport_mode=${shQuote(target.gitMirror.githubTransport.mode)} +github_ssh_secret=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.secretName : "")} +github_ssh_private_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySecretKey : "")} +github_ssh_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")} +github_ssh_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")} +github_ssh_known_hosts_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSecretKey ?? "" : "")} +github_ssh_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")} +github_ssh_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")} +github_token_secret=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSecretName : "")} +github_token_key=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSecretKey : "")} +github_token_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSourceRef : "")} +github_token_source_key=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSourceKey : "")} +gitmirror_egress_proxy_json=${shQuote(gitMirrorEgressProxyJson)} +pipeline=${shQuote(target.tekton.pipelineName)} +service_account=${shQuote(target.tekton.serviceAccountName)} +runtime_observer_role=${shQuote(target.tekton.runtimeObserverRbac.roleName)} +runtime_observer_rolebinding=${shQuote(target.tekton.runtimeObserverRbac.roleBindingName)} +ci_git_secret=${shQuote(target.tekton.gitWorkspaceSecret.name)} +ci_git_private_key=${shQuote(target.tekton.gitWorkspaceSecret.privateKeySecretKey)} +ci_git_known_hosts_key=${shQuote(target.tekton.gitWorkspaceSecret.knownHostsSecretKey)} +ci_git_source_ref_from=${shQuote(target.tekton.gitWorkspaceSecret.sourceRefFrom)} +ci_git_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")} +ci_git_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")} +ci_git_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")} +ci_git_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")} +argo_ns=${shQuote(target.argo.namespace)} +argo_project=${shQuote(target.argo.projectName)} +argo_app=${shQuote(target.argo.applicationName)} +argo_observer_role=${shQuote(target.tekton.argoObserverRbac.roleName)} +argo_observer_rolebinding=${shQuote(target.tekton.argoObserverRbac.roleBindingName)} +registry=${shQuote(nodeSpec.registry.endpoint)} +tools_image=${shQuote(target.tekton.toolsImage.output)} +tekton_required_crds_json=${shQuote(tektonRequiredCrds)} +tekton_deployment_namespaces_json=${shQuote(tektonDeploymentNamespaces)} +required_crds_json=${shQuote(requiredCrds)} +argo_deployments_json=${shQuote(argoDeployments)} +argo_statefulsets_json=${shQuote(argoStatefulSets)} +tekton_runtime_proxy_json=${shQuote(tektonRuntimeProxy)} +argo_runtime_proxy_json=${shQuote(argoRuntimeProxy)} +k3s_managed=${k3s === null ? "false" : "true"} +k3s_service=${shQuote(k3s?.serviceName ?? "")} +k3s_dropin=${shQuote(k3s?.dropInPath ?? "")} +k3s_node=${shQuote(k3s?.nodeStatusName ?? "")} +k3s_desired_max_pods=${shQuote(String(k3s?.kubelet.maxPods ?? ""))} +k3s_expected_sha=${shQuote(k3s === null ? "" : sha256Short(k3sDropIn))} +exists_ns() { kubectl get ns "$1" >/dev/null 2>&1 && printf true || printf false; } +exists_res() { kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 && printf true || printf false; } +deploy_ready() { desired=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } +sts_ready() { desired=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } +endpoint_ready() { endpoints=$(kubectl -n "$1" get endpoints "$2" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n "$endpoints" ] && printf true || printf false; } +can_i_runtime() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$runtime_ns" >/dev/null 2>&1 && printf true || printf false; } +can_i_argo() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$argo_ns" >/dev/null 2>&1 && printf true || printf false; } +runtime_observer_role_exists=$(exists_res "$runtime_ns" role "$runtime_observer_role") +runtime_observer_rolebinding_exists=$(exists_res "$runtime_ns" rolebinding "$runtime_observer_rolebinding") +runtime_observer_can_list_deployments=$(can_i_runtime list deployments.apps) +runtime_observer_can_list_statefulsets=$(can_i_runtime list statefulsets.apps) +runtime_observer_ready=false +if [ "$runtime_observer_role_exists" = true ] && [ "$runtime_observer_rolebinding_exists" = true ] && [ "$runtime_observer_can_list_deployments" = true ] && [ "$runtime_observer_can_list_statefulsets" = true ]; then runtime_observer_ready=true; fi +argo_observer_role_exists=$(exists_res "$argo_ns" role "$argo_observer_role") +argo_observer_rolebinding_exists=$(exists_res "$argo_ns" rolebinding "$argo_observer_rolebinding") +argo_observer_can_get_application=$(can_i_argo get applications.argoproj.io) +argo_observer_ready=false +if [ "$argo_observer_role_exists" = true ] && [ "$argo_observer_rolebinding_exists" = true ] && [ "$argo_observer_can_get_application" = true ]; then argo_observer_ready=true; fi +github_transport_json=$(python3 - "$github_transport_mode" "$gitmirror_ns" "$github_ssh_secret" "$github_ssh_private_key" "$github_ssh_private_source_ref" "$github_ssh_private_source_key" "$github_ssh_known_hosts_key" "$github_ssh_known_hosts_source_ref" "$github_ssh_known_hosts_source_key" "$github_token_secret" "$github_token_key" "$github_token_source_ref" "$github_token_source_key" <<'PY' +import hashlib, json, subprocess, sys +mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14] +def read_secret(name): + proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", name, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if proc.returncode != 0: + return False, {}, {} + try: + obj = json.loads(proc.stdout) + except Exception: + obj = {} + return True, obj.get("data") or {}, obj.get("metadata", {}).get("annotations") or {} +def fingerprint(value): + return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None +if mode == "ssh": + exists, data, annotations = read_secret(ssh_secret) + private_encoded = data.get(ssh_private_key) if isinstance(data, dict) else None + known_hosts_encoded = data.get(ssh_known_hosts_key) if ssh_known_hosts_key and isinstance(data, dict) else None + private_present = isinstance(private_encoded, str) and len(private_encoded) > 0 + known_hosts_expected = bool(ssh_known_hosts_key) + known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0 + print(json.dumps({ + "mode": mode, + "required": True, + "ready": exists and private_present and (not known_hosts_expected or known_hosts_present), + "secretName": ssh_secret, + "privateKeySecretKey": ssh_private_key, + "privateKeySourceRef": ssh_private_source_ref, + "privateKeySourceKey": ssh_private_source_key, + "privateKeySecretExists": exists, + "privateKeyPresent": private_present, + "privateKeyBytes": len(private_encoded) if private_present else 0, + "privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded), + "knownHostsSecretKey": ssh_known_hosts_key or None, + "knownHostsSourceRef": ssh_known_hosts_source_ref or None, + "knownHostsSourceKey": ssh_known_hosts_source_key or None, + "knownHostsPresent": (known_hosts_present if known_hosts_expected else None), + "knownHostsBytes": (len(known_hosts_encoded) if known_hosts_present else 0) if known_hosts_expected else None, + "knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded), + "valuesPrinted": False, + })) + raise SystemExit(0) +if mode != "https": + print(json.dumps({"mode": mode, "required": True, "ready": False, "valuesPrinted": False})) + raise SystemExit(0) +exists, data, _ = read_secret(token_secret) +encoded = data.get(token_key) if isinstance(data, dict) else None +present = isinstance(encoded, str) and len(encoded) > 0 +print(json.dumps({"mode": mode, "required": True, "ready": exists and present, "tokenSecretName": token_secret, "tokenSecretKey": token_key, "tokenSourceRef": token_source_ref, "tokenSourceKey": token_source_key, "tokenSecretExists": exists, "tokenKeyPresent": present, "tokenKeyBytes": len(encoded) if present else 0, "tokenFingerprint": fingerprint(encoded), "valuesPrinted": False})) +PY +) +ci_git_workspace_json=$(python3 - "$ci_ns" "$ci_git_secret" "$ci_git_private_key" "$ci_git_known_hosts_key" "$ci_git_source_ref_from" "$ci_git_private_source_ref" "$ci_git_private_source_key" "$ci_git_known_hosts_source_ref" "$ci_git_known_hosts_source_key" <<'PY' +import hashlib, json, subprocess, sys +namespace, secret, private_key, known_hosts_key, source_ref_from, private_source_ref, private_source_key, known_hosts_source_ref, known_hosts_source_key = sys.argv[1:10] +proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", secret, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +exists = proc.returncode == 0 +obj = {} +if exists: + try: + obj = json.loads(proc.stdout) + except Exception: + obj = {} +data = obj.get("data") or {} +annotations = obj.get("metadata", {}).get("annotations") or {} +def fingerprint(value): + return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None +private_encoded = data.get(private_key) if isinstance(data, dict) else None +known_hosts_encoded = data.get(known_hosts_key) if isinstance(data, dict) else None +private_present = isinstance(private_encoded, str) and len(private_encoded) > 0 +known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0 +print(json.dumps({ + "required": True, + "ready": exists and private_present and known_hosts_present, + "namespace": namespace, + "secretName": secret, + "sourceRefFrom": source_ref_from, + "privateKeySecretKey": private_key, + "privateKeySourceRef": private_source_ref, + "privateKeySourceKey": private_source_key, + "privateKeySecretExists": exists, + "privateKeyPresent": private_present, + "privateKeyBytes": len(private_encoded) if private_present else 0, + "privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded), + "knownHostsSecretKey": known_hosts_key, + "knownHostsSourceRef": known_hosts_source_ref, + "knownHostsSourceKey": known_hosts_source_key, + "knownHostsPresent": known_hosts_present, + "knownHostsBytes": len(known_hosts_encoded) if known_hosts_present else 0, + "knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded), + "valuesPrinted": False, +})) +PY +) +registry_ready=false +if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi +tools_repo_tag=\${tools_image#\${registry}/} +tools_repo=\${tools_repo_tag%:*} +tools_tag=\${tools_repo_tag##*:} +tools_image_ready=false +manifest_accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' +if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 -H "Accept: $manifest_accept" "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi +cache_host_path_ready=false +if [ -n "$cache_host_path" ] && kubectl -n "$gitmirror_ns" exec deploy/"$read_deploy" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_ready=true; fi +k3s_fragment=$(python3 - "$k3s_managed" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_expected_sha" <<'PY' +import hashlib, json, re, subprocess, sys +managed = sys.argv[1] == "true" +service, dropin, node_name, desired_raw, expected_sha = sys.argv[2:7] +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +def to_int(value): + try: + return int(value) + except Exception: + return None +if not managed: + print(json.dumps({"managed": False, "ready": True})) + raise SystemExit(0) +desired = to_int(desired_raw) +node_json = run(["kubectl", "get", "node", node_name, "-o", "json"]) +capacity = None +allocatable = None +node_ready = False +if node_json.returncode == 0: + data = json.loads(node_json.stdout) + capacity = to_int(data.get("status", {}).get("capacity", {}).get("pods")) + allocatable = to_int(data.get("status", {}).get("allocatable", {}).get("pods")) + for condition in data.get("status", {}).get("conditions", []): + if condition.get("type") == "Ready": + node_ready = condition.get("status") == "True" +unit = run(["systemctl", "cat", service]) +unit_text = unit.stdout if unit.returncode == 0 else "" +dropin_read = run(["cat", dropin]) +dropin_exists = dropin_read.returncode == 0 +dropin_text = dropin_read.stdout if dropin_exists else "" +dropin_sha = "sha256:" + hashlib.sha256(dropin_text.encode()).hexdigest() if dropin_exists else None +matches = re.findall(r"max-pods=([0-9]+)", unit_text + "\\n" + dropin_text) +configured = to_int(matches[-1]) if matches else None +dropin_matches = dropin_sha == expected_sha +ready = dropin_matches and capacity == desired and allocatable == desired +source = "managed-dropin" if dropin_matches else ("systemd-or-config" if configured is not None else "kubelet-default") +print(json.dumps({ + "managed": True, + "ready": ready, + "serviceName": service, + "dropInPath": dropin, + "dropInExists": dropin_exists, + "dropInSha256": dropin_sha, + "expectedDropInSha256": expected_sha, + "dropInMatches": dropin_matches, + "configuredMaxPods": configured, + "desiredMaxPods": desired, + "liveNodeName": node_name, + "liveCapacityPods": capacity, + "liveAllocatablePods": allocatable, + "nodeReady": node_ready, + "restartRequired": not ready, + "source": source, + "unitReadable": unit.returncode == 0, +})) +PY +) +python3 - "$tekton_required_crds_json" "$tekton_deployment_namespaces_json" "$tekton_runtime_proxy_json" <<'PY' >/tmp/hwlab-node-tekton-status-fragments.json +import json, subprocess, sys +required_crds=json.loads(sys.argv[1]) +namespaces=json.loads(sys.argv[2]) +runtime_proxy=json.loads(sys.argv[3]) +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +def exists(args): + return run(args).returncode == 0 +def env_map(container): + result={} + for item in container.get("env") or []: + name=item.get("name") + if isinstance(name, str) and "value" in item: + result[name]=str(item.get("value") or "") + return result +def runtime_proxy_workload(kind, namespace, name, cfg): + proc=run(["kubectl", "-n", namespace, "get", kind, name, "-o", "json"]) + if proc.returncode != 0: + return {"kind": kind, "namespace": namespace, "name": name, "exists": False, "ready": False} + obj=json.loads(proc.stdout or "{}") + template=obj.get("spec", {}).get("template", {}) + spec=template.get("spec", {}) + annotations=template.get("metadata", {}).get("annotations") or {} + containers=spec.get("containers") or [] + expected_env={ + "HTTP_PROXY": str(cfg.get("proxyUrl") or ""), + "HTTPS_PROXY": str(cfg.get("proxyUrl") or ""), + "ALL_PROXY": str(cfg.get("proxyUrl") or ""), + "NO_PROXY": ",".join([str(item) for item in cfg.get("noProxy") or []]), + "http_proxy": str(cfg.get("proxyUrl") or ""), + "https_proxy": str(cfg.get("proxyUrl") or ""), + "all_proxy": str(cfg.get("proxyUrl") or ""), + "no_proxy": ",".join([str(item) for item in cfg.get("noProxy") or []]), + } + env_matches=True + if cfg.get("injectEnv"): + env_matches=bool(containers) and all(all(env_map(container).get(key) == value for key, value in expected_env.items()) for container in containers) + host_network_matches=(not cfg.get("hostNetwork")) or spec.get("hostNetwork") is True + dns_policy_matches=(not cfg.get("hostNetwork")) or spec.get("dnsPolicy") == "ClusterFirstWithHostNet" + annotation_matches=annotations.get("unidesk.ai/runtime-proxy") == "host-route" and annotations.get("unidesk.ai/runtime-proxy-config-ref") == str(cfg.get("configRef") or "") + return { + "kind": kind, + "namespace": namespace, + "name": name, + "exists": True, + "ready": host_network_matches and dns_policy_matches and env_matches and annotation_matches, + "hostNetwork": spec.get("hostNetwork") is True, + "hostNetworkMatches": host_network_matches, + "dnsPolicy": spec.get("dnsPolicy") or None, + "dnsPolicyMatches": dns_policy_matches, + "injectEnv": bool(cfg.get("injectEnv")), + "envMatches": env_matches, + "annotationMatches": annotation_matches, + } +def runtime_proxy_status(cfg, namespace_values): + if not cfg.get("enabled"): + return {"enabled": False, "ready": True, "mode": cfg.get("mode") or "host-route", "workloads": [], "valuesPrinted": False} + workloads=[] + missing=[] + for kind, names in (("deployment", cfg.get("deployments") or []), ("statefulset", cfg.get("statefulSets") or [])): + for name in [str(item) for item in names]: + found=[] + for namespace in namespace_values: + proc=run(["kubectl", "-n", namespace, "get", kind, name, "-o", "name"]) + if proc.returncode == 0: + found.append(namespace) + if not found: + missing.append({"kind": kind, "name": name}) + workloads.append({"kind": kind, "name": name, "exists": False, "ready": False}) + else: + workloads.extend(runtime_proxy_workload(kind, namespace, name, cfg) for namespace in found) + return { + "enabled": True, + "mode": cfg.get("mode") or "host-route", + "configRef": cfg.get("configRef"), + "hostNetwork": bool(cfg.get("hostNetwork")), + "injectEnv": bool(cfg.get("injectEnv")), + "selectedDeployments": cfg.get("deployments") or [], + "selectedStatefulSets": cfg.get("statefulSets") or [], + "missing": missing, + "workloads": workloads, + "ready": len(missing) == 0 and all(item.get("ready") for item in workloads), + "valuesPrinted": False, + } +def namespace_deployments(namespace): + proc = run(["kubectl", "-n", namespace, "get", "deploy", "-o", "json"]) + if proc.returncode != 0: + return {"namespace": namespace, "namespaceExists": False, "deployments": [], "ready": False} + try: + data=json.loads(proc.stdout) + except Exception: + return {"namespace": namespace, "namespaceExists": True, "deployments": [], "ready": False} + deployments=[] + for item in data.get("items", []): + desired=int(item.get("spec", {}).get("replicas") or 0) + ready=int(item.get("status", {}).get("readyReplicas") or 0) + deployments.append({"name": item.get("metadata", {}).get("name"), "desired": desired, "readyReplicas": ready, "ready": desired > 0 and ready == desired}) + return {"namespace": namespace, "namespaceExists": True, "deployments": deployments, "ready": len(deployments) > 0 and all(item["ready"] for item in deployments)} +crds=[{"name": name, "exists": exists(["kubectl", "get", "crd", name])} for name in required_crds] +deployment_namespaces=[namespace_deployments(ns) for ns in namespaces] +print(json.dumps({"crds": crds, "deploymentNamespaces": deployment_namespaces, "crdsReady": all(item["exists"] for item in crds), "deploymentsReady": all(item["ready"] for item in deployment_namespaces) if deployment_namespaces else True, "runtimeProxy": runtime_proxy_status(runtime_proxy, namespaces)})) +PY +tekton_fragment=$(cat /tmp/hwlab-node-tekton-status-fragments.json 2>/dev/null || printf '{}') +tekton_installed=$(python3 - "$tekton_fragment" <<'PY' +import json, sys +data=json.loads(sys.argv[1] or '{}') +print('true' if data.get('crdsReady') and data.get('deploymentsReady') else 'false') +PY +) +python3 - "$required_crds_json" "$argo_deployments_json" "$argo_statefulsets_json" "$argo_runtime_proxy_json" <<'PY' >/tmp/hwlab-node-status-fragments.json +import json, subprocess, sys +required_crds=json.loads(sys.argv[1]) +deployments=json.loads(sys.argv[2]) +statefulsets=json.loads(sys.argv[3]) +runtime_proxy=json.loads(sys.argv[4]) +ns="${target.argo.namespace}" +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +def exists(args): + return run(args).returncode == 0 +def env_map(container): + result={} + for item in container.get("env") or []: + name=item.get("name") + if isinstance(name, str) and "value" in item: + result[name]=str(item.get("value") or "") + return result +def runtime_proxy_workload(kind, namespace, name, cfg): + proc=run(["kubectl", "-n", namespace, "get", kind, name, "-o", "json"]) + if proc.returncode != 0: + return {"kind": kind, "namespace": namespace, "name": name, "exists": False, "ready": False} + obj=json.loads(proc.stdout or "{}") + template=obj.get("spec", {}).get("template", {}) + spec=template.get("spec", {}) + annotations=template.get("metadata", {}).get("annotations") or {} + containers=spec.get("containers") or [] + expected_env={ + "HTTP_PROXY": str(cfg.get("proxyUrl") or ""), + "HTTPS_PROXY": str(cfg.get("proxyUrl") or ""), + "ALL_PROXY": str(cfg.get("proxyUrl") or ""), + "NO_PROXY": ",".join([str(item) for item in cfg.get("noProxy") or []]), + "http_proxy": str(cfg.get("proxyUrl") or ""), + "https_proxy": str(cfg.get("proxyUrl") or ""), + "all_proxy": str(cfg.get("proxyUrl") or ""), + "no_proxy": ",".join([str(item) for item in cfg.get("noProxy") or []]), + } + env_matches=True + if cfg.get("injectEnv"): + env_matches=bool(containers) and all(all(env_map(container).get(key) == value for key, value in expected_env.items()) for container in containers) + host_network_matches=(not cfg.get("hostNetwork")) or spec.get("hostNetwork") is True + dns_policy_matches=(not cfg.get("hostNetwork")) or spec.get("dnsPolicy") == "ClusterFirstWithHostNet" + annotation_matches=annotations.get("unidesk.ai/runtime-proxy") == "host-route" and annotations.get("unidesk.ai/runtime-proxy-config-ref") == str(cfg.get("configRef") or "") + return { + "kind": kind, + "namespace": namespace, + "name": name, + "exists": True, + "ready": host_network_matches and dns_policy_matches and env_matches and annotation_matches, + "hostNetwork": spec.get("hostNetwork") is True, + "hostNetworkMatches": host_network_matches, + "dnsPolicy": spec.get("dnsPolicy") or None, + "dnsPolicyMatches": dns_policy_matches, + "injectEnv": bool(cfg.get("injectEnv")), + "envMatches": env_matches, + "annotationMatches": annotation_matches, + } +def runtime_proxy_status(cfg): + if not cfg.get("enabled"): + return {"enabled": False, "ready": True, "mode": cfg.get("mode") or "host-route", "workloads": [], "valuesPrinted": False} + workloads=[] + missing=[] + for kind, names in (("deployment", cfg.get("deployments") or []), ("statefulset", cfg.get("statefulSets") or [])): + for name in [str(item) for item in names]: + proc=run(["kubectl", "-n", ns, "get", kind, name, "-o", "name"]) + if proc.returncode != 0: + missing.append({"kind": kind, "name": name}) + workloads.append({"kind": kind, "name": name, "exists": False, "ready": False}) + else: + workloads.append(runtime_proxy_workload(kind, ns, name, cfg)) + return { + "enabled": True, + "mode": cfg.get("mode") or "host-route", + "configRef": cfg.get("configRef"), + "hostNetwork": bool(cfg.get("hostNetwork")), + "injectEnv": bool(cfg.get("injectEnv")), + "selectedDeployments": cfg.get("deployments") or [], + "selectedStatefulSets": cfg.get("statefulSets") or [], + "missing": missing, + "workloads": workloads, + "ready": len(missing) == 0 and all(item.get("ready") for item in workloads), + "valuesPrinted": False, + } +def ready(kind, name): + data = run(["kubectl", "-n", ns, "get", kind, name, "-o", "json"]) + if data.returncode != 0: + return {"name": name, "exists": False, "ready": False, "desired": None, "readyReplicas": None} + obj=json.loads(data.stdout) + desired=int(obj.get("spec", {}).get("replicas") or 0) + ready_replicas=int(obj.get("status", {}).get("readyReplicas") or 0) + return {"name": name, "exists": True, "ready": desired > 0 and ready_replicas == desired, "desired": desired, "readyReplicas": ready_replicas} +crds=[{"name": name, "exists": exists(["kubectl", "get", "crd", name])} for name in required_crds] +deploy=[ready("deployment", name) for name in deployments] +sts=[ready("statefulset", name) for name in statefulsets] +print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crdsReady": all(item["exists"] for item in crds), "deploymentsReady": all(item["ready"] for item in deploy) if deploy else True, "statefulSetsReady": all(item["ready"] for item in sts) if sts else True, "runtimeProxy": runtime_proxy_status(runtime_proxy)})) +PY +argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') +cat </dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"argoObserverRbac":{"roleName":"$argo_observer_role","roleExists":$argo_observer_role_exists,"roleBindingName":"$argo_observer_rolebinding","roleBindingExists":$argo_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canGetApplication":$argo_observer_can_get_application,"ready":$argo_observer_ready},"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} +JSON +`; +} + +export function applyScript(yaml: string, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { + const encoded = Buffer.from(yaml, "utf8").toString("base64"); + return ` +set +e +manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml) +printf %s ${shQuote(encoded)} | base64 -d >"$manifest" +field_manager=${shQuote(controlPlaneFieldManager(target))} +kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err +kubectl_rc=$? +${k3sApplyScriptFragment(node.k3s, target)} +python3 - "$kubectl_rc" "$k3s_report_file" <<'PY' +import json, pathlib, sys +k3s_report = {} +try: + k3s_report = json.loads(pathlib.Path(sys.argv[2]).read_text(errors='replace')) +except Exception as exc: + k3s_report = {"managed": None, "ok": False, "parseError": str(exc)} +out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else '' +err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else '' +print(json.dumps({'k3sNodeConfig': k3s_report, 'kubernetesApply': {'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}}, ensure_ascii=False)) +PY +rm -f "$manifest" +if [ "$kubectl_rc" != 0 ]; then exit "$kubectl_rc"; fi +exit "$k3s_rc" +`; +} + +export function controlPlaneFieldManager(target: ControlPlaneTargetSpec): string { + return `unidesk-hwlab-${target.node.toLowerCase()}-${target.lane}-control-plane`; +} + +export function k3sApplyScriptFragment(spec: ControlPlaneK3sNodeSpec | null, target: ControlPlaneTargetSpec): string { + if (spec === null) { + return ` +k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json) +printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$k3s_report_file" +k3s_rc=0 +`; + } + const content = k3sDropInContent(spec); + const encoded = Buffer.from(content, "utf8").toString("base64"); + return ` +k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json) +k3s_service=${shQuote(spec.serviceName)} +k3s_dropin=${shQuote(spec.dropInPath)} +k3s_node=${shQuote(spec.nodeStatusName)} +k3s_namespace=${shQuote(target.ciNamespace)} +k3s_image=${shQuote(target.tekton.toolsImage.output)} +k3s_desired_max_pods=${shQuote(String(spec.kubelet.maxPods))} +k3s_expected_sha=${shQuote(sha256Short(content))} +k3s_before_capacity=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) +k3s_before_allocatable=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) +capacity_restart=false +if [ "$k3s_before_capacity" != "$k3s_desired_max_pods" ] || [ "$k3s_before_allocatable" != "$k3s_desired_max_pods" ]; then capacity_restart=true; fi +k3s_current_dropin_sha= +if [ -f "$k3s_dropin" ]; then k3s_current_dropin_sha=$(sha256sum "$k3s_dropin" | awk '{print "sha256:"$1}'); fi +if [ "$k3s_current_dropin_sha" = "$k3s_expected_sha" ] && [ "$capacity_restart" = false ]; then + python3 - "$k3s_current_dropin_sha" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_before_capacity" "$k3s_before_allocatable" <<'PY' >"$k3s_report_file" +import json, sys +dropin_sha, expected_sha, service, dropin, node_name, desired, before_capacity, before_allocatable = sys.argv[1:9] +print(json.dumps({ + "managed": True, + "ok": True, + "mutation": False, + "applyMode": "noop", + "completionPending": False, + "serviceName": service, + "dropInPath": dropin, + "dropInSha256": dropin_sha, + "expectedDropInSha256": expected_sha, + "dropInMatches": dropin_sha == expected_sha, + "nodeName": node_name, + "desiredMaxPods": int(desired), + "beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None, + "beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None, +}, ensure_ascii=False)) +PY + k3s_rc=0 +else +k3s_job="hwlab-node-k3s-config-$(date +%s)" +k3s_job_manifest=$(mktemp /tmp/hwlab-node-k3s-job.XXXXXX.json) +k3s_host_script=$(mktemp /tmp/hwlab-node-k3s-host.XXXXXX.sh) +k3s_job_apply_stdout=/tmp/hwlab-node-k3s-job-apply.out +k3s_job_apply_stderr=/tmp/hwlab-node-k3s-job-apply.err +k3s_docker_stdout=/tmp/hwlab-node-k3s-docker.out +k3s_docker_stderr=/tmp/hwlab-node-k3s-docker.err +k3s_host_report="/tmp/$k3s_job-report.json" +rm -f "$k3s_host_report" +python3 - "$k3s_job_manifest" "$k3s_host_script" "$k3s_job" "$k3s_namespace" "$k3s_image" "$k3s_dropin" ${shQuote(encoded)} "$k3s_service" "$k3s_desired_max_pods" "$k3s_expected_sha" "$capacity_restart" "$k3s_host_report" <<'PY' +import json, os, shlex, sys +manifest_path, host_script_path, job, namespace, image, dropin, encoded, service, desired, expected_sha, capacity_restart, report_path = sys.argv[1:13] +script = f"""#!/bin/sh +set -eu +expected=/tmp/unidesk-k3s-dropin.conf +printf %s {shlex.quote(encoded)} | base64 -d > "$expected" +host_dropin=/host{shlex.quote(dropin)} +host_report=/host{shlex.quote(report_path)} +mkdir -p "$(dirname "$host_dropin")" +before_sha= +if [ -f "$host_dropin" ]; then before_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi +changed=false +if ! cmp -s "$expected" "$host_dropin" 2>/dev/null; then + cp "$expected" "$host_dropin" + chown 0:0 "$host_dropin" 2>/dev/null || true + chmod 0644 "$host_dropin" + changed=true +fi +nsenter_path=$(command -v nsenter || true) +host_systemctl() {{ + if command -v chroot >/dev/null 2>&1 && [ -x /host/usr/bin/systemctl ]; then + chroot /host /usr/bin/systemctl "$@" + return $? + fi + if [ -n "$nsenter_path" ]; then + "$nsenter_path" -t 1 -m -u -i -n -p -- /usr/bin/systemctl "$@" + return $? + fi + return 127 +}} +daemon_reload_rc=0 +restart_rc=0 +restarted=false +if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then + host_systemctl daemon-reload || daemon_reload_rc=$? + if [ "$changed" = true ] || [ {shlex.quote(capacity_restart)} = true ]; then + restarted=true + host_systemctl restart {shlex.quote(service)} || restart_rc=$? + fi +else + daemon_reload_rc=127 + restart_rc=127 +fi +after_sha= +if [ -f "$host_dropin" ]; then after_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi +service_active=unknown +if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then service_active=$(host_systemctl is-active {shlex.quote(service)} 2>/dev/null || true); fi +python3 - "$changed" "$restarted" "$daemon_reload_rc" "$restart_rc" "$before_sha" "$after_sha" "$service_active" "$nsenter_path" <<'REPORT' >"$host_report" +import json, sys +changed, restarted = sys.argv[1] == "true", sys.argv[2] == "true" +daemon_reload_rc, restart_rc = int(sys.argv[3] or "0"), int(sys.argv[4] or "0") +print(json.dumps({{ + "jobChanged": changed, + "jobRestarted": restarted, + "daemonReloadExitCode": daemon_reload_rc, + "restartExitCode": restart_rc, + "beforeDropInSha256": sys.argv[5] or None, + "dropInSha256": sys.argv[6] or None, + "expectedDropInSha256": {json.dumps(expected_sha)}, + "dropInMatches": sys.argv[6] == {json.dumps(expected_sha)}, + "serviceActiveText": sys.argv[7] or None, + "nsenterPresent": bool(sys.argv[8]), +}})) +REPORT +chmod 0644 "$host_report" 2>/dev/null || true +cat "$host_report" +""" +with open(host_script_path, "w", encoding="utf-8") as handle: + handle.write(script) +os.chmod(host_script_path, 0o755) +manifest = { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": {"name": job, "namespace": namespace, "labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}}, + "spec": { + "backoffLimit": 0, + "ttlSecondsAfterFinished": 300, + "template": { + "metadata": {"labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}}, + "spec": { + "restartPolicy": "Never", + "hostPID": True, + "hostNetwork": True, + "containers": [{ + "name": "apply-k3s-node-config", + "image": image, + "imagePullPolicy": "IfNotPresent", + "securityContext": {"privileged": True}, + "command": ["/bin/sh", "-lc", script], + "volumeMounts": [{"name": "host-root", "mountPath": "/host"}], + }], + "volumes": [{"name": "host-root", "hostPath": {"path": "/", "type": "Directory"}}], + }, + }, + }, +} +with open(manifest_path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle) +PY +k3s_render_rc=$? +if [ "$k3s_render_rc" != 0 ]; then + python3 - "$k3s_render_rc" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" <<'PY' >"$k3s_report_file" +import json, sys +render_rc = int(sys.argv[1] or "1") +expected_sha, service, dropin, node_name, desired = sys.argv[2:7] +print(json.dumps({ + "managed": True, + "ok": False, + "mutation": False, + "renderExitCode": render_rc, + "serviceName": service, + "dropInPath": dropin, + "expectedDropInSha256": expected_sha, + "nodeName": node_name, + "desiredMaxPods": int(desired), +}, ensure_ascii=False)) +PY + k3s_rc=$k3s_render_rc +else +kubectl apply -f "$k3s_job_manifest" >"$k3s_job_apply_stdout" 2>"$k3s_job_apply_stderr" +k3s_job_apply_rc=$? +k3s_apply_mode=kubernetes-job +k3s_docker_rc=127 +if [ "$k3s_job_apply_rc" != 0 ] && command -v docker >/dev/null 2>&1; then + k3s_apply_mode=docker-host-fallback + docker run --rm --privileged --pid=host --network=host -v /:/host --entrypoint /bin/sh "$k3s_image" "/host$k3s_host_script" >"$k3s_docker_stdout" 2>"$k3s_docker_stderr" + k3s_docker_rc=$? +fi +k3s_submit_rc=$k3s_job_apply_rc +if [ "$k3s_job_apply_rc" != 0 ] && [ "$k3s_docker_rc" = 0 ]; then k3s_submit_rc=0; fi +python3 - "$k3s_submit_rc" "$k3s_job_apply_rc" "$k3s_docker_rc" "$k3s_apply_mode" "$k3s_before_capacity" "$k3s_before_allocatable" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_job" "$k3s_namespace" "$k3s_host_report" "$k3s_job_apply_stdout" "$k3s_job_apply_stderr" "$k3s_docker_stdout" "$k3s_docker_stderr" <<'PY' >"$k3s_report_file" +import json, pathlib, sys +submit_rc, job_apply_rc, docker_rc = [int(value or "0") for value in sys.argv[1:4]] +apply_mode = sys.argv[4] +before_capacity, before_allocatable = sys.argv[5:7] +expected_sha, service, dropin, node_name, desired, job_name, namespace, host_report = sys.argv[7:15] +def read(path): + return pathlib.Path(path).read_text(errors='replace') if pathlib.Path(path).exists() else '' +try: + host_report_data = json.loads(read(host_report) or "{}") +except Exception: + host_report_data = {} +apply_ok = submit_rc == 0 +print(json.dumps({ + "managed": True, + "ok": apply_ok, + "mutation": apply_ok, + "completionPending": apply_ok and apply_mode == "kubernetes-job", + "applyMode": apply_mode, + "jobName": job_name, + "namespace": namespace, + "jobApplyExitCode": job_apply_rc, + "dockerFallbackExitCode": docker_rc, + "serviceName": service, + "dropInPath": dropin, + "dropInSha256": host_report_data.get("dropInSha256"), + "expectedDropInSha256": expected_sha, + "dropInMatches": host_report_data.get("dropInSha256") == expected_sha if host_report_data else None, + "daemonReloadExitCode": host_report_data.get("daemonReloadExitCode"), + "restartExitCode": host_report_data.get("restartExitCode"), + "serviceActive": host_report_data.get("serviceActiveText") == "active" if host_report_data else None, + "nodeName": node_name, + "desiredMaxPods": int(desired), + "beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None, + "beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None, + "hostReportPath": host_report, + "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra status --node {node_name.upper()} --lane ${target.lane}", + "jobCompletionCommand": f"kubectl -n {namespace} wait --for=condition=complete job/{job_name} --timeout=120s", + "jobLogsCommand": f"kubectl -n {namespace} logs job/{job_name} --tail=120", + "jobApplyStdoutPreview": read(sys.argv[15])[-1000:], + "jobApplyStderrPreview": read(sys.argv[16])[-1000:], + "dockerStdoutPreview": read(sys.argv[17])[-1000:], + "dockerStderrPreview": read(sys.argv[18])[-1000:], +}, ensure_ascii=False)) +PY +k3s_rc=$k3s_submit_rc +fi +rm -f "$k3s_job_manifest" "$k3s_host_script" +fi +`; +} + +export function toolsImageStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, timeoutSeconds: number): { + registryReady: boolean; + toolsImageReady: boolean; + result: Record; +} { + const result = runTransK3s(node.kubeRoute, registryStatusScript(node.registry.endpoint, target.tekton.toolsImage.output), timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; + return { + registryReady: boolField(status, "registryReady"), + toolsImageReady: boolField(status, "toolsImageReady"), + result: { + status, + command: compactCommandResult(result), + }, + }; +} + +export function applyNext(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, imageStatus: { registryReady: boolean; toolsImageReady: boolean }): Record { + if (!imageStatus.registryReady) { + return { + status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, + blockedBy: "node-local-registry-not-ready", + }; + } + if (!imageStatus.toolsImageReady) { + return { + status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, + blockedBy: "tools-image-missing", + applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, + buildToolsImage: "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。", + }; + } + return { apply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm` }; +} + +export function statusNext( + node: ControlPlaneNodeSpec, + target: ControlPlaneTargetSpec, + registry: Record, + gitMirror: Record, + tekton: Record, + argo: Record, + ciNamespace: Record, + runtimeNamespace: Record, + k3sNodeConfig: Record, +): Record { + const bootstrapMissing = !boolField(ciNamespace, "exists") + || !boolField(record(ciNamespace.gitWorkspaceSecret), "ready") + || !boolField(gitMirror, "namespaceExists") + || !boolField(gitMirror, "readServiceExists") + || !boolField(gitMirror, "writeServiceExists") + || (!boolField(gitMirror, "cachePvcExists") && !boolField(gitMirror, "cacheHostPathReady")); + const blockers: string[] = []; + if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) blockers.push("k3s-node-config-not-applied"); + if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); + if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); + if (!boolField(record(ciNamespace.gitWorkspaceSecret), "ready")) blockers.push("ci-git-workspace-secret-not-ready"); + if (!boolField(runtimeNamespace, "exists")) blockers.push("runtime-namespace-missing"); + if (!boolField(record(runtimeNamespace.runtimeObserverRbac), "ready")) blockers.push("runtime-observer-rbac-not-ready"); + if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); + const gitMirrorGithubTransport = record(gitMirror.githubTransport); + if (gitMirrorGithubTransport.required === true && !boolField(gitMirrorGithubTransport, "ready")) blockers.push("git-mirror-github-token-secret-not-ready"); + const tektonInstall = record(tekton.install); + if (!boolField(tekton, "installed")) blockers.push("tekton-not-installed"); + else if (!boolField(tektonInstall, "crdsReady")) blockers.push("tekton-crds-not-ready"); + else if (!boolField(tektonInstall, "deploymentsReady")) blockers.push("tekton-deployments-not-ready"); + else if (!runtimeProxyReady(tektonInstall)) blockers.push("tekton-runtime-proxy-not-ready"); + const argoInstall = record(argo.install); + if (!boolField(argo, "installed")) blockers.push("argocd-not-installed"); + else if (!boolField(argoInstall, "crdsReady")) blockers.push("argocd-crds-not-ready"); + else if (!boolField(argoInstall, "deploymentsReady")) blockers.push("argocd-deployments-not-ready"); + else if (!boolField(argoInstall, "statefulSetsReady")) blockers.push("argocd-statefulsets-not-ready"); + else if (!runtimeProxyReady(argoInstall)) blockers.push("argocd-runtime-proxy-not-ready"); + else if (!boolField(argo, "projectExists")) blockers.push("argocd-project-missing"); + else if (!boolField(argo, "applicationExists")) blockers.push("argocd-application-missing"); + if (!boolField(record(argo.argoObserverRbac), "ready")) blockers.push("argocd-observer-rbac-not-ready"); + const next: Record = { + status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, + dryRun: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --dry-run`, + }; + if (blockers.length > 0) { + next.blockedBy = blockers[0]; + next.blockers = blockers; + } + if (!boolField(registry, "toolsImageReady")) { + next.buildToolsImage = "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。"; + } + if (!boolField(tekton, "installed")) { + next.installTekton = `bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node ${node.id} --lane ${target.lane} --confirm`; + } + if (!boolField(argo, "installed")) { + next.installArgo = "准备受控 D601 Argo CD 安装入口后再进入 runtime rollout。"; + } + if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) { + next.applyK3sNodeConfig = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; + } + if (bootstrapMissing) next.applyBootstrap = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; + else next.reapplyBootstrap = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; + return next; +} + +export function registryStatusScript(registryEndpoint: string, toolsImage: string): string { + return ` +set +e +registry=${shQuote(registryEndpoint)} +tools_image=${shQuote(toolsImage)} +registry_ready=false +if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi +tools_repo_tag=\${tools_image#\${registry}/} +tools_repo=\${tools_repo_tag%:*} +tools_tag=\${tools_repo_tag##*:} +tools_image_ready=false +manifest_accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' +if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 -H "Accept: $manifest_accept" "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi +cat < ["--build-arg", `${key}=${value}`]); + const proxyArgs = node.egressProxy === null + ? [] + : ["--build-arg", "HTTP_PROXY", "--build-arg", "HTTPS_PROXY", "--build-arg", "ALL_PROXY", "--build-arg", "NO_PROXY", "--build-arg", "http_proxy", "--build-arg", "https_proxy", "--build-arg", "all_proxy", "--build-arg", "no_proxy"]; + const networkArgs = target.tekton.toolsImage.buildNetwork === null ? [] : ["--network", target.tekton.toolsImage.buildNetwork]; + const dockerBuildArgs = [...networkArgs, ...buildArgs, ...proxyArgs, "-f", "$dockerfile", "-t", "$image", "$context_dir"].join(" "); + return ` +set -eu +state_dir=${shQuote(stateDir)} +mkdir -p "$state_dir" +if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then + printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" + exit 0 +fi +cat >"$state_dir/job.sh" <<'JOB' +#!/bin/sh +set -eu +state_dir=${shQuote(stateDir)} +image=${shQuote(target.tekton.toolsImage.output)} +context_dir="$state_dir/context" +dockerfile="$state_dir/${target.tekton.toolsImage.dockerfileInline?.filename ?? "Dockerfile"}" +log="$state_dir/job.log" +status="$state_dir/status.json" +write_status() { + state="$1"; shift + message="$1"; shift || true + python3 - "$status" "$state" "$message" "$image" <<'PY' +import json, pathlib, sys, time +path=pathlib.Path(sys.argv[1]) +payload={"state":sys.argv[2],"message":sys.argv[3],"image":sys.argv[4],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} +path.write_text(json.dumps(payload, ensure_ascii=False) + "\\n") +PY +} +run_job() { + write_status running starting + rm -rf "$context_dir" + mkdir -p "$context_dir" + printf %s ${shQuote(dockerfileEncoded)} | base64 -d >"$dockerfile" +${proxyExportBlock(node)} + docker build ${dockerBuildArgs} || return "$?" + docker run --rm "$image" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V' || return "$?" + docker push "$image" || return "$?" + image_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)" + digest="$(docker image inspect "$image" --format '{{join .RepoDigests ","}}' 2>/dev/null || true)" + python3 - "$status" "$image" "$image_id" "$digest" <<'PY' +import json, pathlib, sys, time +path=pathlib.Path(sys.argv[1]) +path.write_text(json.dumps({"state":"succeeded","message":"image-built-and-pushed","image":sys.argv[2],"imageId":sys.argv[3] or None,"repoDigests":[item for item in sys.argv[4].split(",") if item],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") +PY +} +run_job >>"$log" 2>&1 || { + rc=$? + write_status failed "exit-$rc" + exit "$rc" +} +JOB +chmod +x "$state_dir/job.sh" +: >"$state_dir/job.log" +nohup "$state_dir/job.sh" >/dev/null 2>&1 & +pid=$! +printf '%s' "$pid" >"$state_dir/pid" +printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} +`; +} + +export function tektonInstallApplyStartScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { + const stateDir = remoteJobStateDir(target, "tekton"); + const manifestsEncoded = Buffer.from(JSON.stringify(target.tekton.install.manifests), "utf8").toString("base64"); + const crdsEncoded = Buffer.from(JSON.stringify(target.tekton.install.requiredCrds), "utf8").toString("base64"); + const namespacesEncoded = Buffer.from(JSON.stringify(target.tekton.install.expectedDeploymentNamespaces), "utf8").toString("base64"); + const runtimeProxyEncoded = Buffer.from(JSON.stringify(runtimeHostProxyConfig(node, target.tekton.install.runtimeProxy)), "utf8").toString("base64"); + return ` +set -eu +state_dir=${shQuote(stateDir)} +mkdir -p "$state_dir" +if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then + printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" + exit 0 +fi +cat >"$state_dir/job.sh" <<'JOB' +#!/bin/sh +set -eu +state_dir=${shQuote(stateDir)} +field_manager=${shQuote(target.tekton.install.fieldManager)} +readiness_timeout=${shQuote(String(target.tekton.install.readinessTimeoutSeconds))} +log="$state_dir/job.log" +status="$state_dir/status.json" +manifests_json="$state_dir/manifests.json" +crds_json="$state_dir/required-crds.json" +namespaces_json="$state_dir/deployment-namespaces.json" +runtime_proxy_json="$state_dir/runtime-proxy.json" +write_status() { + state="$1"; shift + message="$1"; shift || true + python3 - "$status" "$state" "$message" <<'PY' +import json, pathlib, sys, time +path=pathlib.Path(sys.argv[1]) +path.write_text(json.dumps({"state":sys.argv[2],"message":sys.argv[3],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") +PY +} +{ + write_status running starting +${proxyExportBlock(node)} + printf %s ${shQuote(manifestsEncoded)} | base64 -d >"$manifests_json" + printf %s ${shQuote(crdsEncoded)} | base64 -d >"$crds_json" + printf %s ${shQuote(namespacesEncoded)} | base64 -d >"$namespaces_json" + printf %s ${shQuote(runtimeProxyEncoded)} | base64 -d >"$runtime_proxy_json" + python3 - "$manifests_json" "$crds_json" "$namespaces_json" "$runtime_proxy_json" "$field_manager" "$readiness_timeout" "$state_dir" <<'PY' +import json, pathlib, subprocess, sys, time +manifests=json.loads(pathlib.Path(sys.argv[1]).read_text()) +required_crds=json.loads(pathlib.Path(sys.argv[2]).read_text()) +namespaces=json.loads(pathlib.Path(sys.argv[3]).read_text()) +runtime_proxy=json.loads(pathlib.Path(sys.argv[4]).read_text()) +field_manager=sys.argv[5] +timeout=int(sys.argv[6]) +state_dir=pathlib.Path(sys.argv[7]) +def run(args): + print(json.dumps({"event":"tekton-install-command","argv":args[:2] + ["..."] if len(args) > 2 else args}, ensure_ascii=False), flush=True) + subprocess.run(args, check=True) +def run_checked(args, allow_no_resources=False): + print(json.dumps({"event":"tekton-runtime-proxy-command","argv":args[:4] + ["..."] if len(args) > 4 else args}, ensure_ascii=False), flush=True) + proc=subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + text=(proc.stdout or "") + (proc.stderr or "") + if proc.returncode != 0: + if allow_no_resources and "No resources found" in text: + return "" + sys.stderr.write(text[-2000:]) + raise SystemExit(proc.returncode) + return proc.stdout +def names_for(kind, namespace): + stdout=run_checked(["kubectl", "-n", namespace, "get", kind, "-o", "json"], allow_no_resources=True) + if not stdout: + return [] + data=json.loads(stdout or "{}") + return [item.get("metadata", {}).get("name") for item in data.get("items", []) if item.get("metadata", {}).get("name")] +def runtime_proxy_env_args(cfg): + proxy=str(cfg.get("proxyUrl") or "") + no_proxy=",".join([str(item) for item in cfg.get("noProxy") or []]) + return [ + "HTTP_PROXY=" + proxy, + "HTTPS_PROXY=" + proxy, + "ALL_PROXY=" + proxy, + "NO_PROXY=" + no_proxy, + "http_proxy=" + proxy, + "https_proxy=" + proxy, + "all_proxy=" + proxy, + "no_proxy=" + no_proxy, + ] +def apply_runtime_proxy(cfg): + if not cfg.get("enabled"): + return + selected={ + "deployment": set(str(item) for item in cfg.get("deployments") or []), + "statefulset": set(str(item) for item in cfg.get("statefulSets") or []), + } + observed={"deployment": set(), "statefulset": set()} + remove_env=["HTTP_PROXY-","HTTPS_PROXY-","ALL_PROXY-","NO_PROXY-","http_proxy-","https_proxy-","all_proxy-","no_proxy-"] + for namespace in namespaces: + for kind in ["deployment", "statefulset"]: + for name in names_for(kind, namespace): + observed[kind].add(name) + resource=f"{kind}/{name}" + enabled_for_workload=name in selected[kind] + if enabled_for_workload and cfg.get("injectEnv"): + run_checked(["kubectl", "-n", namespace, "set", "env", resource, *runtime_proxy_env_args(cfg)]) + else: + run_checked(["kubectl", "-n", namespace, "set", "env", resource, *remove_env]) + patch={"spec":{"template":{"metadata":{"annotations":{ + "unidesk.ai/runtime-proxy": "host-route" if enabled_for_workload else None, + "unidesk.ai/runtime-proxy-config-ref": str(cfg.get("configRef") or "") if enabled_for_workload else None, + }},"spec":{ + "hostNetwork": bool(enabled_for_workload and cfg.get("hostNetwork")), + "dnsPolicy": "ClusterFirstWithHostNet" if enabled_for_workload and cfg.get("hostNetwork") else "ClusterFirst", + }}}} + run_checked(["kubectl", "-n", namespace, "patch", kind, name, "--type", "merge", "-p", json.dumps(patch)]) + missing=[] + for kind, selected_names in selected.items(): + missing.extend([f"{kind}/{name}" for name in sorted(selected_names.difference(observed[kind]))]) + if missing: + raise SystemExit("runtime proxy selected missing workload: " + ",".join(missing)) +for manifest in manifests: + name=manifest["name"] + url=manifest["url"] + dest=state_dir / f"{name}.yaml" + run(["curl", "-fsSL", "--connect-timeout", "20", "--max-time", "180", url, "-o", str(dest)]) + run(["kubectl", "apply", "--field-manager", field_manager, "-f", str(dest)]) +apply_runtime_proxy(runtime_proxy) +deadline=time.time()+timeout +while True: + missing=[name for name in required_crds if subprocess.run(["kubectl", "get", "crd", name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0] + if not missing: + break + if time.time() >= deadline: + raise SystemExit(f"tekton CRDs not ready: {','.join(missing)}") + time.sleep(5) +for namespace in namespaces: + while True: + proc=subprocess.run(["kubectl", "-n", namespace, "get", "deploy", "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if proc.returncode == 0: + data=json.loads(proc.stdout) + deployments=data.get("items", []) + ready=True + for item in deployments: + desired=int(item.get("spec", {}).get("replicas") or 0) + ready_replicas=int(item.get("status", {}).get("readyReplicas") or 0) + if desired <= 0 or ready_replicas != desired: + ready=False + break + if deployments and ready: + break + if time.time() >= deadline: + stderr=proc.stderr[-500:] if proc.returncode != 0 else "" + raise SystemExit(f"tekton deployments not ready in namespace {namespace}; stderr={stderr}") + time.sleep(5) +PY + write_status succeeded tekton-install-applied +} >>"$log" 2>&1 || { + rc=$? + write_status failed "exit-$rc" + exit "$rc" +} +JOB +chmod +x "$state_dir/job.sh" +: >"$state_dir/job.log" +nohup "$state_dir/job.sh" >/dev/null 2>&1 & +pid=$! +printf '%s' "$pid" >"$state_dir/pid" +printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tekton status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} +`; +} + +export function argoApplyStartScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, desiredYaml: string): string { + const stateDir = remoteJobStateDir(target, "argo"); + const desiredEncoded = Buffer.from(desiredYaml, "utf8").toString("base64"); + const rewritesEncoded = Buffer.from(JSON.stringify(target.argo.install.imageRewrites), "utf8").toString("base64"); + const preloadEncoded = Buffer.from(JSON.stringify(target.argo.install.preloadImages), "utf8").toString("base64"); + const runtimeProxyEncoded = Buffer.from(JSON.stringify(runtimeHostProxyConfig(node, target.argo.install.runtimeProxy)), "utf8").toString("base64"); + return ` +set -eu +state_dir=${shQuote(stateDir)} +mkdir -p "$state_dir" +if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then + printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" + exit 0 +fi +cat >"$state_dir/job.sh" <<'JOB' +#!/bin/sh +set -eu +state_dir=${shQuote(stateDir)} +namespace=${shQuote(target.argo.namespace)} +manifest_url=${shQuote(target.argo.install.manifestUrl)} +field_manager=${shQuote(target.argo.install.fieldManager)} +readiness_timeout=${shQuote(String(target.argo.install.readinessTimeoutSeconds))} +log="$state_dir/job.log" +status="$state_dir/status.json" +install_yaml="$state_dir/install.yaml" +rendered_yaml="$state_dir/install.rendered.yaml" +desired_yaml="$state_dir/desired.yaml" +write_status() { + state="$1"; shift + message="$1"; shift || true + python3 - "$status" "$state" "$message" <<'PY' +import json, pathlib, sys, time +path=pathlib.Path(sys.argv[1]) +path.write_text(json.dumps({"state":sys.argv[2],"message":sys.argv[3],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") +PY +} +{ + write_status running starting +${proxyExportBlock(node)} + printf %s ${shQuote(desiredEncoded)} | base64 -d >"$desired_yaml" + printf %s ${shQuote(rewritesEncoded)} | base64 -d >"$state_dir/image-rewrites.json" + printf %s ${shQuote(preloadEncoded)} | base64 -d >"$state_dir/preload-images.json" + printf %s ${shQuote(runtimeProxyEncoded)} | base64 -d >"$state_dir/runtime-proxy.json" + kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply --server-side --field-manager="$field_manager" -f - || exit "$?" + python3 - "$state_dir/preload-images.json" "$state_dir/image-rewrites.json" <<'PY' >"$state_dir/pull-images.sh" +import json, pathlib, shlex, sys +preload=json.loads(pathlib.Path(sys.argv[1]).read_text()) +rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) +print("#!/bin/sh") +print("set -eu") +seen=set() +for item in rewrites: + pull=item["pullImage"] + target=item["target"] + if target in seen: + continue + seen.add(target) + print("docker pull " + shlex.quote(pull)) + print("docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) + print("docker push " + shlex.quote(target)) +for image in preload: + if image not in seen and image.startswith("127.0.0.1:5000/"): + print("docker image inspect " + shlex.quote(image) + " >/dev/null") +PY + chmod +x "$state_dir/pull-images.sh" + "$state_dir/pull-images.sh" || exit "$?" + curl -fsSL --max-time 60 "$manifest_url" >"$install_yaml" || exit "$?" + python3 - "$install_yaml" "$state_dir/image-rewrites.json" "$rendered_yaml" ${shQuote(target.argo.install.imagePullPolicy)} <<'PY' +import json, pathlib, sys +text=pathlib.Path(sys.argv[1]).read_text() +rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) +for item in rewrites: + text=text.replace(item["source"], item["target"]) +policy=sys.argv[4] +text=text.replace("imagePullPolicy: Always", "imagePullPolicy: " + policy) +pathlib.Path(sys.argv[3]).write_text(text) +PY + kubectl apply --server-side --field-manager="$field_manager" -n "$namespace" -f "$rendered_yaml" || exit "$?" + python3 - "$state_dir/runtime-proxy.json" "$namespace" <<'PY' +import json, pathlib, subprocess, sys +cfg=json.loads(pathlib.Path(sys.argv[1]).read_text()) +namespace=sys.argv[2] +if not cfg.get("enabled"): + raise SystemExit(0) +def run(args, allow_no_resources=False): + print(json.dumps({"event":"argocd-runtime-proxy-command","argv":args[:4] + ["..."] if len(args) > 4 else args}, ensure_ascii=False), flush=True) + proc=subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + text=(proc.stdout or "") + (proc.stderr or "") + if proc.returncode != 0: + if allow_no_resources and "No resources found" in text: + return + sys.stderr.write(text[-2000:]) + raise SystemExit(proc.returncode) +def names_for(kind): + proc=subprocess.run(["kubectl", "-n", namespace, "get", kind, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + text=(proc.stdout or "") + (proc.stderr or "") + if proc.returncode != 0: + if "No resources found" in text: + return [] + sys.stderr.write(text[-2000:]) + raise SystemExit(proc.returncode) + data=json.loads(proc.stdout or "{}") + return [item.get("metadata", {}).get("name") for item in data.get("items", []) if item.get("metadata", {}).get("name")] +def env_args(): + proxy=str(cfg.get("proxyUrl") or "") + no_proxy=",".join([str(item) for item in cfg.get("noProxy") or []]) + return [ + "HTTP_PROXY=" + proxy, + "HTTPS_PROXY=" + proxy, + "ALL_PROXY=" + proxy, + "NO_PROXY=" + no_proxy, + "http_proxy=" + proxy, + "https_proxy=" + proxy, + "all_proxy=" + proxy, + "no_proxy=" + no_proxy, + ] +env_remove_args=["HTTP_PROXY-","HTTPS_PROXY-","ALL_PROXY-","NO_PROXY-","http_proxy-","https_proxy-","all_proxy-","no_proxy-"] +workloads={ + "deployment": set(str(item) for item in cfg.get("deployments") or []), + "statefulset": set(str(item) for item in cfg.get("statefulSets") or []), +} +for kind in ["deployment", "statefulset"]: + selected=workloads[kind] + all_names=names_for(kind) + missing=sorted(selected.difference(all_names)) + if missing: + raise SystemExit(f"runtime proxy selected missing {kind}: {','.join(missing)}") + for name in all_names: + resource=f"{kind}/{name}" + enabled_for_workload=name in selected + if enabled_for_workload and cfg.get("injectEnv"): + run(["kubectl", "-n", namespace, "set", "env", resource, *env_args()]) + else: + run(["kubectl", "-n", namespace, "set", "env", resource, *env_remove_args]) + patch={"spec":{"template":{"metadata":{"annotations":{ + "unidesk.ai/runtime-proxy": "host-route" if enabled_for_workload else None, + "unidesk.ai/runtime-proxy-config-ref": str(cfg.get("configRef") or "") if enabled_for_workload else None, + }},"spec":{ + "hostNetwork": bool(enabled_for_workload and cfg.get("hostNetwork")), + "dnsPolicy": "ClusterFirstWithHostNet" if enabled_for_workload and cfg.get("hostNetwork") else "ClusterFirst", + }}}} + run(["kubectl", "-n", namespace, "patch", kind, name, "--type", "merge", "-p", json.dumps(patch)]) +PY + deadline=$(( $(date +%s) + readiness_timeout )) + while [ "$(date +%s)" -lt "$deadline" ]; do + kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && break + sleep 5 + done + kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null || exit "$?" + kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" + write_status succeeded argocd-install-applied +} >>"$log" 2>&1 || { + rc=$? + write_status failed "exit-$rc" + exit "$rc" +} +JOB +chmod +x "$state_dir/job.sh" +: >"$state_dir/job.log" +nohup "$state_dir/job.sh" >/dev/null 2>&1 & +pid=$! +printf '%s' "$pid" >"$state_dir/pid" +printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra argo status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} +`; +} + +export function ciBuildBenchmarkStartScript( + target: ControlPlaneTargetSpec, + profile: CiBuildBenchmarkProfileSpec, + manifest: Record, + pipelineName: string, + pipelineRun: string, + sourceCommit: string, + catalogPath: string, +): string { + const stateDir = ciBuildBenchmarkStateDir(target, profile.profile); + const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + return ` +set -eu +state_dir=${shQuote(stateDir)} +status_file="$state_dir/status.json" +ns=${shQuote(target.ciNamespace)} +profile=${shQuote(profile.profile)} +pipeline=${shQuote(pipelineName)} +pipeline_run=${shQuote(pipelineRun)} +source_commit=${shQuote(sourceCommit)} +catalog_path=${shQuote(catalogPath)} +mkdir -p "$state_dir" +previous_run= +if [ -s "$status_file" ]; then + previous_run=$(python3 - "$status_file" <<'PY' || true +import json, sys +try: + data=json.load(open(sys.argv[1], encoding="utf-8")) + print(data.get("pipelineRun") or "") +except Exception: + print("") +PY +) +fi +if [ -n "$previous_run" ]; then + previous_status=$(kubectl -n "$ns" get pipelinerun "$previous_run" -o 'jsonpath={.status.conditions[?(@.type=="Succeeded")].status}' 2>/dev/null || true) + if [ -n "$previous_status" ] && [ "$previous_status" != "True" ] && [ "$previous_status" != "False" ]; then + python3 - "$state_dir" "$previous_run" "$profile" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY' +import json, sys +state_dir, previous_run, profile, node, lane = sys.argv[1:6] +print(json.dumps({ + "started": False, + "state": "already-running", + "pipelineRun": previous_run, + "stateDir": state_dir, + "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}", + "logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}", +}, ensure_ascii=False)) +PY + exit 0 + fi +fi +manifest_path="$state_dir/$pipeline_run.json" +printf '%s' ${shQuote(manifestB64)} | base64 -d >"$manifest_path" +set +e +pipeline_check=$(kubectl -n "$ns" get pipeline "$pipeline" -o name 2>&1) +pipeline_check_rc=$? +create_output= +create_rc=0 +if [ "$pipeline_check_rc" = 0 ]; then + create_output=$(kubectl create -f "$manifest_path" 2>&1) + create_rc=$? +else + create_output="$pipeline_check" + create_rc="$pipeline_check_rc" +fi +printf '%s\\n' "$create_output" >"$state_dir/create.log" +python3 - "$status_file" "$state_dir" "$pipeline_run" "$source_commit" "$profile" "$catalog_path" "$create_rc" "$create_output" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY' +import datetime, json, sys +status_file, state_dir, pipeline_run, source_commit, profile, catalog_path, rc_raw, output, node, lane = sys.argv[1:11] +rc=int(rc_raw or "0") +payload={ + "started": rc == 0, + "state": "started" if rc == 0 else "failed", + "pipelineRun": pipeline_run, + "sourceCommit": source_commit, + "profile": profile, + "catalogPath": catalog_path, + "stateDir": state_dir, + "createdAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "exitCode": rc, + "createOutputTail": output[-2000:], + "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}", + "logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}", +} +open(status_file, "w", encoding="utf-8").write(json.dumps(payload, ensure_ascii=False)) +print(json.dumps(payload, ensure_ascii=False)) +PY +exit "$create_rc" +`; +} + +export function ciBuildBenchmarkStatusScript(target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec, tailLines: number, includeLogs: boolean): string { + const stateDir = ciBuildBenchmarkStateDir(target, profile.profile); + return ` +set +e +state_dir=${shQuote(stateDir)} +status_file="$state_dir/status.json" +ns=${shQuote(target.ciNamespace)} +profile=${shQuote(profile.profile)} +tail_lines=${shQuote(String(tailLines))} +include_logs=${includeLogs ? "true" : "false"} +tmp_dir=$(mktemp -d) +pipeline_run= +if [ -s "$status_file" ]; then + pipeline_run=$(python3 - "$status_file" <<'PY' || true +import json, sys +try: + data=json.load(open(sys.argv[1], encoding="utf-8")) + print(data.get("pipelineRun") or "") +except Exception: + print("") +PY +) +fi +if [ -z "$pipeline_run" ]; then + pipeline_run=$(kubectl -n "$ns" get pipelinerun -l "unidesk.ai/benchmark=ci-build,unidesk.ai/benchmark-profile=$profile" -o 'jsonpath={range .items[*]}{.metadata.creationTimestamp}{" "}{.metadata.name}{"\\n"}{end}' 2>/dev/null | sort | tail -n 1 | awk '{print $2}') +fi +if [ -n "$pipeline_run" ]; then + kubectl -n "$ns" get pipelinerun "$pipeline_run" -o json >"$tmp_dir/pipelinerun.json" 2>"$tmp_dir/pipelinerun.err" + kubectl -n "$ns" get taskrun -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/taskruns.json" 2>"$tmp_dir/taskruns.err" + kubectl -n "$ns" get pod -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/pods.json" 2>"$tmp_dir/pods.err" + if [ "$include_logs" = true ]; then + kubectl -n "$ns" logs -l "tekton.dev/pipelineRun=$pipeline_run" --all-containers --tail="$tail_lines" --prefix=true >"$tmp_dir/logs.txt" 2>"$tmp_dir/logs.err" || true + fi +fi +python3 - "$state_dir" "$status_file" "$tmp_dir" "$pipeline_run" "$include_logs" "$tail_lines" <<'PY' +import json, pathlib, sys +state_dir=pathlib.Path(sys.argv[1]) +status_path=pathlib.Path(sys.argv[2]) +tmp_dir=pathlib.Path(sys.argv[3]) +pipeline_run=sys.argv[4] +include_logs=sys.argv[5] == "true" +tail_lines=int(sys.argv[6]) +log_tail_limit=min(6000, max(2000, tail_lines * 80)) + +def read_json(path): + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None + +def read_text(path, limit=4000): + try: + return path.read_text(encoding="utf-8", errors="replace")[-limit:] + except Exception: + return "" + +def succeeded_condition(obj): + for cond in obj.get("status", {}).get("conditions", []) or []: + if cond.get("type") == "Succeeded": + return cond + return {} + +status=None +if status_path.exists(): + status=read_json(status_path) +pr=read_json(tmp_dir / "pipelinerun.json") if pipeline_run else None +trs=read_json(tmp_dir / "taskruns.json") if pipeline_run else None +pods=read_json(tmp_dir / "pods.json") if pipeline_run else None +pr_cond=succeeded_condition(pr or {}) +task_runs=[] +for item in (trs or {}).get("items", []) or []: + cond=succeeded_condition(item) + labels=item.get("metadata", {}).get("labels", {}) or {} + task_runs.append({ + "name": item.get("metadata", {}).get("name"), + "pipelineTask": labels.get("tekton.dev/pipelineTask"), + "status": cond.get("status"), + "reason": cond.get("reason"), + "message": cond.get("message"), + "startTime": item.get("status", {}).get("startTime"), + "completionTime": item.get("status", {}).get("completionTime"), + "podName": item.get("status", {}).get("podName"), + }) +pod_rows=[] +for item in (pods or {}).get("items", []) or []: + phase=item.get("status", {}).get("phase") + pod_rows.append({ + "name": item.get("metadata", {}).get("name"), + "phase": phase, + "startTime": item.get("status", {}).get("startTime"), + }) +state="not-started" +if pr: + status_value=pr_cond.get("status") + if status_value == "True": + state="succeeded" + elif status_value == "False": + state="failed" + elif status_value: + state="running" + else: + state="pending" +elif pipeline_run: + state="missing" +payload={ + "stateDir": str(state_dir), + "status": status, + "pipelineRunName": pipeline_run or None, + "state": state, + "pipelineRun": None if not pr else { + "name": pr.get("metadata", {}).get("name"), + "status": pr_cond.get("status"), + "reason": pr_cond.get("reason"), + "message": pr_cond.get("message"), + "createdAt": pr.get("metadata", {}).get("creationTimestamp"), + "startTime": pr.get("status", {}).get("startTime"), + "completionTime": pr.get("status", {}).get("completionTime"), + "sourceCommit": (pr.get("metadata", {}).get("labels", {}) or {}).get("hwlab.pikastech.local/source-commit"), + "catalogPath": (pr.get("metadata", {}).get("annotations", {}) or {}).get("unidesk.ai/catalog-path"), + }, + "taskRuns": task_runs, + "pods": pod_rows, + "errors": { + "pipelinerun": read_text(tmp_dir / "pipelinerun.err"), + "taskruns": read_text(tmp_dir / "taskruns.err"), + "pods": read_text(tmp_dir / "pods.err"), + "logs": read_text(tmp_dir / "logs.err") if include_logs else "", + }, + "logTail": read_text(tmp_dir / "logs.txt", log_tail_limit) if include_logs else "", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +rm -rf "$tmp_dir" +`; +} + +export function remoteJobStatusScript(target: ControlPlaneTargetSpec, name: "tools-image" | "tekton" | "argo", tailLines: number): string { + const stateDir = remoteJobStateDir(target, name); + return ` +set +e +state_dir=${shQuote(stateDir)} +status_file="$state_dir/status.json" +log_file="$state_dir/job.log" +pid_file="$state_dir/pid" +running=false +pid=null +if [ -s "$pid_file" ]; then + pid_raw="$(cat "$pid_file" 2>/dev/null || true)" + if [ -n "$pid_raw" ] && kill -0 "$pid_raw" >/dev/null 2>&1; then running=true; pid="$pid_raw"; else pid="$pid_raw"; fi +fi +python3 - "$state_dir" "$status_file" "$log_file" "$running" "$pid" ${shQuote(String(tailLines))} <<'PY' +import json, pathlib, sys +state_dir=pathlib.Path(sys.argv[1]) +status_path=pathlib.Path(sys.argv[2]) +log_path=pathlib.Path(sys.argv[3]) +running=sys.argv[4] == "true" +pid=None if sys.argv[5] in ("", "null") else sys.argv[5] +tail_lines=int(sys.argv[6]) +status=None +if status_path.exists(): + try: + status=json.loads(status_path.read_text()) + except Exception as error: + status={"parseError": str(error), "raw": status_path.read_text(errors="replace")[-1000:]} +log_tail="" +if log_path.exists(): + lines=log_path.read_text(errors="replace").splitlines() + log_tail="\\n".join(lines[-tail_lines:]) +print(json.dumps({"stateDir": str(state_dir), "pid": pid, "running": running, "status": status, "logBytes": log_path.stat().st_size if log_path.exists() else 0, "logTail": log_tail}, ensure_ascii=False)) +PY +`; +} + +export function remoteJobLogs(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, name: "tools-image" | "tekton" | "argo", options: ToolsImageOptions | TektonInstallOptions | ArgoOptions): Record { + const result = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, name, options.tailLines), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + return { + ok: result.exitCode === 0, + command: `hwlab nodes control-plane infra ${name} logs`, + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mutation: false, + job: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, + result: compactCommandResult(result), + }; +} + +export function manifestObjectSummary(manifest: readonly Record[]): Record[] { + return manifest.map((item) => { + const metadata = record(item.metadata); + return { kind: item.kind ?? null, namespace: metadata.namespace ?? null, name: metadata.name ?? null }; + }); +} + +export function runTransK3s(kubeRoute: string, script: string, timeoutSeconds: number): CommandResult { + return runCommand(["/root/.local/bin/trans", kubeRoute, "sh", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); +} + +export function runTransHost(route: string, script: string, timeoutSeconds: number): CommandResult { + return runCommand(["bun", "scripts/ssh-cli.ts", "ssh", route, "sh", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); +} + +export function proxyExportBlock(node: ControlPlaneNodeSpec): string { + const proxy = node.egressProxy; + if (proxy === null) return " : # no egress proxy configured\n"; + if (proxy.mode === "host-route") { + const noProxy = [...new Set(["localhost", "127.0.0.1", "::1", "127.0.0.1:5000", "localhost:5000", ...proxy.noProxy])]; + return ` + if [ ! -s ${shQuote(proxy.proxyEnvPath)} ]; then echo "host route egress proxy env missing: ${proxy.proxyEnvPath}" >&2; exit 41; fi + . ${shQuote(proxy.proxyEnvPath)} + export HTTP_PROXY="${proxy.proxyUrl}" + export HTTPS_PROXY="$HTTP_PROXY" + export ALL_PROXY="$HTTP_PROXY" + export http_proxy="$HTTP_PROXY" + export https_proxy="$HTTP_PROXY" + export all_proxy="$HTTP_PROXY" + export NO_PROXY=${shQuote(noProxy.join(","))} + export no_proxy="$NO_PROXY" +`; + } + const noProxy = [...new Set(["localhost", "127.0.0.1", "::1", "127.0.0.1:5000", "localhost:5000", ...proxy.noProxy])]; + return ` + proxy_ip="$(kubectl -n ${shQuote(proxy.namespace)} get svc ${shQuote(proxy.serviceName)} -o 'jsonpath={.spec.clusterIP}' 2>/dev/null || true)" + if [ -z "$proxy_ip" ]; then echo "egress proxy service missing: ${proxy.namespace}/${proxy.serviceName}" >&2; exit 41; fi + export HTTP_PROXY="http://$proxy_ip:${proxy.port}" + export HTTPS_PROXY="$HTTP_PROXY" + export ALL_PROXY="$HTTP_PROXY" + export http_proxy="$HTTP_PROXY" + export https_proxy="$HTTP_PROXY" + export all_proxy="$HTTP_PROXY" + export NO_PROXY=${shQuote(noProxy.join(","))} + export no_proxy="$NO_PROXY" +`; +} + +export function remoteJobStateDir(target: ControlPlaneTargetSpec, name: "tools-image" | "tekton" | "argo"): string { + return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/${name}`; +} + +export function shellJsonArray(items: readonly string[]): string { + return JSON.stringify([...items]); +} + +export function parseRemoteJson(text: string): unknown { + const trimmed = text.trim(); + if (trimmed.length === 0) return null; + try { return JSON.parse(trimmed); } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { return JSON.parse(trimmed.slice(start, end + 1)); } catch {} + } + } + return null; +} + +export function ciBuildBenchmarkStateDir(target: ControlPlaneTargetSpec, profile: string): string { + return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/ci-build-benchmark-${profile}`; +} + +export function ciBuildBenchmarkLiveOk(job: Record, expectedServices: readonly string[], profile: CiBuildBenchmarkProfileSpec): boolean { + const pipelineRun = record(job.pipelineRun); + const pipelineStatus = renderCell(pipelineRun.status, ""); + if (pipelineStatus === "False") return false; + if (pipelineStatus !== "True") return true; + const taskRuns = ciBuildBenchmarkTaskRunRecords(job); + for (const service of expectedServices) { + if (!taskRuns.some((task) => task.pipelineTask === `build-${service}`)) return false; + } + return ciBuildBenchmarkPolicyOk(job, profile.cachePolicy); +} + +export function ciBuildBenchmarkTaskRows(job: Record): Record[] { + const pipelineRun = record(job.pipelineRun); + const rows: Record[] = []; + if (Object.keys(pipelineRun).length > 0) { + rows.push({ + task: "pipeline-total", + status: ciBuildBenchmarkStatusText(pipelineRun.status), + duration: durationBetweenIso(pipelineRun.startTime, pipelineRun.completionTime), + start: shortIsoTime(pipelineRun.startTime), + end: shortIsoTime(pipelineRun.completionTime), + }); + } + const taskRuns = ciBuildBenchmarkTaskRunRecords(job).sort((left, right) => renderCell(left.startTime, "").localeCompare(renderCell(right.startTime, ""))); + for (const task of taskRuns) { + rows.push({ + task: renderCell(task.pipelineTask ?? task.name), + status: ciBuildBenchmarkStatusText(task.status), + duration: durationBetweenIso(task.startTime, task.completionTime), + start: shortIsoTime(task.startTime), + end: shortIsoTime(task.completionTime), + }); + } + return rows; +} + +export function ciBuildBenchmarkServiceRows(job: Record, servicesValue: unknown): Record[] { + const services = ciBuildBenchmarkExpectedServices(servicesValue); + if (services.length === 0) return []; + const pipelineRun = record(job.pipelineRun); + const pipelineTerminal = pipelineRun.status === "True" || pipelineRun.status === "False"; + const taskRuns = ciBuildBenchmarkTaskRunRecords(job); + return services.map((service) => { + const task = taskRuns.find((item) => item.pipelineTask === `build-${service}`); + if (task === undefined) { + const status = pipelineTerminal ? "missing" : "pending"; + return { + service, + task: `build-${service}`, + status, + duration: "-", + failure: pipelineRun.status === "True" ? "cache-hit-forbidden" : "-", + }; + } + const taskStatus = ciBuildBenchmarkStatusText(task.status); + const failure = task.status === "False" ? classifyCiBuildBenchmarkFailure(`${renderCell(task.reason, "")}\n${renderCell(task.message, "")}`) : "-"; + return { + service, + task: renderCell(task.pipelineTask ?? task.name), + status: taskStatus, + duration: durationBetweenIso(task.startTime, task.completionTime), + failure, + }; + }); +} + +export function ciBuildBenchmarkFailureRows(job: Record, serviceRows: readonly Record[], benchmark: Record): Record[] { + const counts = new Map(); + const add = (family: string, scope: string): void => { + if (family === "-" || family.length === 0) return; + const existing = counts.get(family) ?? { count: 0, scopes: [] }; + existing.count += 1; + if (!existing.scopes.includes(scope)) existing.scopes.push(scope); + counts.set(family, existing); + }; + for (const row of serviceRows) add(row.failure, row.service); + const pipelineRun = record(job.pipelineRun); + if (pipelineRun.status === "False") { + add(classifyCiBuildBenchmarkFailure(`${renderCell(pipelineRun.reason, "")}\n${renderCell(pipelineRun.message, "")}`), "pipeline"); + } + const cachePolicy = record(benchmark.cachePolicy); + if (cachePolicy.forbidBuildkitCache === true && ciBuildBenchmarkLogHasBuildkitCache(job)) add("cache-hit-forbidden", "buildkit-cache"); + if (cachePolicy.forbidGitopsCatalogReuse === true && ciBuildBenchmarkLogHasReuse(job)) add("cache-hit-forbidden", "artifact-reuse"); + return [...counts.entries()].map(([family, value]) => ({ family, count: String(value.count), scope: value.scopes.join(",") })); +} + +export function ciBuildBenchmarkPolicyOk(job: Record, cachePolicy: CiBuildBenchmarkCachePolicy): boolean { + if (cachePolicy.forbidBuildkitCache && ciBuildBenchmarkLogHasBuildkitCache(job)) return false; + if (cachePolicy.forbidGitopsCatalogReuse && ciBuildBenchmarkLogHasReuse(job)) return false; + return true; +} + +export function ciBuildBenchmarkLogHasBuildkitCache(job: Record): boolean { + const logTail = typeof job.logTail === "string" ? job.logTail : ""; + return /"buildkitCacheRef"\s*:\s*"[^"]+"|--import-cache|--export-cache|writing cache image manifest/iu.test(logTail); +} + +export function ciBuildBenchmarkLogHasReuse(job: Record): boolean { + const logTail = typeof job.logTail === "string" ? job.logTail : ""; + return /"reusedFrom"\s*:\s*"(?!null)|"status"\s*:\s*"reused"/iu.test(logTail); +} + +export function ciBuildBenchmarkTaskRunRecords(job: Record): Record[] { + return Array.isArray(job.taskRuns) ? job.taskRuns.map(record) : []; +} + +export function ciBuildBenchmarkExpectedServices(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : []; +} + +export function ciBuildBenchmarkStatusText(value: unknown): string { + if (value === "True") return "succeeded"; + if (value === "False") return "failed"; + if (value === "Unknown") return "running"; + return renderCell(value, "pending"); +} + +export function classifyCiBuildBenchmarkFailure(text: string): string { + const value = text.toLowerCase(); + if (/cache-hit-forbidden|reused-from|reuse/i.test(text)) return "cache-hit-forbidden"; + if (/no such host|could not resolve|enotfound|dns/i.test(text)) return "dns"; + if (/429|rate limit|too many requests|toomanyrequests/i.test(text)) return "rate-limit"; + if (/tls|certificate|x509|timeout|timed out|i\/o timeout/i.test(text)) return "tls-timeout"; + if (/proxy|connect|connection reset|connection refused|econn/i.test(text)) return "proxy-connect"; + if (/unauthorized|authentication required|permission denied|forbidden|denied/i.test(text)) return "auth"; + if (/imagepullbackoff|errimagepull|imagepolicy|pull access denied/i.test(text)) return "image-policy"; + if (/push|registry|blob upload|manifest invalid|manifest unknown/i.test(text)) return "registry-push"; + return value.trim().length === 0 ? "unknown" : "build-script"; +} + +export function durationBetweenIso(startValue: unknown, endValue: unknown): string { + if (typeof startValue !== "string" || startValue.length === 0) return "-"; + const start = Date.parse(startValue); + if (!Number.isFinite(start)) return "-"; + const end = typeof endValue === "string" && endValue.length > 0 ? Date.parse(endValue) : Date.now(); + if (!Number.isFinite(end) || end < start) return "-"; + return formatDurationMs(end - start); +} + +export function formatDurationMs(ms: number): string { + const seconds = Math.round(ms / 1000); + const minutes = Math.floor(seconds / 60); + const rest = seconds % 60; + return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`; +} + +export function shortIsoTime(value: unknown): string { + if (typeof value !== "string" || value.length === 0) return "-"; + return value.replace(/^\d{4}-\d{2}-\d{2}T/u, "").replace(/Z$/u, "Z"); +} + +export function shortDisplay(value: string): string { + return /^[0-9a-f]{40}$/iu.test(value) ? value.slice(0, 12).toLowerCase() : value; +} + +export function validateBenchmarkProfileName(value: string, path: string): void { + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 63) throw new Error(`${path} must be a DNS-label style benchmark profile`); +} + +export function validateBenchmarkCatalogPathTemplate(value: string, path: string): void { + if (!value.includes("{profile}") || !value.includes("{pipelineRun}")) throw new Error(`${path} must include {profile} and {pipelineRun}`); + if (value.startsWith("/") || value.includes("\n") || value.includes("\r")) throw new Error(`${path} must be a relative repo path template`); + const rendered = value.replace(/\{profile\}/gu, "profile").replace(/\{pipelineRun\}/gu, "pipeline-run"); + if (rendered.split("/").some((segment) => segment === ".." || segment.length === 0)) throw new Error(`${path} must not contain empty or parent path segments`); + if (!rendered.endsWith(".json")) throw new Error(`${path} must render to a JSON catalog path`); +} + +export function asRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`); + return value as Record; +} + +export function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +export function renderTable(headers: string[], rows: string[][]): string[] { + const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0))); + const render = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); + return [render(headers), ...rows.map(render)]; +} + +export function renderCell(value: unknown, fallback = "-"): string { + if (value === undefined || value === null || value === "") return fallback; + return String(value); +} + +export function optionsModeFromCommand(command: unknown): string { + const value = String(command ?? ""); + if (value.endsWith(" status") || value.endsWith(" logs")) return "status"; + return "benchmark"; +} + +export function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value; +} + +export function optionalStringField(obj: Record, key: string, path: string): string | undefined { + const value = obj[key]; + if (value === undefined) return undefined; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value; +} + +export function absoluteConfigPathField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!value.startsWith("/") || value.includes("\0") || value.includes("..")) throw new Error(`${path}.${key} must be an absolute path without '..'`); + return value; +} + +export function validateKubernetesName(value: string, path: string): void { + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} must be a Kubernetes resource name`); +} + +export function validateSecretKey(value: string, path: string): void { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`); +} + +export function validateEnvKey(value: string, path: string): void { + if (!/^[A-Z0-9_]+$/u.test(value)) throw new Error(`${path} must be an env key`); +} + +export function validateSourceRef(value: string, path: string): void { + if (!/^[A-Za-z0-9_./-]+$/u.test(value) || value.includes("..")) throw new Error(`${path} has an unsupported sourceRef format`); +} + +export function numberField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`); + return value; +} + +export function positiveConfigIntegerField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`); + return value; +} + +export function nonNegativeIntegerField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`); + return value; +} + +export function numberArrayField(obj: Record, key: string, path: string): number[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) throw new Error(`${path}.${key} must be an array of integers`); + return [...value] as number[]; +} + +export function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path}.${key} must be an array of non-empty strings`); + return [...value] as string[]; +} + +export function stringRecordField(obj: Record, path: string): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`${path}.${key} has an unsupported key format`); + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + result[key] = value; + } + return result; +} + +export function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +export function boolField(obj: Record, key: string): boolean { + return obj[key] === true; +} + +export function numberValue(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +export function requiredOption(args: string[], name: string): string { + const index = args.indexOf(name); + if (index === -1) throw new Error(`${name} is required`); + const value = args[index + 1]; + if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`); + return value; +} + +export function optionValue(args: string[], name: string): string | null { + const index = args.indexOf(name); + if (index === -1) return null; + const value = args[index + 1]; + if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`); + return value; +} + +export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { + const raw = optionValue(args, name); + if (raw === null) return defaultValue; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); + return Math.min(value, maxValue); +} + +export function compactCommandResult(result: CommandResult): Record { + return { + exitCode: result.exitCode, + timedOut: result.timedOut, + stdoutBytes: Buffer.byteLength(result.stdout), + stderrBytes: Buffer.byteLength(result.stderr), + stdoutTail: result.stdout.slice(-2000), + stderrTail: result.stderr.slice(-2000), + }; +} + +export function shQuote(value: string): string { + return `'${value.replace(/'/gu, `'"'"'`)}'`; +} + +export function validateHttpsUrl(value: string, path: string): void { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new Error(`${path} must be a valid URL`); + } + if (parsed.protocol !== "https:") throw new Error(`${path} must use https://`); +} + +export function hostRouteProxyHostAllowed(value: string): boolean { + if (value === "127.0.0.1" || value === "localhost") return true; + const parts = value.split(".").map((part) => Number(part)); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + return parts[0] === 10 + || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) + || (parts[0] === 192 && parts[1] === 168); +} + +export function sha256Short(text: string): string { + return `sha256:${createHash("sha256").update(text).digest("hex")}`; +} diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 52a980a7..10eca299 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -5,290 +5,98 @@ import { runCommand, type CommandResult } from "./command"; import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark"; import { resolveEgressProxySourceRef } from "./egress-proxy-sources"; import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; +import { + HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + type ArgoOptions, + type CiBuildBenchmarkCachePolicy, + type CiBuildBenchmarkProfileSpec, + type CiBuildBenchmarkOptions, + type ControlPlaneConfig, + type ControlPlaneEgressProxySpec, + type ControlPlaneGitMirrorEgressProxySpec, + type ControlPlaneGitMirrorGithubTransportSpec, + type ControlPlaneHostRouteEgressProxySpec, + type ControlPlaneImagePolicy, + type ControlPlaneK3sInstallSpec, + type ControlPlaneK3sNodeSpec, + type ControlPlaneK8sServiceEgressProxySpec, + type ControlPlaneNodeSpec, + type ControlPlaneRuntimeProxySpec, + type ControlPlaneTargetSpec, + type ControlPlaneTektonArgoObserverRbacSpec, + type ControlPlaneTektonGitWorkspaceSecretSpec, + type ControlPlaneTektonInstallManifestSpec, + type ControlPlaneTektonInstallSpec, + type ControlPlaneTektonRuntimeObserverRbacSpec, + type DockerfileInlineSpec, + type EgressBenchmarkOptions, + type ImageRewriteSpec, + type InfraOptions, + type K3sInstallOptions, + type TektonInstallOptions, + type ToolsImageOptions, +} from "./hwlab-node-control-plane-model"; +import { + applyNext, + applyScript, + argoApplyStartScript, + argoDesiredManifest, + argoApplicationSkeleton, + argoProjectSkeleton, + ciBuildBenchmarkFailureRows, + ciBuildBenchmarkLiveOk, + ciBuildBenchmarkServiceRows, + ciBuildBenchmarkStartScript, + ciBuildBenchmarkStatusScript, + ciBuildBenchmarkTaskRows, + controlPlaneEgressProxySummary, + expectedSummary, + gitMirrorGithubTransportSummary, + gitMirrorRuntimeProxySpec, + k3sInstallPlan, + k3sInstallStatusScript, + k3sInstallSubmitScript, + k3sNodeConfigPlan, + manifestObjectSummary, + parseRemoteJson, + planSummary, + remoteJobLogs, + remoteJobStateDir, + remoteJobStatusScript, + runTransHost, + runTransK3s, + runtimeHostProxyConfig, + runtimeHostProxyEnv, + runtimeProxyReady, + shortDisplay, + statusNext, + statusScript, + tektonInstallApplyStartScript, + toolsImageBuildStartScript, + toolsImageDockerfile, + toolsImageStatus, +} from "./hwlab-node-control-plane-runtime"; import type { RenderedCliResult } from "./output"; import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; -export const HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH = "config/hwlab-node-control-plane.yaml"; - -type InfraAction = "plan" | "status" | "apply"; -type ToolsImageAction = "status" | "build" | "logs"; -type ArgoAction = "status" | "apply" | "logs"; -type EgressBenchmarkAction = "benchmark" | "status" | "logs"; -type CiBuildBenchmarkAction = "benchmark" | "status" | "logs"; - -interface InfraOptions { - action: InfraAction; - node: string; - lane: string; - dryRun: boolean; - confirm: boolean; - timeoutSeconds: number; -} - -interface ToolsImageOptions { - action: ToolsImageAction; - node: string; - lane: string; - dryRun: boolean; - confirm: boolean; - timeoutSeconds: number; - tailLines: number; -} - -interface ArgoOptions { - action: ArgoAction; - node: string; - lane: string; - dryRun: boolean; - confirm: boolean; - timeoutSeconds: number; - tailLines: number; -} - -interface EgressBenchmarkOptions { - action: EgressBenchmarkAction; - node: string; - lane: string; - profile: "no-mirror"; - dryRun: boolean; - confirm: boolean; - samples: number; - sampleTimeoutSeconds: number; - timeoutSeconds: number; - tailLines: number; -} - -interface CiBuildBenchmarkOptions { - action: CiBuildBenchmarkAction; - node: string; - lane: string; - profile: string; - dryRun: boolean; - confirm: boolean; - timeoutSeconds: number; - tailLines: number; -} - -interface CiBuildBenchmarkCachePolicy { - noPipelineRunReuse: boolean; - forceFullBuild: boolean; - forbidGitopsCatalogReuse: boolean; - forbidDependencyCache: boolean; - forbidBuildkitCache: boolean; - forbidRegistryMirror: boolean; - forbidLocalPreheatedImages: boolean; -} - -interface CiBuildBenchmarkProfileSpec { - profile: string; - runtimeLaneConfigRef: string; - pipelineRunPrefix: string; - catalogPathTemplate: string; - imageTagMode: "full"; - pipelineTimeoutSeconds: number; - cachePolicy: CiBuildBenchmarkCachePolicy; - requiredTimings: readonly string[]; - failureFamilies: readonly string[]; -} - -interface ControlPlaneEgressProxySpec { - mode: "k8s-service-cluster-ip"; - clientName: string; - namespace: string; - serviceName: string; - port: number; - sourceConfigRef: string | null; - sourceFingerprint: string | null; - sourceRef: string; - sourceKey: string; - sourceType: "subscription-url" | "master-shadowsocks"; - preferredOutbound: "vless-reality" | "hysteria2" | null; - noProxy: readonly string[]; -} - -interface ControlPlaneGitMirrorEgressProxySpec { - mode: "node-global" | "direct"; - required: boolean; -} - -type ControlPlaneGitMirrorGithubTransportSpec = - | { - mode: "ssh"; - privateKeySecretKey: string; - privateKeySourceRef: string; - privateKeySourceKey: string; - privateKeySourceEncoding: "plain" | "base64"; - knownHostsSecretKey: string | null; - knownHostsSourceRef: string | null; - knownHostsSourceKey: string | null; - knownHostsSourceEncoding: "plain" | "base64" | null; - } - | { - mode: "https"; - username: string; - tokenSecretName: string; - tokenSecretKey: string; - tokenSourceRef: string; - tokenSourceKey: string; - }; - -interface ControlPlaneTektonGitWorkspaceSecretSpec { - name: string; - namespace: string; - sourceRefFrom: "gitMirror.githubTransport"; - privateKeySecretKey: string; - knownHostsSecretKey: string; -} - -interface ControlPlaneTektonRuntimeObserverRbacSpec { - namespace: string; - roleName: string; - roleBindingName: string; -} - -interface ControlPlaneTektonArgoObserverRbacSpec { - namespace: string; - roleName: string; - roleBindingName: string; -} - -interface ControlPlaneNodeSpec { - id: string; - route: string; - kubeRoute: string; - k3s: ControlPlaneK3sNodeSpec | null; - registry: { endpoint: string }; - egressProxy: ControlPlaneEgressProxySpec | null; -} - -interface ControlPlaneK3sNodeSpec { - serviceName: string; - dropInPath: string; - nodeStatusName: string; - execStartPre: readonly (readonly string[])[]; - serverArgs: readonly string[]; - kubelet: { maxPods: number }; -} - -interface DockerfileInlineSpec { - filename: string; - lines: readonly string[]; -} - -interface ImageRewriteSpec { - source: string; - pullImage: string; - target: string; -} - -interface ControlPlaneTargetSpec { - id: string; - node: string; - lane: string; - enabled: boolean; - ciNamespace: string; - runtimeNamespace: string; - source: { repository: string; branch: string }; - gitops: { branch: string; path: string }; - gitMirror: { - namespace: string; - serviceReadName: string; - serviceWriteName: string; - cachePvcName: string; - cachePvcStorage: string; - cacheHostPath: string | null; - servicePort: number; - deploymentReplicas: number; - secretName: string; - syncConfigMapName: string; - syncJobPrefix: string; - flushJobPrefix: string; - readUrl: string; - writeUrl: string; - egressProxy: ControlPlaneGitMirrorEgressProxySpec | null; - githubTransport: ControlPlaneGitMirrorGithubTransportSpec; - }; - tekton: { - pipelineName: string; - serviceAccountName: string; - pipelineRunPrefix: string; - gitWorkspaceSecret: ControlPlaneTektonGitWorkspaceSecretSpec; - runtimeObserverRbac: ControlPlaneTektonRuntimeObserverRbacSpec; - argoObserverRbac: ControlPlaneTektonArgoObserverRbacSpec; - toolsImage: { - output: string; - imagePullPolicy: "Always" | "IfNotPresent" | "Never"; - sourceKind: "dockerfile" | "docker-compose"; - context: string; - dockerfile?: string; - dockerfileInline?: DockerfileInlineSpec; - composeFile?: string; - buildArgs: Readonly>; - buildNetwork: string | null; - publicBaseImages: readonly string[]; - buildOwner: string; - buildMode: string; - }; - }; - ciBuildBenchmarks: readonly CiBuildBenchmarkProfileSpec[]; - argo: { - namespace: string; - projectName: string; - applicationName: string; - applicationFile: string; - install: { - enabled: boolean; - sourceKind: "url"; - version: string; - manifestUrl: string; - fieldManager: string; - imagePullPolicy: "Always" | "IfNotPresent" | "Never"; - preloadImages: readonly string[]; - imageRewrites: readonly ImageRewriteSpec[]; - requiredCrds: readonly string[]; - expectedDeployments: readonly string[]; - expectedStatefulSets: readonly string[]; - readinessTimeoutSeconds: number; - }; - }; -} - -interface ControlPlaneImagePolicy { - requireReproducibleBuildSource: boolean; - forbidPrivateOrNodeLocalImagesAsInputs: boolean; - allowNodeLocalRegistryAsBuildOutput: boolean; - requiredSourceKinds: readonly ("dockerfile" | "docker-compose")[]; -} - -interface ControlPlaneConfig { - version: number; - kind: string; - metadata: { owner: string; relatedIssues: readonly number[] }; - imagePolicy: ControlPlaneImagePolicy; - nodes: Record; - targets: readonly ControlPlaneTargetSpec[]; -} - -export interface HwlabNodeControlPlaneSourceWorkspaceBootstrapSpec { - readonly configPath: string; - readonly targetId: string; - readonly node: string; - readonly lane: string; - readonly ciNamespace: string; - readonly serviceAccountName: string; - readonly toolsImage: string; - readonly imagePullPolicy: "Always" | "IfNotPresent" | "Never"; - readonly gitReadUrl: string; - readonly gitWriteUrl: string; - readonly gitMirrorNamespace: string; -} +export { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH } from "./hwlab-node-control-plane-model"; export function runHwlabNodeControlPlaneInfra(args: string[]): Record | RenderedCliResult { + if (args[0] === "k3s") { + const options = parseK3sInstallOptions(args.slice(1)); + const { config, node, target } = controlPlaneContext(options.node, options.lane); + return runK3sInstallCommand(config, node, target, options); + } if (args[0] === "tools-image") { const options = parseToolsImageOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runToolsImageCommand(config, node, target, options); } + if (args[0] === "tekton") { + const options = parseTektonInstallOptions(args.slice(1)); + const { config, node, target } = controlPlaneContext(options.node, options.lane); + return runTektonInstallCommand(config, node, target, options); + } if (args[0] === "argo") { const options = parseArgoOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); @@ -358,10 +166,17 @@ export function hwlabNodeControlPlaneInfraHelp(): Record { "bun scripts/cli.ts hwlab nodes control-plane infra status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra k3s plan --node JD01 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node JD01 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra tekton status --node JD01 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node JD01 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node JD01 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", @@ -418,6 +233,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta const gitMirror = record(components.gitMirror); const gitMirrorGithubTransport = record(gitMirror.githubTransport); const tekton = record(components.tekton); + const tektonInstall = record(tekton.install); const ciNamespace = record(components.ciNamespace); const ciGitWorkspaceSecret = record(ciNamespace.gitWorkspaceSecret); const runtimeNamespace = record(components.runtimeNamespace); @@ -428,9 +244,14 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta || (boolField(k3sNodeConfig, "dropInMatches") && numberValue(k3sNodeConfig.liveCapacityPods) === node.k3s.kubelet.maxPods && numberValue(k3sNodeConfig.liveAllocatablePods) === node.k3s.kubelet.maxPods); + const tektonRuntimeProxyReady = runtimeProxyReady(tektonInstall); + const argoRuntimeProxyReady = runtimeProxyReady(argoInstall); const ok = result.exitCode === 0 && k3sNodeConfigReady && boolField(tekton, "installed") + && boolField(tektonInstall, "crdsReady") + && boolField(tektonInstall, "deploymentsReady") + && tektonRuntimeProxyReady && boolField(ciNamespace, "exists") && boolField(ciGitWorkspaceSecret, "ready") && boolField(runtimeNamespace, "exists") @@ -448,7 +269,8 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && boolField(argoObserverRbac, "ready") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") - && boolField(argoInstall, "statefulSetsReady"); + && boolField(argoInstall, "statefulSetsReady") + && argoRuntimeProxyReady; return { ok, command: "hwlab nodes control-plane infra status", @@ -463,6 +285,9 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta ok, k3sNodeConfigReady, tektonInstalled: boolField(tekton, "installed"), + tektonCrdsReady: boolField(tektonInstall, "crdsReady"), + tektonDeploymentsReady: boolField(tektonInstall, "deploymentsReady"), + tektonRuntimeProxyReady, ciNamespaceExists: boolField(ciNamespace, "exists"), ciGitWorkspaceSecretReady: boolField(ciGitWorkspaceSecret, "ready"), runtimeNamespaceExists: boolField(runtimeNamespace, "exists"), @@ -482,11 +307,12 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta argoCrdsReady: boolField(argoInstall, "crdsReady"), argoDeploymentsReady: boolField(argoInstall, "deploymentsReady"), argoStatefulSetsReady: boolField(argoInstall, "statefulSetsReady"), + argoRuntimeProxyReady, registryReady: boolField(registry, "ready"), toolsImageReady: boolField(registry, "toolsImageReady"), }, result: compactCommandResult(result), - next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace, runtimeNamespace, k3sNodeConfig), + next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, tekton, argo, ciNamespace, runtimeNamespace, k3sNodeConfig), }; } @@ -545,12 +371,83 @@ function infraApply(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, tar }; } +function runK3sInstallCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: K3sInstallOptions): Record { + const spec = node.k3s?.install ?? null; + if (node.k3s === null || spec === null || !spec.enabled) { + throw new Error(`nodes.${node.id}.k3s.install must be enabled in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); + } + const plan = k3sInstallPlan(node, target, spec); + if (options.action === "plan" || (options.action === "install" && options.dryRun)) { + return { + ok: true, + command: `hwlab nodes control-plane infra k3s ${options.action}`, + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: options.action === "install" ? "dry-run" : "plan", + mutation: false, + plan, + next: { + install: `bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node ${node.id} --lane ${target.lane} --confirm`, + status: `bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node ${node.id} --lane ${target.lane}`, + }, + }; + } + if (options.action === "status") { + const result = runTransHost(node.route, k3sInstallStatusScript(node, target, spec, options.tailLines), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + const status = typeof parsed === "object" && parsed !== null ? parsed as Record : { stdoutPreview: result.stdout.slice(0, 2000) }; + const checks = record(status.checks); + const ok = result.exitCode === 0 + && boolField(checks, "binarySha256Ok") + && boolField(checks, "serviceActive") + && boolField(checks, "nodeReady"); + return { + ok, + command: "hwlab nodes control-plane infra k3s status", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: "status", + mutation: false, + plan, + status, + result: compactCommandResult(result), + next: ok + ? { k3sRouteSmoke: `trans ${node.kubeRoute} kubectl get nodes -o wide` } + : { install: `bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node ${node.id} --lane ${target.lane} --confirm` }, + }; + } + + const result = runTransHost(node.route, k3sInstallSubmitScript(node, target, spec), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + return { + ok: result.exitCode === 0, + command: "hwlab nodes control-plane infra k3s install", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: "confirmed-submit", + mutation: result.exitCode === 0, + plan, + start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, + result: compactCommandResult(result), + next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node ${node.id} --lane ${target.lane}` }, + }; +} + function runToolsImageCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { if (options.action === "status") return toolsImageCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "tools-image", options); return toolsImageBuild(node, target, options); } +function runTektonInstallCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: TektonInstallOptions): Record { + if (options.action === "status") return tektonInstallCommandStatus(node, target, options); + if (options.action === "logs") return remoteJobLogs(node, target, "tekton", options); + return tektonInstallApply(node, target, options); +} + function runArgoCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { if (options.action === "status") return argoCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "argo", options); @@ -1014,6 +911,93 @@ function toolsImageBuild(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS }; } +function tektonInstallCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: TektonInstallOptions): Record { + const result = runTransK3s(node.kubeRoute, statusScript(node, target), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; + const tekton = record(record(status.components).tekton); + const install = record(tekton.install); + const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "tekton", options.tailLines), options.timeoutSeconds); + const jobStatus = parseRemoteJson(jobResult.stdout); + const ok = boolField(tekton, "installed") + && boolField(install, "crdsReady") + && boolField(install, "deploymentsReady") + && runtimeProxyReady(install); + return { + ok, + command: "hwlab nodes control-plane infra tekton status", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mutation: false, + expected: { + install: target.tekton.install, + }, + readiness: { + installed: boolField(tekton, "installed"), + crdsReady: boolField(install, "crdsReady"), + deploymentsReady: boolField(install, "deploymentsReady"), + runtimeProxyReady: runtimeProxyReady(install), + }, + tekton, + job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, + result: { k3s: compactCommandResult(result), job: compactCommandResult(jobResult) }, + next: ok + ? { infraApply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm` } + : { apply: `bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node ${node.id} --lane ${target.lane} --confirm`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node ${node.id} --lane ${target.lane}` }, + }; +} + +function tektonInstallApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: TektonInstallOptions): Record { + if (options.confirm && options.dryRun) throw new Error("tekton apply accepts only one of --dry-run or --confirm"); + if (!target.tekton.install.enabled) throw new Error(`targets.${target.id}.tekton.install.enabled=false`); + const dryRun = options.dryRun || !options.confirm; + const applyPlan = { + version: target.tekton.install.version, + sourceKind: target.tekton.install.sourceKind, + fieldManager: target.tekton.install.fieldManager, + manifests: target.tekton.install.manifests, + requiredCrds: target.tekton.install.requiredCrds, + expectedDeploymentNamespaces: target.tekton.install.expectedDeploymentNamespaces, + readinessTimeoutSeconds: target.tekton.install.readinessTimeoutSeconds, + stateDir: remoteJobStateDir(target, "tekton"), + egressProxy: controlPlaneEgressProxySummary(node.egressProxy), + runtimeProxy: runtimeHostProxyConfig(node, target.tekton.install.runtimeProxy), + }; + if (dryRun) { + return { + ok: true, + command: "hwlab nodes control-plane infra tekton apply", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: "dry-run", + mutation: false, + applyPlan, + next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node ${node.id} --lane ${target.lane} --confirm` }, + }; + } + const result = runTransK3s(node.kubeRoute, tektonInstallApplyStartScript(node, target), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + return { + ok: result.exitCode === 0, + command: "hwlab nodes control-plane infra tekton apply", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: "confirmed-start", + mutation: result.exitCode === 0, + applyPlan, + start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, + result: compactCommandResult(result), + next: { + status: `bun scripts/cli.ts hwlab nodes control-plane infra tekton status --node ${node.id} --lane ${target.lane}`, + logs: `bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node ${node.id} --lane ${target.lane}`, + infraApply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, + }, + }; +} + function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { const result = runTransK3s(node.kubeRoute, statusScript(node, target), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); @@ -1027,7 +1011,8 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") - && boolField(argoInstall, "statefulSetsReady"); + && boolField(argoInstall, "statefulSetsReady") + && runtimeProxyReady(argoInstall); return { ok, command: "hwlab nodes control-plane infra argo status", @@ -1048,6 +1033,7 @@ function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTarge crdsReady: boolField(argoInstall, "crdsReady"), deploymentsReady: boolField(argoInstall, "deploymentsReady"), statefulSetsReady: boolField(argoInstall, "statefulSetsReady"), + runtimeProxyReady: runtimeProxyReady(argoInstall), }, argo, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, @@ -1076,6 +1062,7 @@ function argoApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, o desiredSha256: sha256Short(desiredYaml), stateDir: remoteJobStateDir(target, "argo"), egressProxy: controlPlaneEgressProxySummary(node.egressProxy), + runtimeProxy: runtimeHostProxyConfig(node, target.argo.install.runtimeProxy), }; if (dryRun) { return { @@ -1131,6 +1118,28 @@ function parseInfraOptions(args: string[]): InfraOptions { }; } +function parseK3sInstallOptions(args: string[]): K3sInstallOptions { + const [actionRaw] = args; + if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") { + throw new Error("infra k3s usage: k3s plan|install|status --node NODE --lane vNN [--dry-run|--confirm]"); + } + if (actionRaw !== "plan" && actionRaw !== "install" && actionRaw !== "status") { + throw new Error(`unsupported k3s action ${actionRaw}; expected plan|install|status`); + } + const confirm = args.includes("--confirm"); + const explicitDryRun = args.includes("--dry-run"); + if (confirm && explicitDryRun) throw new Error("k3s install accepts only one of --confirm or --dry-run"); + return { + action: actionRaw, + node: requiredOption(args, "--node"), + lane: requiredOption(args, "--lane"), + confirm, + dryRun: actionRaw === "install" ? explicitDryRun || !confirm : true, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), + tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), + }; +} + function parseToolsImageOptions(args: string[]): ToolsImageOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra tools-image usage: tools-image status|build|logs --node NODE --lane vNN [--dry-run|--confirm]"); @@ -1149,6 +1158,24 @@ function parseToolsImageOptions(args: string[]): ToolsImageOptions { }; } +function parseTektonInstallOptions(args: string[]): TektonInstallOptions { + const [actionRaw] = args; + if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra tekton usage: tekton status|apply|logs --node NODE --lane vNN [--dry-run|--confirm]"); + if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "logs") throw new Error(`unsupported tekton action ${actionRaw}; expected status|apply|logs`); + const confirm = args.includes("--confirm"); + const explicitDryRun = args.includes("--dry-run"); + if (confirm && explicitDryRun) throw new Error("tekton accepts only one of --confirm or --dry-run"); + return { + action: actionRaw, + node: requiredOption(args, "--node"), + lane: requiredOption(args, "--lane"), + confirm, + dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true, + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), + tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), + }; +} + function parseArgoOptions(args: string[]): ArgoOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra argo usage: argo status|apply|logs --node NODE --lane vNN [--dry-run|--confirm]"); @@ -1295,6 +1322,74 @@ function toolsImageSpec(raw: Record, path: string): ControlPlan }; } +function tektonInstallManifestSpec(raw: Record, path: string): ControlPlaneTektonInstallManifestSpec { + const name = stringField(raw, "name", path); + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(name)) throw new Error(`${path}.name must be a lowercase identifier`); + const url = stringField(raw, "url", path); + validateHttpsUrl(url, `${path}.url`); + return { name, url }; +} + +function tektonInstallSpec(raw: Record, path: string): ControlPlaneTektonInstallSpec { + const sourceKind = stringField(raw, "sourceKind", path); + if (sourceKind !== "url") throw new Error(`${path}.sourceKind must be url`); + const manifestsRaw = raw.manifests; + if (!Array.isArray(manifestsRaw) || manifestsRaw.length === 0) throw new Error(`${path}.manifests must be a non-empty array`); + const manifests = manifestsRaw.map((item, index) => tektonInstallManifestSpec(asRecord(item, `${path}.manifests[${index}]`), `${path}.manifests[${index}]`)); + const manifestNames = new Set(); + for (const manifest of manifests) { + if (manifestNames.has(manifest.name)) throw new Error(`${path}.manifests contains duplicate name ${manifest.name}`); + manifestNames.add(manifest.name); + } + const requiredCrds = stringArrayField(raw, "requiredCrds", path); + for (const crd of requiredCrds) { + if (!/^[a-z0-9.-]+$/u.test(crd)) throw new Error(`${path}.requiredCrds contains invalid CRD name ${crd}`); + } + const expectedDeploymentNamespaces = stringArrayField(raw, "expectedDeploymentNamespaces", path); + for (const namespace of expectedDeploymentNamespaces) validateKubernetesName(namespace, `${path}.expectedDeploymentNamespaces`); + return { + enabled: booleanField(raw, "enabled", path), + sourceKind, + version: stringField(raw, "version", path), + fieldManager: stringField(raw, "fieldManager", path), + manifests, + requiredCrds, + expectedDeploymentNamespaces, + readinessTimeoutSeconds: positiveConfigIntegerField(raw, "readinessTimeoutSeconds", path), + runtimeProxy: runtimeProxySpec(raw.runtimeProxy ?? { enabled: false }, `${path}.runtimeProxy`), + }; + } + +function runtimeProxySpec(value: unknown, path: string): ControlPlaneRuntimeProxySpec { + const raw = asRecord(value, path); + const enabled = booleanField(raw, "enabled", path); + if (!enabled) { + return { + enabled, + mode: "host-route", + configRef: optionalStringField(raw, "configRef", path) ?? null, + hostNetwork: false, + injectEnv: false, + deployments: [], + statefulSets: [], + }; + } + const mode = stringField(raw, "mode", path); + if (mode !== "host-route") throw new Error(`${path}.mode must be host-route`); + const configRef = stringField(raw, "configRef", path); + if (!/^nodes\.[A-Za-z0-9_.-]+\.egressProxy$/u.test(configRef)) { + throw new Error(`${path}.configRef must point at nodes..egressProxy`); + } + const hostNetwork = booleanField(raw, "hostNetwork", path); + const injectEnv = booleanField(raw, "injectEnv", path); + if (!hostNetwork && !injectEnv) throw new Error(`${path} must enable at least one of hostNetwork or injectEnv`); + const deployments = raw.deployments === undefined ? [] : stringArrayField(raw, "deployments", path); + const statefulSets = raw.statefulSets === undefined ? [] : stringArrayField(raw, "statefulSets", path); + for (const name of [...deployments, ...statefulSets]) validateKubernetesName(name, path); + if (deployments.length + statefulSets.length === 0) throw new Error(`${path} must declare deployments or statefulSets when enabled=true`); + return { enabled, mode, configRef, hostNetwork, injectEnv, deployments, statefulSets }; +} + function dockerfileInlineSpec(raw: Record, path: string): DockerfileInlineSpec { const filename = stringField(raw, "filename", path); if (!/^[A-Za-z0-9._/-]+$/u.test(filename) || filename.includes("..")) throw new Error(`${path}.filename has an unsupported format`); @@ -1313,21 +1408,22 @@ function argoInstallSpec(raw: Record, path: string): ControlPla const imageRewrites = imageRewritesRaw.map((item, index) => imageRewriteSpec(asRecord(item, `${path}.imageRewrites[${index}]`), `${path}.imageRewrites[${index}]`)); const manifestUrl = stringField(raw, "manifestUrl", path); validateHttpsUrl(manifestUrl, `${path}.manifestUrl`); - return { - enabled: booleanField(raw, "enabled", path), - sourceKind, - version: stringField(raw, "version", path), - manifestUrl, + return { + enabled: booleanField(raw, "enabled", path), + sourceKind, + version: stringField(raw, "version", path), + manifestUrl, fieldManager: stringField(raw, "fieldManager", path), imagePullPolicy, preloadImages: stringArrayField(raw, "preloadImages", path), imageRewrites, - requiredCrds: stringArrayField(raw, "requiredCrds", path), - expectedDeployments: stringArrayField(raw, "expectedDeployments", path), - expectedStatefulSets: stringArrayField(raw, "expectedStatefulSets", path), - readinessTimeoutSeconds: positiveConfigIntegerField(raw, "readinessTimeoutSeconds", path), - }; -} + requiredCrds: stringArrayField(raw, "requiredCrds", path), + expectedDeployments: stringArrayField(raw, "expectedDeployments", path), + expectedStatefulSets: stringArrayField(raw, "expectedStatefulSets", path), + readinessTimeoutSeconds: positiveConfigIntegerField(raw, "readinessTimeoutSeconds", path), + runtimeProxy: runtimeProxySpec(raw.runtimeProxy ?? { enabled: false }, `${path}.runtimeProxy`), + }; + } function ciBuildBenchmarkProfileSpecs(raw: unknown, path: string): readonly CiBuildBenchmarkProfileSpec[] { if (raw === undefined) return []; @@ -1443,6 +1539,7 @@ function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3 const nodeStatusName = stringField(raw, "nodeStatusName", path); if (!/^[A-Za-z0-9_.-]+$/u.test(nodeStatusName)) throw new Error(`${path}.nodeStatusName has an unsupported Kubernetes node name`); const execStartPre = execStartPreField(raw.execStartPre, `${path}.execStartPre`); + const install = raw.install === undefined ? null : k3sInstallSpec(asRecord(raw.install, `${path}.install`), `${path}.install`); const serverArgs = stringArrayField(raw, "serverArgs", path); if (serverArgs.length === 0 || serverArgs[0] !== "server") throw new Error(`${path}.serverArgs must start with k3s server`); for (const [index, arg] of serverArgs.entries()) { @@ -1460,11 +1557,59 @@ function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3 dropInPath, nodeStatusName, execStartPre, + install, serverArgs, kubelet: { maxPods }, }; } +function k3sInstallSpec(raw: Record, path: string): ControlPlaneK3sInstallSpec { + const localRegistryRaw = asRecord(raw.localRegistry, `${path}.localRegistry`); + const stateRaw = asRecord(raw.state, `${path}.state`); + const downloadsRaw = asRecord(raw.downloads, `${path}.downloads`); + const installScriptUrl = stringField(raw, "installScriptUrl", path); + const binaryUrl = stringField(raw, "binaryUrl", path); + const sha256Url = stringField(raw, "sha256Url", path); + validateHttpsUrl(installScriptUrl, `${path}.installScriptUrl`); + validateHttpsUrl(binaryUrl, `${path}.binaryUrl`); + validateHttpsUrl(sha256Url, `${path}.sha256Url`); + const expectedSha256 = stringField(raw, "expectedSha256", path).toLowerCase(); + if (!/^[0-9a-f]{64}$/u.test(expectedSha256)) throw new Error(`${path}.expectedSha256 must be a 64-character sha256 hex digest`); + const hostProxyConfigRef = stringField(raw, "hostProxyConfigRef", path); + if (!hostProxyConfigRef.startsWith("config/platform-infra/host-proxy.yaml#targets.")) { + throw new Error(`${path}.hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); + } + return { + enabled: booleanField(raw, "enabled", path), + channel: stringField(raw, "channel", path), + version: stringField(raw, "version", path), + installScriptUrl, + binaryUrl, + sha256Url, + expectedSha256, + hostProxyConfigRef, + proxyEnvPath: absoluteConfigPathField(raw, "proxyEnvPath", path), + registriesYamlPath: absoluteConfigPathField(raw, "registriesYamlPath", path), + localRegistry: { + containerName: stringField(localRegistryRaw, "containerName", `${path}.localRegistry`), + image: stringField(localRegistryRaw, "image", `${path}.localRegistry`), + canonicalImage: stringField(localRegistryRaw, "canonicalImage", `${path}.localRegistry`), + bind: stringField(localRegistryRaw, "bind", `${path}.localRegistry`), + }, + state: { + dir: absoluteConfigPathField(stateRaw, "dir", `${path}.state`), + logPath: absoluteConfigPathField(stateRaw, "logPath", `${path}.state`), + statusPath: absoluteConfigPathField(stateRaw, "statusPath", `${path}.state`), + }, + downloads: { + connectTimeoutSeconds: positiveConfigIntegerField(downloadsRaw, "connectTimeoutSeconds", `${path}.downloads`), + maxTimeSeconds: positiveConfigIntegerField(downloadsRaw, "maxTimeSeconds", `${path}.downloads`), + retry: positiveConfigIntegerField(downloadsRaw, "retry", `${path}.downloads`), + retryDelaySeconds: positiveConfigIntegerField(downloadsRaw, "retryDelaySeconds", `${path}.downloads`), + }, + }; +} + function execStartPreField(raw: unknown, path: string): readonly (readonly string[])[] { if (raw === undefined) return []; if (!Array.isArray(raw)) throw new Error(`${path} must be an array of argv arrays`); @@ -1484,7 +1629,35 @@ function execStartPreField(raw: unknown, path: string): readonly (readonly strin function egressProxySpec(raw: Record, path: string): ControlPlaneEgressProxySpec { const mode = stringField(raw, "mode", path); - if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`); + if (mode === "host-route") { + const hostProxyConfigRef = stringField(raw, "hostProxyConfigRef", path); + if (!hostProxyConfigRef.startsWith("config/platform-infra/host-proxy.yaml#targets.")) { + throw new Error(`${path}.hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); + } + const proxyUrl = stringField(raw, "proxyUrl", path); + const parsed = new URL(proxyUrl); + if ( + parsed.protocol !== "http:" + || parsed.port.length === 0 + || parsed.username.length > 0 + || parsed.password.length > 0 + || parsed.hash.length > 0 + || (parsed.pathname !== "/" && parsed.pathname.length > 0) + || parsed.search.length > 0 + || !hostRouteProxyHostAllowed(parsed.hostname) + ) { + throw new Error(`${path}.proxyUrl must be a credential-free http://: URL`); + } + return { + mode, + clientName: stringField(raw, "clientName", path), + hostProxyConfigRef, + proxyEnvPath: absoluteConfigPathField(raw, "proxyEnvPath", path), + proxyUrl, + noProxy: stringArrayField(raw, "noProxy", path), + }; + } + if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`); const sourceConfigRef = optionalStringField(raw, "sourceConfigRef", path) ?? null; const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`); const sourceType = source?.sourceType ?? stringField(raw, "sourceType", path); @@ -1526,10 +1699,15 @@ function preferredOutboundField(raw: Record, key: string, path: function gitMirrorEgressProxySpec(raw: Record, path: string): ControlPlaneGitMirrorEgressProxySpec { const mode = stringField(raw, "mode", path); - if (mode !== "node-global" && mode !== "direct") throw new Error(`${path}.mode must be node-global or direct`); + if (mode !== "node-global" && mode !== "host-route" && mode !== "direct") throw new Error(`${path}.mode must be node-global, host-route, or direct`); + const podHostNetwork = raw.podHostNetwork === undefined ? false : booleanField(raw, "podHostNetwork", path); + const injectPodEnv = raw.injectPodEnv === undefined ? false : booleanField(raw, "injectPodEnv", path); + if (mode === "host-route" && !podHostNetwork && !injectPodEnv) throw new Error(`${path} must enable podHostNetwork or injectPodEnv when mode=host-route`); return { mode, required: raw.required === undefined ? mode !== "direct" : booleanField(raw, "required", path), + podHostNetwork, + injectPodEnv, }; } @@ -1696,6 +1874,8 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), cacheHostPath: optionalStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`) ?? null, servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), + readContainerPort: numberField(gitMirror, "readContainerPort", `${path}.gitMirror`), + writeContainerPort: numberField(gitMirror, "writeContainerPort", `${path}.gitMirror`), deploymentReplicas: nonNegativeIntegerField(gitMirror, "deploymentReplicas", `${path}.gitMirror`), secretName: stringField(gitMirror, "secretName", `${path}.gitMirror`), syncConfigMapName: stringField(gitMirror, "syncConfigMapName", `${path}.gitMirror`), @@ -1707,6 +1887,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa githubTransport, }, tekton: { + install: tektonInstallSpec(asRecord(tekton.install, `${path}.tekton.install`), `${path}.tekton.install`), pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), @@ -2057,7 +2238,9 @@ function service(name: string, namespace: string, labels: Record function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return sha256Short(JSON.stringify({ repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], + ports: { read: target.gitMirror.readContainerPort, write: target.gitMirror.writeContainerPort }, githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport), + runtimeProxy: runtimeHostProxyConfig(node, gitMirrorRuntimeProxySpec(node, target)), server: gitMirrorServerJs(), status: gitMirrorStatusShell(), sync: gitMirrorSyncShell(node, target), @@ -2066,6 +2249,31 @@ function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTar } function gitMirrorDeployment(name: string, namespace: string, labels: Record, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, mode: "read" | "write"): Record { + const runtimeProxy = gitMirrorRuntimeProxySpec(node, target); + const containerPort = mode === "read" ? target.gitMirror.readContainerPort : target.gitMirror.writeContainerPort; + const podSpec: Record = { + containers: [{ + name: "git-mirror", + image: target.tekton.toolsImage.output, + command: ["node", "/etc/git-mirror/server.js"], + env: [ + { name: "PORT", value: String(containerPort) }, + { name: "GIT_PROJECT_ROOT", value: "/cache" }, + { name: "GIT_MIRROR_MODE", value: mode }, + ...runtimeHostProxyEnv(node, runtimeProxy), + ], + ports: [{ name: "http", containerPort }], + volumeMounts: [{ name: "cache", mountPath: "/cache" }, { name: "config", mountPath: "/etc/git-mirror" }], + }], + volumes: [ + { name: "cache", ...(target.gitMirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: target.gitMirror.cachePvcName } } : { hostPath: { path: target.gitMirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, + { name: "config", configMap: { name: target.gitMirror.syncConfigMapName, defaultMode: 0o755 } }, + ], + }; + if (runtimeProxy.enabled && runtimeProxy.hostNetwork) { + podSpec.hostNetwork = true; + podSpec.dnsPolicy = "ClusterFirstWithHostNet"; + } return { apiVersion: "apps/v1", kind: "Deployment", @@ -2073,32 +2281,16 @@ function gitMirrorDeployment(name: string, namespace: string, labels: Record&2`, @@ -2427,6 +2636,7 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", + "discarded_stale_gitops=false", "if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", @@ -2434,10 +2644,14 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", + " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && [ \"${UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS:-false}\" = \"true\" ]; then", + " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", + " discarded_stale_gitops=true", + " printf '%s\\n' \"git-mirror sync: discarded stale local gitops ref local=${local_gitops} github=${github_gitops}\" >&2", " fi", "fi", "git --git-dir=\"$repo\" update-server-info", - "export repository source_branch gitops_branch started_at", + "export repository source_branch gitops_branch started_at discarded_stale_gitops", "node <<'NODE' | tee /cache/HWLAB.last-sync.json", "const { execFileSync } = require('node:child_process');", "const repository = process.env.repository;", @@ -2450,7 +2664,7 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg "const localGitops = rev(`refs/heads/${gitopsBranch}`);", "const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);", "const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));", - "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush }));", + "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));", "NODE", "cat /cache/HWLAB.last-sync.json", "", @@ -2526,1552 +2740,6 @@ function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTar ].join("\n"); } -function argoDesiredManifest(target: ControlPlaneTargetSpec): Record[] { - return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; -} - -function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record { - return { - apiVersion: "argoproj.io/v1alpha1", - kind: "AppProject", - metadata: { name: target.argo.projectName, namespace: target.argo.namespace }, - spec: { - sourceRepos: [target.gitMirror.readUrl], - destinations: [{ server: "https://kubernetes.default.svc", namespace: target.runtimeNamespace }], - clusterResourceWhitelist: [{ group: "*", kind: "*" }], - namespaceResourceWhitelist: [{ group: "*", kind: "*" }], - }, - }; -} - -function argoApplicationSkeleton(target: ControlPlaneTargetSpec): Record { - return { - apiVersion: "argoproj.io/v1alpha1", - kind: "Application", - metadata: { name: target.argo.applicationName, namespace: target.argo.namespace }, - spec: { - project: target.argo.projectName, - source: { repoURL: target.gitMirror.readUrl, targetRevision: target.gitops.branch, path: target.gitops.path }, - destination: { server: "https://kubernetes.default.svc", namespace: target.runtimeNamespace }, - syncPolicy: { automated: { prune: true, selfHeal: true } }, - }, - }; -} - -function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { - return { - id: target.id, - node: node.id, - kubeRoute: node.kubeRoute, - lane: target.lane, - enabled: target.enabled, - ciNamespace: target.ciNamespace, - runtimeNamespace: target.runtimeNamespace, - k3sNodeConfig: k3sNodeConfigPlan(node), - registry: node.registry.endpoint, - egressProxy: controlPlaneEgressProxySummary(node.egressProxy), - sourceBranch: target.source.branch, - gitopsBranch: target.gitops.branch, - gitopsPath: target.gitops.path, - gitMirrorNamespace: target.gitMirror.namespace, - readUrl: target.gitMirror.readUrl, - writeUrl: target.gitMirror.writeUrl, - pipeline: target.tekton.pipelineName, - pipelineRunPrefix: target.tekton.pipelineRunPrefix, - serviceAccount: target.tekton.serviceAccountName, - gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), - runtimeObserverRbac: target.tekton.runtimeObserverRbac, - argoObserverRbac: target.tekton.argoObserverRbac, - toolsImage: target.tekton.toolsImage, - argoApplication: target.argo.applicationName, - argoInstall: { - enabled: target.argo.install.enabled, - version: target.argo.install.version, - manifestUrl: target.argo.install.manifestUrl, - imageRewrites: target.argo.install.imageRewrites, - }, - }; -} - -function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { - return { - sourceRepo: target.source.repository, - branch: target.source.branch, - gitopsBranch: target.gitops.branch, - runtimePath: target.gitops.path, - runtimeNamespace: target.runtimeNamespace, - namespace: target.ciNamespace, - k3sNodeConfig: k3sNodeConfigPlan(node), - gitMirror: { - namespace: target.gitMirror.namespace, - readUrl: target.gitMirror.readUrl, - writeUrl: target.gitMirror.writeUrl, - cachePvc: target.gitMirror.cachePvcName, - cachePvcStorage: target.gitMirror.cachePvcStorage, - cacheHostPath: target.gitMirror.cacheHostPath, - servicePort: target.gitMirror.servicePort, - deploymentReplicas: target.gitMirror.deploymentReplicas, - syncConfigMap: target.gitMirror.syncConfigMapName, - egressProxy: target.gitMirror.egressProxy, - effectiveEgressProxy: gitMirrorEffectiveEgressProxySummary(node, target), - githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport), - statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"], - }, - pipeline: target.tekton.pipelineName, - pipelineRunPrefix: target.tekton.pipelineRunPrefix, - serviceAccount: target.tekton.serviceAccountName, - gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), - runtimeObserverRbac: target.tekton.runtimeObserverRbac, - argoObserverRbac: target.tekton.argoObserverRbac, - toolsImage: target.tekton.toolsImage, - argoNamespace: target.argo.namespace, - argoApplication: target.argo.applicationName, - argoInstall: { - enabled: target.argo.install.enabled, - sourceKind: target.argo.install.sourceKind, - version: target.argo.install.version, - manifestUrl: target.argo.install.manifestUrl, - preloadImages: target.argo.install.preloadImages, - imageRewrites: target.argo.install.imageRewrites, - requiredCrds: target.argo.install.requiredCrds, - expectedDeployments: target.argo.install.expectedDeployments, - expectedStatefulSets: target.argo.install.expectedStatefulSets, - }, - registry: node.registry.endpoint, - imagePolicy: { - noPrivateInputImages: true, - buildInput: { sourceKind: target.tekton.toolsImage.sourceKind, context: target.tekton.toolsImage.context, dockerfile: target.tekton.toolsImage.dockerfile ?? null, dockerfileInline: target.tekton.toolsImage.dockerfileInline ?? null, composeFile: target.tekton.toolsImage.composeFile ?? null, publicBaseImages: target.tekton.toolsImage.publicBaseImages }, - outputImage: target.tekton.toolsImage.output, - }, - }; -} - -function k3sNodeConfigPlan(node: ControlPlaneNodeSpec): Record { - if (node.k3s === null) return { managed: false }; - const dropIn = k3sDropInContent(node.k3s); - return { - managed: true, - serviceName: node.k3s.serviceName, - dropInPath: node.k3s.dropInPath, - nodeStatusName: node.k3s.nodeStatusName, - desiredMaxPods: node.k3s.kubelet.maxPods, - dropInSha256: sha256Short(dropIn), - execStartPreCount: node.k3s.execStartPre.length, - serverArgCount: node.k3s.serverArgs.length, - }; -} - -function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string { - return [ - "# Managed by UniDesk. Source: config/hwlab-node-control-plane.yaml nodes..k3s", - "[Service]", - ...spec.execStartPre.map((command) => `ExecStartPre=${command.map(systemdExecArg).join(" ")}`), - "ExecStart=", - `ExecStart=${["/usr/local/bin/k3s", ...spec.serverArgs].map(systemdExecArg).join(" ")}`, - "", - ].join("\n"); -} - -function controlPlaneEgressProxySummary(proxy: ControlPlaneEgressProxySpec | null): Record | null { - if (proxy === null) return null; - return { - mode: proxy.mode, - clientName: proxy.clientName, - namespace: proxy.namespace, - serviceName: proxy.serviceName, - port: proxy.port, - sourceConfigRef: proxy.sourceConfigRef, - sourceType: proxy.sourceType, - sourceRef: proxy.sourceRef, - sourceKey: proxy.sourceKey, - sourceFingerprint: proxy.sourceFingerprint, - preferredOutbound: proxy.preferredOutbound, - valuesPrinted: false, - }; -} - -function gitMirrorEffectiveEgressProxySummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { - const config = target.gitMirror.egressProxy; - if (config === null || config.mode === "direct") { - return { - mode: "direct", - required: false, - transport: target.gitMirror.githubTransport.mode, - valuesPrinted: false, - }; - } - const proxy = node.egressProxy; - return { - mode: config.mode, - required: config.required, - transport: target.gitMirror.githubTransport.mode, - ready: proxy !== null, - nodeProxy: controlPlaneEgressProxySummary(proxy), - valuesPrinted: false, - }; -} - -function gitMirrorGithubTransportSummary(transport: ControlPlaneGitMirrorGithubTransportSpec): Record { - if (transport.mode === "ssh") { - return { - mode: "ssh", - privateKeySecretKey: transport.privateKeySecretKey, - privateKeySourceRef: transport.privateKeySourceRef, - privateKeySourceKey: transport.privateKeySourceKey, - privateKeySourceEncoding: transport.privateKeySourceEncoding, - knownHostsSecretKey: transport.knownHostsSecretKey, - knownHostsSourceRef: transport.knownHostsSourceRef, - knownHostsSourceKey: transport.knownHostsSourceKey, - knownHostsSourceEncoding: transport.knownHostsSourceEncoding, - valuesPrinted: false, - }; - } - return { - mode: "https", - username: transport.username, - tokenSecretName: transport.tokenSecretName, - tokenSecretKey: transport.tokenSecretKey, - tokenSourceRef: transport.tokenSourceRef, - tokenSourceKey: transport.tokenSourceKey, - valuesPrinted: false, - }; -} - -function tektonGitWorkspaceSecretSummary(target: ControlPlaneTargetSpec): Record { - const transport = target.gitMirror.githubTransport; - const secret = target.tekton.gitWorkspaceSecret; - return { - name: secret.name, - namespace: secret.namespace, - sourceRefFrom: secret.sourceRefFrom, - privateKeySecretKey: secret.privateKeySecretKey, - privateKeySourceRef: transport.mode === "ssh" ? transport.privateKeySourceRef : null, - privateKeySourceKey: transport.mode === "ssh" ? transport.privateKeySourceKey : null, - knownHostsSecretKey: secret.knownHostsSecretKey, - knownHostsSourceRef: transport.mode === "ssh" ? transport.knownHostsSourceRef : null, - knownHostsSourceKey: transport.mode === "ssh" ? transport.knownHostsSourceKey : null, - valuesPrinted: false, - }; -} - -function systemdExecArg(value: string): string { - if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value; - return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`; -} - -function statusScript(nodeSpec: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { - const requiredCrds = shellJsonArray(target.argo.install.requiredCrds); - const argoDeployments = shellJsonArray(target.argo.install.expectedDeployments); - const argoStatefulSets = shellJsonArray(target.argo.install.expectedStatefulSets); - const k3s = nodeSpec.k3s; - const k3sDropIn = k3s === null ? "" : k3sDropInContent(k3s); - const gitMirrorEgressProxyJson = JSON.stringify(gitMirrorEffectiveEgressProxySummary(nodeSpec, target)); - return ` -set +e -node=${shQuote(target.node)} -lane=${shQuote(target.lane)} -ci_ns=${shQuote(target.ciNamespace)} -runtime_ns=${shQuote(target.runtimeNamespace)} -gitmirror_ns=${shQuote(target.gitMirror.namespace)} -read_deploy=${shQuote(target.gitMirror.serviceReadName)} -write_deploy=${shQuote(target.gitMirror.serviceWriteName)} -read_svc=${shQuote(target.gitMirror.serviceReadName)} -write_svc=${shQuote(target.gitMirror.serviceWriteName)} -cache_pvc=${shQuote(target.gitMirror.cachePvcName)} -cache_host_path=${shQuote(target.gitMirror.cacheHostPath ?? "")} -github_transport_mode=${shQuote(target.gitMirror.githubTransport.mode)} -github_ssh_secret=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.secretName : "")} -github_ssh_private_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySecretKey : "")} -github_ssh_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")} -github_ssh_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")} -github_ssh_known_hosts_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSecretKey ?? "" : "")} -github_ssh_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")} -github_ssh_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")} -github_token_secret=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSecretName : "")} -github_token_key=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSecretKey : "")} -github_token_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSourceRef : "")} -github_token_source_key=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSourceKey : "")} -gitmirror_egress_proxy_json=${shQuote(gitMirrorEgressProxyJson)} -pipeline=${shQuote(target.tekton.pipelineName)} -service_account=${shQuote(target.tekton.serviceAccountName)} -runtime_observer_role=${shQuote(target.tekton.runtimeObserverRbac.roleName)} -runtime_observer_rolebinding=${shQuote(target.tekton.runtimeObserverRbac.roleBindingName)} -ci_git_secret=${shQuote(target.tekton.gitWorkspaceSecret.name)} -ci_git_private_key=${shQuote(target.tekton.gitWorkspaceSecret.privateKeySecretKey)} -ci_git_known_hosts_key=${shQuote(target.tekton.gitWorkspaceSecret.knownHostsSecretKey)} -ci_git_source_ref_from=${shQuote(target.tekton.gitWorkspaceSecret.sourceRefFrom)} -ci_git_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")} -ci_git_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")} -ci_git_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")} -ci_git_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")} -argo_ns=${shQuote(target.argo.namespace)} -argo_project=${shQuote(target.argo.projectName)} -argo_app=${shQuote(target.argo.applicationName)} -argo_observer_role=${shQuote(target.tekton.argoObserverRbac.roleName)} -argo_observer_rolebinding=${shQuote(target.tekton.argoObserverRbac.roleBindingName)} -registry=${shQuote(nodeSpec.registry.endpoint)} -tools_image=${shQuote(target.tekton.toolsImage.output)} -required_crds_json=${shQuote(requiredCrds)} -argo_deployments_json=${shQuote(argoDeployments)} -argo_statefulsets_json=${shQuote(argoStatefulSets)} -k3s_managed=${k3s === null ? "false" : "true"} -k3s_service=${shQuote(k3s?.serviceName ?? "")} -k3s_dropin=${shQuote(k3s?.dropInPath ?? "")} -k3s_node=${shQuote(k3s?.nodeStatusName ?? "")} -k3s_desired_max_pods=${shQuote(String(k3s?.kubelet.maxPods ?? ""))} -k3s_expected_sha=${shQuote(k3s === null ? "" : sha256Short(k3sDropIn))} -exists_ns() { kubectl get ns "$1" >/dev/null 2>&1 && printf true || printf false; } -exists_res() { kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 && printf true || printf false; } -deploy_ready() { desired=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } -sts_ready() { desired=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } -endpoint_ready() { endpoints=$(kubectl -n "$1" get endpoints "$2" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n "$endpoints" ] && printf true || printf false; } -can_i_runtime() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$runtime_ns" >/dev/null 2>&1 && printf true || printf false; } -can_i_argo() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$argo_ns" >/dev/null 2>&1 && printf true || printf false; } -runtime_observer_role_exists=$(exists_res "$runtime_ns" role "$runtime_observer_role") -runtime_observer_rolebinding_exists=$(exists_res "$runtime_ns" rolebinding "$runtime_observer_rolebinding") -runtime_observer_can_list_deployments=$(can_i_runtime list deployments.apps) -runtime_observer_can_list_statefulsets=$(can_i_runtime list statefulsets.apps) -runtime_observer_ready=false -if [ "$runtime_observer_role_exists" = true ] && [ "$runtime_observer_rolebinding_exists" = true ] && [ "$runtime_observer_can_list_deployments" = true ] && [ "$runtime_observer_can_list_statefulsets" = true ]; then runtime_observer_ready=true; fi -argo_observer_role_exists=$(exists_res "$argo_ns" role "$argo_observer_role") -argo_observer_rolebinding_exists=$(exists_res "$argo_ns" rolebinding "$argo_observer_rolebinding") -argo_observer_can_get_application=$(can_i_argo get applications.argoproj.io) -argo_observer_ready=false -if [ "$argo_observer_role_exists" = true ] && [ "$argo_observer_rolebinding_exists" = true ] && [ "$argo_observer_can_get_application" = true ]; then argo_observer_ready=true; fi -github_transport_json=$(python3 - "$github_transport_mode" "$gitmirror_ns" "$github_ssh_secret" "$github_ssh_private_key" "$github_ssh_private_source_ref" "$github_ssh_private_source_key" "$github_ssh_known_hosts_key" "$github_ssh_known_hosts_source_ref" "$github_ssh_known_hosts_source_key" "$github_token_secret" "$github_token_key" "$github_token_source_ref" "$github_token_source_key" <<'PY' -import hashlib, json, subprocess, sys -mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14] -def read_secret(name): - proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", name, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - if proc.returncode != 0: - return False, {}, {} - try: - obj = json.loads(proc.stdout) - except Exception: - obj = {} - return True, obj.get("data") or {}, obj.get("metadata", {}).get("annotations") or {} -def fingerprint(value): - return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None -if mode == "ssh": - exists, data, annotations = read_secret(ssh_secret) - private_encoded = data.get(ssh_private_key) if isinstance(data, dict) else None - known_hosts_encoded = data.get(ssh_known_hosts_key) if ssh_known_hosts_key and isinstance(data, dict) else None - private_present = isinstance(private_encoded, str) and len(private_encoded) > 0 - known_hosts_expected = bool(ssh_known_hosts_key) - known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0 - print(json.dumps({ - "mode": mode, - "required": True, - "ready": exists and private_present and (not known_hosts_expected or known_hosts_present), - "secretName": ssh_secret, - "privateKeySecretKey": ssh_private_key, - "privateKeySourceRef": ssh_private_source_ref, - "privateKeySourceKey": ssh_private_source_key, - "privateKeySecretExists": exists, - "privateKeyPresent": private_present, - "privateKeyBytes": len(private_encoded) if private_present else 0, - "privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded), - "knownHostsSecretKey": ssh_known_hosts_key or None, - "knownHostsSourceRef": ssh_known_hosts_source_ref or None, - "knownHostsSourceKey": ssh_known_hosts_source_key or None, - "knownHostsPresent": (known_hosts_present if known_hosts_expected else None), - "knownHostsBytes": (len(known_hosts_encoded) if known_hosts_present else 0) if known_hosts_expected else None, - "knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded), - "valuesPrinted": False, - })) - raise SystemExit(0) -if mode != "https": - print(json.dumps({"mode": mode, "required": True, "ready": False, "valuesPrinted": False})) - raise SystemExit(0) -exists, data, _ = read_secret(token_secret) -encoded = data.get(token_key) if isinstance(data, dict) else None -present = isinstance(encoded, str) and len(encoded) > 0 -print(json.dumps({"mode": mode, "required": True, "ready": exists and present, "tokenSecretName": token_secret, "tokenSecretKey": token_key, "tokenSourceRef": token_source_ref, "tokenSourceKey": token_source_key, "tokenSecretExists": exists, "tokenKeyPresent": present, "tokenKeyBytes": len(encoded) if present else 0, "tokenFingerprint": fingerprint(encoded), "valuesPrinted": False})) -PY -) -ci_git_workspace_json=$(python3 - "$ci_ns" "$ci_git_secret" "$ci_git_private_key" "$ci_git_known_hosts_key" "$ci_git_source_ref_from" "$ci_git_private_source_ref" "$ci_git_private_source_key" "$ci_git_known_hosts_source_ref" "$ci_git_known_hosts_source_key" <<'PY' -import hashlib, json, subprocess, sys -namespace, secret, private_key, known_hosts_key, source_ref_from, private_source_ref, private_source_key, known_hosts_source_ref, known_hosts_source_key = sys.argv[1:10] -proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", secret, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -exists = proc.returncode == 0 -obj = {} -if exists: - try: - obj = json.loads(proc.stdout) - except Exception: - obj = {} -data = obj.get("data") or {} -annotations = obj.get("metadata", {}).get("annotations") or {} -def fingerprint(value): - return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None -private_encoded = data.get(private_key) if isinstance(data, dict) else None -known_hosts_encoded = data.get(known_hosts_key) if isinstance(data, dict) else None -private_present = isinstance(private_encoded, str) and len(private_encoded) > 0 -known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0 -print(json.dumps({ - "required": True, - "ready": exists and private_present and known_hosts_present, - "namespace": namespace, - "secretName": secret, - "sourceRefFrom": source_ref_from, - "privateKeySecretKey": private_key, - "privateKeySourceRef": private_source_ref, - "privateKeySourceKey": private_source_key, - "privateKeySecretExists": exists, - "privateKeyPresent": private_present, - "privateKeyBytes": len(private_encoded) if private_present else 0, - "privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded), - "knownHostsSecretKey": known_hosts_key, - "knownHostsSourceRef": known_hosts_source_ref, - "knownHostsSourceKey": known_hosts_source_key, - "knownHostsPresent": known_hosts_present, - "knownHostsBytes": len(known_hosts_encoded) if known_hosts_present else 0, - "knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded), - "valuesPrinted": False, -})) -PY -) -registry_ready=false -if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi -tools_repo_tag=\${tools_image#\${registry}/} -tools_repo=\${tools_repo_tag%:*} -tools_tag=\${tools_repo_tag##*:} -tools_image_ready=false -if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi -cache_host_path_ready=false -if [ -n "$cache_host_path" ] && kubectl -n "$gitmirror_ns" exec deploy/"$read_deploy" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_ready=true; fi -k3s_fragment=$(python3 - "$k3s_managed" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_expected_sha" <<'PY' -import hashlib, json, re, subprocess, sys -managed = sys.argv[1] == "true" -service, dropin, node_name, desired_raw, expected_sha = sys.argv[2:7] -def run(args): - return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -def to_int(value): - try: - return int(value) - except Exception: - return None -if not managed: - print(json.dumps({"managed": False, "ready": True})) - raise SystemExit(0) -desired = to_int(desired_raw) -node_json = run(["kubectl", "get", "node", node_name, "-o", "json"]) -capacity = None -allocatable = None -node_ready = False -if node_json.returncode == 0: - data = json.loads(node_json.stdout) - capacity = to_int(data.get("status", {}).get("capacity", {}).get("pods")) - allocatable = to_int(data.get("status", {}).get("allocatable", {}).get("pods")) - for condition in data.get("status", {}).get("conditions", []): - if condition.get("type") == "Ready": - node_ready = condition.get("status") == "True" -unit = run(["systemctl", "cat", service]) -unit_text = unit.stdout if unit.returncode == 0 else "" -dropin_read = run(["cat", dropin]) -dropin_exists = dropin_read.returncode == 0 -dropin_text = dropin_read.stdout if dropin_exists else "" -dropin_sha = "sha256:" + hashlib.sha256(dropin_text.encode()).hexdigest() if dropin_exists else None -matches = re.findall(r"max-pods=([0-9]+)", unit_text + "\\n" + dropin_text) -configured = to_int(matches[-1]) if matches else None -dropin_matches = dropin_sha == expected_sha -ready = dropin_matches and capacity == desired and allocatable == desired -source = "managed-dropin" if dropin_matches else ("systemd-or-config" if configured is not None else "kubelet-default") -print(json.dumps({ - "managed": True, - "ready": ready, - "serviceName": service, - "dropInPath": dropin, - "dropInExists": dropin_exists, - "dropInSha256": dropin_sha, - "expectedDropInSha256": expected_sha, - "dropInMatches": dropin_matches, - "configuredMaxPods": configured, - "desiredMaxPods": desired, - "liveNodeName": node_name, - "liveCapacityPods": capacity, - "liveAllocatablePods": allocatable, - "nodeReady": node_ready, - "restartRequired": not ready, - "source": source, - "unitReadable": unit.returncode == 0, -})) -PY -) -python3 - "$required_crds_json" "$argo_deployments_json" "$argo_statefulsets_json" <<'PY' >/tmp/hwlab-node-status-fragments.json -import json, subprocess, sys -required_crds=json.loads(sys.argv[1]) -deployments=json.loads(sys.argv[2]) -statefulsets=json.loads(sys.argv[3]) -ns="${target.argo.namespace}" -def run(args): - return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -def exists(args): - return run(args).returncode == 0 -def ready(kind, name): - data = run(["kubectl", "-n", ns, "get", kind, name, "-o", "json"]) - if data.returncode != 0: - return {"name": name, "exists": False, "ready": False, "desired": None, "readyReplicas": None} - obj=json.loads(data.stdout) - desired=int(obj.get("spec", {}).get("replicas") or 0) - ready_replicas=int(obj.get("status", {}).get("readyReplicas") or 0) - return {"name": name, "exists": True, "ready": desired > 0 and ready_replicas == desired, "desired": desired, "readyReplicas": ready_replicas} -crds=[{"name": name, "exists": exists(["kubectl", "get", "crd", name])} for name in required_crds] -deploy=[ready("deployment", name) for name in deployments] -sts=[ready("statefulset", name) for name in statefulsets] -print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crdsReady": all(item["exists"] for item in crds), "deploymentsReady": all(item["ready"] for item in deploy) if deploy else True, "statefulSetsReady": all(item["ready"] for item in sts) if sts else True})) -PY -argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') -cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"argoObserverRbac":{"roleName":"$argo_observer_role","roleExists":$argo_observer_role_exists,"roleBindingName":"$argo_observer_rolebinding","roleBindingExists":$argo_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canGetApplication":$argo_observer_can_get_application,"ready":$argo_observer_ready},"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} -JSON -`; -} - -function applyScript(yaml: string, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { - const encoded = Buffer.from(yaml, "utf8").toString("base64"); - return ` -set +e -manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml) -printf %s ${shQuote(encoded)} | base64 -d >"$manifest" -field_manager=${shQuote(controlPlaneFieldManager(target))} -kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err -kubectl_rc=$? -${k3sApplyScriptFragment(node.k3s, target)} -python3 - "$kubectl_rc" "$k3s_report_file" <<'PY' -import json, pathlib, sys -k3s_report = {} -try: - k3s_report = json.loads(pathlib.Path(sys.argv[2]).read_text(errors='replace')) -except Exception as exc: - k3s_report = {"managed": None, "ok": False, "parseError": str(exc)} -out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else '' -err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else '' -print(json.dumps({'k3sNodeConfig': k3s_report, 'kubernetesApply': {'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}}, ensure_ascii=False)) -PY -rm -f "$manifest" -if [ "$kubectl_rc" != 0 ]; then exit "$kubectl_rc"; fi -exit "$k3s_rc" -`; -} - -function controlPlaneFieldManager(target: ControlPlaneTargetSpec): string { - return `unidesk-hwlab-${target.node.toLowerCase()}-${target.lane}-control-plane`; -} - -function k3sApplyScriptFragment(spec: ControlPlaneK3sNodeSpec | null, target: ControlPlaneTargetSpec): string { - if (spec === null) { - return ` -k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json) -printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$k3s_report_file" -k3s_rc=0 -`; - } - const content = k3sDropInContent(spec); - const encoded = Buffer.from(content, "utf8").toString("base64"); - return ` -k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json) -k3s_service=${shQuote(spec.serviceName)} -k3s_dropin=${shQuote(spec.dropInPath)} -k3s_node=${shQuote(spec.nodeStatusName)} -k3s_namespace=${shQuote(target.ciNamespace)} -k3s_image=${shQuote(target.tekton.toolsImage.output)} -k3s_desired_max_pods=${shQuote(String(spec.kubelet.maxPods))} -k3s_expected_sha=${shQuote(sha256Short(content))} -k3s_before_capacity=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) -k3s_before_allocatable=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) -capacity_restart=false -if [ "$k3s_before_capacity" != "$k3s_desired_max_pods" ] || [ "$k3s_before_allocatable" != "$k3s_desired_max_pods" ]; then capacity_restart=true; fi -k3s_current_dropin_sha= -if [ -f "$k3s_dropin" ]; then k3s_current_dropin_sha=$(sha256sum "$k3s_dropin" | awk '{print "sha256:"$1}'); fi -if [ "$k3s_current_dropin_sha" = "$k3s_expected_sha" ] && [ "$capacity_restart" = false ]; then - python3 - "$k3s_current_dropin_sha" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_before_capacity" "$k3s_before_allocatable" <<'PY' >"$k3s_report_file" -import json, sys -dropin_sha, expected_sha, service, dropin, node_name, desired, before_capacity, before_allocatable = sys.argv[1:9] -print(json.dumps({ - "managed": True, - "ok": True, - "mutation": False, - "applyMode": "noop", - "completionPending": False, - "serviceName": service, - "dropInPath": dropin, - "dropInSha256": dropin_sha, - "expectedDropInSha256": expected_sha, - "dropInMatches": dropin_sha == expected_sha, - "nodeName": node_name, - "desiredMaxPods": int(desired), - "beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None, - "beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None, -}, ensure_ascii=False)) -PY - k3s_rc=0 -else -k3s_job="hwlab-node-k3s-config-$(date +%s)" -k3s_job_manifest=$(mktemp /tmp/hwlab-node-k3s-job.XXXXXX.json) -k3s_host_script=$(mktemp /tmp/hwlab-node-k3s-host.XXXXXX.sh) -k3s_job_apply_stdout=/tmp/hwlab-node-k3s-job-apply.out -k3s_job_apply_stderr=/tmp/hwlab-node-k3s-job-apply.err -k3s_docker_stdout=/tmp/hwlab-node-k3s-docker.out -k3s_docker_stderr=/tmp/hwlab-node-k3s-docker.err -k3s_host_report="/tmp/$k3s_job-report.json" -rm -f "$k3s_host_report" -python3 - "$k3s_job_manifest" "$k3s_host_script" "$k3s_job" "$k3s_namespace" "$k3s_image" "$k3s_dropin" ${shQuote(encoded)} "$k3s_service" "$k3s_desired_max_pods" "$k3s_expected_sha" "$capacity_restart" "$k3s_host_report" <<'PY' -import json, os, shlex, sys -manifest_path, host_script_path, job, namespace, image, dropin, encoded, service, desired, expected_sha, capacity_restart, report_path = sys.argv[1:13] -script = f"""#!/bin/sh -set -eu -expected=/tmp/unidesk-k3s-dropin.conf -printf %s {shlex.quote(encoded)} | base64 -d > "$expected" -host_dropin=/host{shlex.quote(dropin)} -host_report=/host{shlex.quote(report_path)} -mkdir -p "$(dirname "$host_dropin")" -before_sha= -if [ -f "$host_dropin" ]; then before_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi -changed=false -if ! cmp -s "$expected" "$host_dropin" 2>/dev/null; then - cp "$expected" "$host_dropin" - chown 0:0 "$host_dropin" 2>/dev/null || true - chmod 0644 "$host_dropin" - changed=true -fi -nsenter_path=$(command -v nsenter || true) -host_systemctl() {{ - if command -v chroot >/dev/null 2>&1 && [ -x /host/usr/bin/systemctl ]; then - chroot /host /usr/bin/systemctl "$@" - return $? - fi - if [ -n "$nsenter_path" ]; then - "$nsenter_path" -t 1 -m -u -i -n -p -- /usr/bin/systemctl "$@" - return $? - fi - return 127 -}} -daemon_reload_rc=0 -restart_rc=0 -restarted=false -if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then - host_systemctl daemon-reload || daemon_reload_rc=$? - if [ "$changed" = true ] || [ {shlex.quote(capacity_restart)} = true ]; then - restarted=true - host_systemctl restart {shlex.quote(service)} || restart_rc=$? - fi -else - daemon_reload_rc=127 - restart_rc=127 -fi -after_sha= -if [ -f "$host_dropin" ]; then after_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi -service_active=unknown -if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then service_active=$(host_systemctl is-active {shlex.quote(service)} 2>/dev/null || true); fi -python3 - "$changed" "$restarted" "$daemon_reload_rc" "$restart_rc" "$before_sha" "$after_sha" "$service_active" "$nsenter_path" <<'REPORT' >"$host_report" -import json, sys -changed, restarted = sys.argv[1] == "true", sys.argv[2] == "true" -daemon_reload_rc, restart_rc = int(sys.argv[3] or "0"), int(sys.argv[4] or "0") -print(json.dumps({{ - "jobChanged": changed, - "jobRestarted": restarted, - "daemonReloadExitCode": daemon_reload_rc, - "restartExitCode": restart_rc, - "beforeDropInSha256": sys.argv[5] or None, - "dropInSha256": sys.argv[6] or None, - "expectedDropInSha256": {json.dumps(expected_sha)}, - "dropInMatches": sys.argv[6] == {json.dumps(expected_sha)}, - "serviceActiveText": sys.argv[7] or None, - "nsenterPresent": bool(sys.argv[8]), -}})) -REPORT -chmod 0644 "$host_report" 2>/dev/null || true -cat "$host_report" -""" -with open(host_script_path, "w", encoding="utf-8") as handle: - handle.write(script) -os.chmod(host_script_path, 0o755) -manifest = { - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": {"name": job, "namespace": namespace, "labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}}, - "spec": { - "backoffLimit": 0, - "ttlSecondsAfterFinished": 300, - "template": { - "metadata": {"labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}}, - "spec": { - "restartPolicy": "Never", - "hostPID": True, - "hostNetwork": True, - "containers": [{ - "name": "apply-k3s-node-config", - "image": image, - "imagePullPolicy": "IfNotPresent", - "securityContext": {"privileged": True}, - "command": ["/bin/sh", "-lc", script], - "volumeMounts": [{"name": "host-root", "mountPath": "/host"}], - }], - "volumes": [{"name": "host-root", "hostPath": {"path": "/", "type": "Directory"}}], - }, - }, - }, -} -with open(manifest_path, "w", encoding="utf-8") as handle: - json.dump(manifest, handle) -PY -k3s_render_rc=$? -if [ "$k3s_render_rc" != 0 ]; then - python3 - "$k3s_render_rc" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" <<'PY' >"$k3s_report_file" -import json, sys -render_rc = int(sys.argv[1] or "1") -expected_sha, service, dropin, node_name, desired = sys.argv[2:7] -print(json.dumps({ - "managed": True, - "ok": False, - "mutation": False, - "renderExitCode": render_rc, - "serviceName": service, - "dropInPath": dropin, - "expectedDropInSha256": expected_sha, - "nodeName": node_name, - "desiredMaxPods": int(desired), -}, ensure_ascii=False)) -PY - k3s_rc=$k3s_render_rc -else -kubectl apply -f "$k3s_job_manifest" >"$k3s_job_apply_stdout" 2>"$k3s_job_apply_stderr" -k3s_job_apply_rc=$? -k3s_apply_mode=kubernetes-job -k3s_docker_rc=127 -if [ "$k3s_job_apply_rc" != 0 ] && command -v docker >/dev/null 2>&1; then - k3s_apply_mode=docker-host-fallback - docker run --rm --privileged --pid=host --network=host -v /:/host --entrypoint /bin/sh "$k3s_image" "/host$k3s_host_script" >"$k3s_docker_stdout" 2>"$k3s_docker_stderr" - k3s_docker_rc=$? -fi -k3s_submit_rc=$k3s_job_apply_rc -if [ "$k3s_job_apply_rc" != 0 ] && [ "$k3s_docker_rc" = 0 ]; then k3s_submit_rc=0; fi -python3 - "$k3s_submit_rc" "$k3s_job_apply_rc" "$k3s_docker_rc" "$k3s_apply_mode" "$k3s_before_capacity" "$k3s_before_allocatable" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_job" "$k3s_namespace" "$k3s_host_report" "$k3s_job_apply_stdout" "$k3s_job_apply_stderr" "$k3s_docker_stdout" "$k3s_docker_stderr" <<'PY' >"$k3s_report_file" -import json, pathlib, sys -submit_rc, job_apply_rc, docker_rc = [int(value or "0") for value in sys.argv[1:4]] -apply_mode = sys.argv[4] -before_capacity, before_allocatable = sys.argv[5:7] -expected_sha, service, dropin, node_name, desired, job_name, namespace, host_report = sys.argv[7:15] -def read(path): - return pathlib.Path(path).read_text(errors='replace') if pathlib.Path(path).exists() else '' -try: - host_report_data = json.loads(read(host_report) or "{}") -except Exception: - host_report_data = {} -apply_ok = submit_rc == 0 -print(json.dumps({ - "managed": True, - "ok": apply_ok, - "mutation": apply_ok, - "completionPending": apply_ok and apply_mode == "kubernetes-job", - "applyMode": apply_mode, - "jobName": job_name, - "namespace": namespace, - "jobApplyExitCode": job_apply_rc, - "dockerFallbackExitCode": docker_rc, - "serviceName": service, - "dropInPath": dropin, - "dropInSha256": host_report_data.get("dropInSha256"), - "expectedDropInSha256": expected_sha, - "dropInMatches": host_report_data.get("dropInSha256") == expected_sha if host_report_data else None, - "daemonReloadExitCode": host_report_data.get("daemonReloadExitCode"), - "restartExitCode": host_report_data.get("restartExitCode"), - "serviceActive": host_report_data.get("serviceActiveText") == "active" if host_report_data else None, - "nodeName": node_name, - "desiredMaxPods": int(desired), - "beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None, - "beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None, - "hostReportPath": host_report, - "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra status --node {node_name.upper()} --lane ${target.lane}", - "jobCompletionCommand": f"kubectl -n {namespace} wait --for=condition=complete job/{job_name} --timeout=120s", - "jobLogsCommand": f"kubectl -n {namespace} logs job/{job_name} --tail=120", - "jobApplyStdoutPreview": read(sys.argv[15])[-1000:], - "jobApplyStderrPreview": read(sys.argv[16])[-1000:], - "dockerStdoutPreview": read(sys.argv[17])[-1000:], - "dockerStderrPreview": read(sys.argv[18])[-1000:], -}, ensure_ascii=False)) -PY -k3s_rc=$k3s_submit_rc -fi -rm -f "$k3s_job_manifest" "$k3s_host_script" -fi -`; -} - -function toolsImageStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, timeoutSeconds: number): { - registryReady: boolean; - toolsImageReady: boolean; - result: Record; -} { - const result = runTransK3s(node.kubeRoute, registryStatusScript(node.registry.endpoint, target.tekton.toolsImage.output), timeoutSeconds); - const parsed = parseRemoteJson(result.stdout); - const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; - return { - registryReady: boolField(status, "registryReady"), - toolsImageReady: boolField(status, "toolsImageReady"), - result: { - status, - command: compactCommandResult(result), - }, - }; -} - -function applyNext(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, imageStatus: { registryReady: boolean; toolsImageReady: boolean }): Record { - if (!imageStatus.registryReady) { - return { - status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, - blockedBy: "node-local-registry-not-ready", - }; - } - if (!imageStatus.toolsImageReady) { - return { - status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, - blockedBy: "tools-image-missing", - applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, - buildToolsImage: "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。", - }; - } - return { apply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm` }; -} - -function statusNext( - node: ControlPlaneNodeSpec, - target: ControlPlaneTargetSpec, - registry: Record, - gitMirror: Record, - argo: Record, - ciNamespace: Record, - runtimeNamespace: Record, - k3sNodeConfig: Record, -): Record { - const bootstrapMissing = !boolField(ciNamespace, "exists") - || !boolField(record(ciNamespace.gitWorkspaceSecret), "ready") - || !boolField(gitMirror, "namespaceExists") - || !boolField(gitMirror, "readServiceExists") - || !boolField(gitMirror, "writeServiceExists") - || (!boolField(gitMirror, "cachePvcExists") && !boolField(gitMirror, "cacheHostPathReady")); - const blockers: string[] = []; - if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) blockers.push("k3s-node-config-not-applied"); - if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); - if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); - if (!boolField(record(ciNamespace.gitWorkspaceSecret), "ready")) blockers.push("ci-git-workspace-secret-not-ready"); - if (!boolField(runtimeNamespace, "exists")) blockers.push("runtime-namespace-missing"); - if (!boolField(record(runtimeNamespace.runtimeObserverRbac), "ready")) blockers.push("runtime-observer-rbac-not-ready"); - if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); - const gitMirrorGithubTransport = record(gitMirror.githubTransport); - if (gitMirrorGithubTransport.required === true && !boolField(gitMirrorGithubTransport, "ready")) blockers.push("git-mirror-github-token-secret-not-ready"); - const argoInstall = record(argo.install); - if (!boolField(argo, "installed")) blockers.push("argocd-not-installed"); - else if (!boolField(argoInstall, "crdsReady")) blockers.push("argocd-crds-not-ready"); - else if (!boolField(argoInstall, "deploymentsReady")) blockers.push("argocd-deployments-not-ready"); - else if (!boolField(argoInstall, "statefulSetsReady")) blockers.push("argocd-statefulsets-not-ready"); - else if (!boolField(argo, "projectExists")) blockers.push("argocd-project-missing"); - else if (!boolField(argo, "applicationExists")) blockers.push("argocd-application-missing"); - if (!boolField(record(argo.argoObserverRbac), "ready")) blockers.push("argocd-observer-rbac-not-ready"); - const next: Record = { - status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, - dryRun: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --dry-run`, - }; - if (blockers.length > 0) { - next.blockedBy = blockers[0]; - next.blockers = blockers; - } - if (!boolField(registry, "toolsImageReady")) { - next.buildToolsImage = "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。"; - } - if (!boolField(argo, "installed")) { - next.installArgo = "准备受控 D601 Argo CD 安装入口后再进入 runtime rollout。"; - } - if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) { - next.applyK3sNodeConfig = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; - } - if (bootstrapMissing) next.applyBootstrap = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; - else next.reapplyBootstrap = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; - return next; -} - -function registryStatusScript(registryEndpoint: string, toolsImage: string): string { - return ` -set +e -registry=${shQuote(registryEndpoint)} -tools_image=${shQuote(toolsImage)} -registry_ready=false -if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi -tools_repo_tag=\${tools_image#\${registry}/} -tools_repo=\${tools_repo_tag%:*} -tools_tag=\${tools_repo_tag##*:} -tools_image_ready=false -if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi -cat < ["--build-arg", `${key}=${value}`]); - const proxyArgs = node.egressProxy === null - ? [] - : ["--build-arg", "HTTP_PROXY", "--build-arg", "HTTPS_PROXY", "--build-arg", "ALL_PROXY", "--build-arg", "NO_PROXY", "--build-arg", "http_proxy", "--build-arg", "https_proxy", "--build-arg", "all_proxy", "--build-arg", "no_proxy"]; - const networkArgs = target.tekton.toolsImage.buildNetwork === null ? [] : ["--network", target.tekton.toolsImage.buildNetwork]; - const dockerBuildArgs = [...networkArgs, ...buildArgs, ...proxyArgs, "-f", "$dockerfile", "-t", "$image", "$context_dir"].join(" "); - return ` -set -eu -state_dir=${shQuote(stateDir)} -mkdir -p "$state_dir" -if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then - printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" - exit 0 -fi -cat >"$state_dir/job.sh" <<'JOB' -#!/bin/sh -set -eu -state_dir=${shQuote(stateDir)} -image=${shQuote(target.tekton.toolsImage.output)} -context_dir="$state_dir/context" -dockerfile="$state_dir/${target.tekton.toolsImage.dockerfileInline?.filename ?? "Dockerfile"}" -log="$state_dir/job.log" -status="$state_dir/status.json" -write_status() { - state="$1"; shift - message="$1"; shift || true - python3 - "$status" "$state" "$message" "$image" <<'PY' -import json, pathlib, sys, time -path=pathlib.Path(sys.argv[1]) -payload={"state":sys.argv[2],"message":sys.argv[3],"image":sys.argv[4],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} -path.write_text(json.dumps(payload, ensure_ascii=False) + "\\n") -PY -} -run_job() { - write_status running starting - rm -rf "$context_dir" - mkdir -p "$context_dir" - printf %s ${shQuote(dockerfileEncoded)} | base64 -d >"$dockerfile" -${proxyExportBlock(node)} - docker build ${dockerBuildArgs} || return "$?" - docker run --rm "$image" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V' || return "$?" - docker push "$image" || return "$?" - image_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)" - digest="$(docker image inspect "$image" --format '{{join .RepoDigests ","}}' 2>/dev/null || true)" - python3 - "$status" "$image" "$image_id" "$digest" <<'PY' -import json, pathlib, sys, time -path=pathlib.Path(sys.argv[1]) -path.write_text(json.dumps({"state":"succeeded","message":"image-built-and-pushed","image":sys.argv[2],"imageId":sys.argv[3] or None,"repoDigests":[item for item in sys.argv[4].split(",") if item],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") -PY -} -run_job >>"$log" 2>&1 || { - rc=$? - write_status failed "exit-$rc" - exit "$rc" -} -JOB -chmod +x "$state_dir/job.sh" -: >"$state_dir/job.log" -nohup "$state_dir/job.sh" >/dev/null 2>&1 & -pid=$! -printf '%s' "$pid" >"$state_dir/pid" -printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} -`; -} - -function argoApplyStartScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, desiredYaml: string): string { - const stateDir = remoteJobStateDir(target, "argo"); - const desiredEncoded = Buffer.from(desiredYaml, "utf8").toString("base64"); - const rewritesEncoded = Buffer.from(JSON.stringify(target.argo.install.imageRewrites), "utf8").toString("base64"); - const preloadEncoded = Buffer.from(JSON.stringify(target.argo.install.preloadImages), "utf8").toString("base64"); - return ` -set -eu -state_dir=${shQuote(stateDir)} -mkdir -p "$state_dir" -if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then - printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" - exit 0 -fi -cat >"$state_dir/job.sh" <<'JOB' -#!/bin/sh -set -eu -state_dir=${shQuote(stateDir)} -namespace=${shQuote(target.argo.namespace)} -manifest_url=${shQuote(target.argo.install.manifestUrl)} -field_manager=${shQuote(target.argo.install.fieldManager)} -readiness_timeout=${shQuote(String(target.argo.install.readinessTimeoutSeconds))} -log="$state_dir/job.log" -status="$state_dir/status.json" -install_yaml="$state_dir/install.yaml" -rendered_yaml="$state_dir/install.rendered.yaml" -desired_yaml="$state_dir/desired.yaml" -write_status() { - state="$1"; shift - message="$1"; shift || true - python3 - "$status" "$state" "$message" <<'PY' -import json, pathlib, sys, time -path=pathlib.Path(sys.argv[1]) -path.write_text(json.dumps({"state":sys.argv[2],"message":sys.argv[3],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") -PY -} -{ - write_status running starting -${proxyExportBlock(node)} - printf %s ${shQuote(desiredEncoded)} | base64 -d >"$desired_yaml" - printf %s ${shQuote(rewritesEncoded)} | base64 -d >"$state_dir/image-rewrites.json" - printf %s ${shQuote(preloadEncoded)} | base64 -d >"$state_dir/preload-images.json" - kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply --server-side --field-manager="$field_manager" -f - || exit "$?" - python3 - "$state_dir/preload-images.json" "$state_dir/image-rewrites.json" <<'PY' >"$state_dir/pull-images.sh" -import json, pathlib, shlex, sys -preload=json.loads(pathlib.Path(sys.argv[1]).read_text()) -rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) -print("#!/bin/sh") -print("set -eu") -seen=set() -for item in rewrites: - pull=item["pullImage"] - target=item["target"] - if target in seen: - continue - seen.add(target) - print("docker pull " + shlex.quote(pull)) - print("docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) - print("docker push " + shlex.quote(target)) -for image in preload: - if image not in seen and image.startswith("127.0.0.1:5000/"): - print("docker image inspect " + shlex.quote(image) + " >/dev/null") -PY - chmod +x "$state_dir/pull-images.sh" - "$state_dir/pull-images.sh" || exit "$?" - curl -fsSL --max-time 60 "$manifest_url" >"$install_yaml" || exit "$?" - python3 - "$install_yaml" "$state_dir/image-rewrites.json" "$rendered_yaml" ${shQuote(target.argo.install.imagePullPolicy)} <<'PY' -import json, pathlib, sys -text=pathlib.Path(sys.argv[1]).read_text() -rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) -for item in rewrites: - text=text.replace(item["source"], item["target"]) -policy=sys.argv[4] -text=text.replace("imagePullPolicy: Always", "imagePullPolicy: " + policy) -pathlib.Path(sys.argv[3]).write_text(text) -PY - kubectl apply --server-side --field-manager="$field_manager" -n "$namespace" -f "$rendered_yaml" || exit "$?" - deadline=$(( $(date +%s) + readiness_timeout )) - while [ "$(date +%s)" -lt "$deadline" ]; do - kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && break - sleep 5 - done - kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null || exit "$?" - kubectl apply --server-side --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" - write_status succeeded argocd-install-applied -} >>"$log" 2>&1 || { - rc=$? - write_status failed "exit-$rc" - exit "$rc" -} -JOB -chmod +x "$state_dir/job.sh" -: >"$state_dir/job.log" -nohup "$state_dir/job.sh" >/dev/null 2>&1 & -pid=$! -printf '%s' "$pid" >"$state_dir/pid" -printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra argo status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} -`; -} - -function ciBuildBenchmarkStartScript( - target: ControlPlaneTargetSpec, - profile: CiBuildBenchmarkProfileSpec, - manifest: Record, - pipelineName: string, - pipelineRun: string, - sourceCommit: string, - catalogPath: string, -): string { - const stateDir = ciBuildBenchmarkStateDir(target, profile.profile); - const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); - return ` -set -eu -state_dir=${shQuote(stateDir)} -status_file="$state_dir/status.json" -ns=${shQuote(target.ciNamespace)} -profile=${shQuote(profile.profile)} -pipeline=${shQuote(pipelineName)} -pipeline_run=${shQuote(pipelineRun)} -source_commit=${shQuote(sourceCommit)} -catalog_path=${shQuote(catalogPath)} -mkdir -p "$state_dir" -previous_run= -if [ -s "$status_file" ]; then - previous_run=$(python3 - "$status_file" <<'PY' || true -import json, sys -try: - data=json.load(open(sys.argv[1], encoding="utf-8")) - print(data.get("pipelineRun") or "") -except Exception: - print("") -PY -) -fi -if [ -n "$previous_run" ]; then - previous_status=$(kubectl -n "$ns" get pipelinerun "$previous_run" -o 'jsonpath={.status.conditions[?(@.type=="Succeeded")].status}' 2>/dev/null || true) - if [ -n "$previous_status" ] && [ "$previous_status" != "True" ] && [ "$previous_status" != "False" ]; then - python3 - "$state_dir" "$previous_run" "$profile" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY' -import json, sys -state_dir, previous_run, profile, node, lane = sys.argv[1:6] -print(json.dumps({ - "started": False, - "state": "already-running", - "pipelineRun": previous_run, - "stateDir": state_dir, - "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}", - "logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}", -}, ensure_ascii=False)) -PY - exit 0 - fi -fi -manifest_path="$state_dir/$pipeline_run.json" -printf '%s' ${shQuote(manifestB64)} | base64 -d >"$manifest_path" -set +e -pipeline_check=$(kubectl -n "$ns" get pipeline "$pipeline" -o name 2>&1) -pipeline_check_rc=$? -create_output= -create_rc=0 -if [ "$pipeline_check_rc" = 0 ]; then - create_output=$(kubectl create -f "$manifest_path" 2>&1) - create_rc=$? -else - create_output="$pipeline_check" - create_rc="$pipeline_check_rc" -fi -printf '%s\\n' "$create_output" >"$state_dir/create.log" -python3 - "$status_file" "$state_dir" "$pipeline_run" "$source_commit" "$profile" "$catalog_path" "$create_rc" "$create_output" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY' -import datetime, json, sys -status_file, state_dir, pipeline_run, source_commit, profile, catalog_path, rc_raw, output, node, lane = sys.argv[1:11] -rc=int(rc_raw or "0") -payload={ - "started": rc == 0, - "state": "started" if rc == 0 else "failed", - "pipelineRun": pipeline_run, - "sourceCommit": source_commit, - "profile": profile, - "catalogPath": catalog_path, - "stateDir": state_dir, - "createdAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "exitCode": rc, - "createOutputTail": output[-2000:], - "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}", - "logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}", -} -open(status_file, "w", encoding="utf-8").write(json.dumps(payload, ensure_ascii=False)) -print(json.dumps(payload, ensure_ascii=False)) -PY -exit "$create_rc" -`; -} - -function ciBuildBenchmarkStatusScript(target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec, tailLines: number, includeLogs: boolean): string { - const stateDir = ciBuildBenchmarkStateDir(target, profile.profile); - return ` -set +e -state_dir=${shQuote(stateDir)} -status_file="$state_dir/status.json" -ns=${shQuote(target.ciNamespace)} -profile=${shQuote(profile.profile)} -tail_lines=${shQuote(String(tailLines))} -include_logs=${includeLogs ? "true" : "false"} -tmp_dir=$(mktemp -d) -pipeline_run= -if [ -s "$status_file" ]; then - pipeline_run=$(python3 - "$status_file" <<'PY' || true -import json, sys -try: - data=json.load(open(sys.argv[1], encoding="utf-8")) - print(data.get("pipelineRun") or "") -except Exception: - print("") -PY -) -fi -if [ -z "$pipeline_run" ]; then - pipeline_run=$(kubectl -n "$ns" get pipelinerun -l "unidesk.ai/benchmark=ci-build,unidesk.ai/benchmark-profile=$profile" -o 'jsonpath={range .items[*]}{.metadata.creationTimestamp}{" "}{.metadata.name}{"\\n"}{end}' 2>/dev/null | sort | tail -n 1 | awk '{print $2}') -fi -if [ -n "$pipeline_run" ]; then - kubectl -n "$ns" get pipelinerun "$pipeline_run" -o json >"$tmp_dir/pipelinerun.json" 2>"$tmp_dir/pipelinerun.err" - kubectl -n "$ns" get taskrun -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/taskruns.json" 2>"$tmp_dir/taskruns.err" - kubectl -n "$ns" get pod -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/pods.json" 2>"$tmp_dir/pods.err" - if [ "$include_logs" = true ]; then - kubectl -n "$ns" logs -l "tekton.dev/pipelineRun=$pipeline_run" --all-containers --tail="$tail_lines" --prefix=true >"$tmp_dir/logs.txt" 2>"$tmp_dir/logs.err" || true - fi -fi -python3 - "$state_dir" "$status_file" "$tmp_dir" "$pipeline_run" "$include_logs" "$tail_lines" <<'PY' -import json, pathlib, sys -state_dir=pathlib.Path(sys.argv[1]) -status_path=pathlib.Path(sys.argv[2]) -tmp_dir=pathlib.Path(sys.argv[3]) -pipeline_run=sys.argv[4] -include_logs=sys.argv[5] == "true" -tail_lines=int(sys.argv[6]) -log_tail_limit=min(6000, max(2000, tail_lines * 80)) - -def read_json(path): - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return None - -def read_text(path, limit=4000): - try: - return path.read_text(encoding="utf-8", errors="replace")[-limit:] - except Exception: - return "" - -def succeeded_condition(obj): - for cond in obj.get("status", {}).get("conditions", []) or []: - if cond.get("type") == "Succeeded": - return cond - return {} - -status=None -if status_path.exists(): - status=read_json(status_path) -pr=read_json(tmp_dir / "pipelinerun.json") if pipeline_run else None -trs=read_json(tmp_dir / "taskruns.json") if pipeline_run else None -pods=read_json(tmp_dir / "pods.json") if pipeline_run else None -pr_cond=succeeded_condition(pr or {}) -task_runs=[] -for item in (trs or {}).get("items", []) or []: - cond=succeeded_condition(item) - labels=item.get("metadata", {}).get("labels", {}) or {} - task_runs.append({ - "name": item.get("metadata", {}).get("name"), - "pipelineTask": labels.get("tekton.dev/pipelineTask"), - "status": cond.get("status"), - "reason": cond.get("reason"), - "message": cond.get("message"), - "startTime": item.get("status", {}).get("startTime"), - "completionTime": item.get("status", {}).get("completionTime"), - "podName": item.get("status", {}).get("podName"), - }) -pod_rows=[] -for item in (pods or {}).get("items", []) or []: - phase=item.get("status", {}).get("phase") - pod_rows.append({ - "name": item.get("metadata", {}).get("name"), - "phase": phase, - "startTime": item.get("status", {}).get("startTime"), - }) -state="not-started" -if pr: - status_value=pr_cond.get("status") - if status_value == "True": - state="succeeded" - elif status_value == "False": - state="failed" - elif status_value: - state="running" - else: - state="pending" -elif pipeline_run: - state="missing" -payload={ - "stateDir": str(state_dir), - "status": status, - "pipelineRunName": pipeline_run or None, - "state": state, - "pipelineRun": None if not pr else { - "name": pr.get("metadata", {}).get("name"), - "status": pr_cond.get("status"), - "reason": pr_cond.get("reason"), - "message": pr_cond.get("message"), - "createdAt": pr.get("metadata", {}).get("creationTimestamp"), - "startTime": pr.get("status", {}).get("startTime"), - "completionTime": pr.get("status", {}).get("completionTime"), - "sourceCommit": (pr.get("metadata", {}).get("labels", {}) or {}).get("hwlab.pikastech.local/source-commit"), - "catalogPath": (pr.get("metadata", {}).get("annotations", {}) or {}).get("unidesk.ai/catalog-path"), - }, - "taskRuns": task_runs, - "pods": pod_rows, - "errors": { - "pipelinerun": read_text(tmp_dir / "pipelinerun.err"), - "taskruns": read_text(tmp_dir / "taskruns.err"), - "pods": read_text(tmp_dir / "pods.err"), - "logs": read_text(tmp_dir / "logs.err") if include_logs else "", - }, - "logTail": read_text(tmp_dir / "logs.txt", log_tail_limit) if include_logs else "", -} -print(json.dumps(payload, ensure_ascii=False)) -PY -rm -rf "$tmp_dir" -`; -} - -function remoteJobStatusScript(target: ControlPlaneTargetSpec, name: "tools-image" | "argo", tailLines: number): string { - const stateDir = remoteJobStateDir(target, name); - return ` -set +e -state_dir=${shQuote(stateDir)} -status_file="$state_dir/status.json" -log_file="$state_dir/job.log" -pid_file="$state_dir/pid" -running=false -pid=null -if [ -s "$pid_file" ]; then - pid_raw="$(cat "$pid_file" 2>/dev/null || true)" - if [ -n "$pid_raw" ] && kill -0 "$pid_raw" >/dev/null 2>&1; then running=true; pid="$pid_raw"; else pid="$pid_raw"; fi -fi -python3 - "$state_dir" "$status_file" "$log_file" "$running" "$pid" ${shQuote(String(tailLines))} <<'PY' -import json, pathlib, sys -state_dir=pathlib.Path(sys.argv[1]) -status_path=pathlib.Path(sys.argv[2]) -log_path=pathlib.Path(sys.argv[3]) -running=sys.argv[4] == "true" -pid=None if sys.argv[5] in ("", "null") else sys.argv[5] -tail_lines=int(sys.argv[6]) -status=None -if status_path.exists(): - try: - status=json.loads(status_path.read_text()) - except Exception as error: - status={"parseError": str(error), "raw": status_path.read_text(errors="replace")[-1000:]} -log_tail="" -if log_path.exists(): - lines=log_path.read_text(errors="replace").splitlines() - log_tail="\\n".join(lines[-tail_lines:]) -print(json.dumps({"stateDir": str(state_dir), "pid": pid, "running": running, "status": status, "logBytes": log_path.stat().st_size if log_path.exists() else 0, "logTail": log_tail}, ensure_ascii=False)) -PY -`; -} - -function remoteJobLogs(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, name: "tools-image" | "argo", options: ToolsImageOptions | ArgoOptions): Record { - const result = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, name, options.tailLines), options.timeoutSeconds); - const parsed = parseRemoteJson(result.stdout); - return { - ok: result.exitCode === 0, - command: `hwlab nodes control-plane infra ${name === "tools-image" ? "tools-image" : "argo"} logs`, - configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, - node: node.id, - lane: target.lane, - mutation: false, - job: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, - result: compactCommandResult(result), - }; -} - -function manifestObjectSummary(manifest: readonly Record[]): Record[] { - return manifest.map((item) => { - const metadata = record(item.metadata); - return { kind: item.kind ?? null, namespace: metadata.namespace ?? null, name: metadata.name ?? null }; - }); -} - -function runTransK3s(kubeRoute: string, script: string, timeoutSeconds: number): CommandResult { - return runCommand(["/root/.local/bin/trans", kubeRoute, "sh", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); -} - -function proxyExportBlock(node: ControlPlaneNodeSpec): string { - const proxy = node.egressProxy; - if (proxy === null) return " : # no egress proxy configured\n"; - const noProxy = [...new Set(["localhost", "127.0.0.1", "::1", "127.0.0.1:5000", "localhost:5000", ...proxy.noProxy])]; - return ` - proxy_ip="$(kubectl -n ${shQuote(proxy.namespace)} get svc ${shQuote(proxy.serviceName)} -o 'jsonpath={.spec.clusterIP}' 2>/dev/null || true)" - if [ -z "$proxy_ip" ]; then echo "egress proxy service missing: ${proxy.namespace}/${proxy.serviceName}" >&2; exit 41; fi - export HTTP_PROXY="http://$proxy_ip:${proxy.port}" - export HTTPS_PROXY="$HTTP_PROXY" - export ALL_PROXY="$HTTP_PROXY" - export http_proxy="$HTTP_PROXY" - export https_proxy="$HTTP_PROXY" - export all_proxy="$HTTP_PROXY" - export NO_PROXY=${shQuote(noProxy.join(","))} - export no_proxy="$NO_PROXY" -`; -} - -function remoteJobStateDir(target: ControlPlaneTargetSpec, name: "tools-image" | "argo"): string { - return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/${name}`; -} - -function shellJsonArray(items: readonly string[]): string { - return JSON.stringify([...items]); -} - -function parseRemoteJson(text: string): unknown { - const trimmed = text.trim(); - if (trimmed.length === 0) return null; - try { return JSON.parse(trimmed); } catch { - const start = trimmed.indexOf("{"); - const end = trimmed.lastIndexOf("}"); - if (start >= 0 && end > start) { - try { return JSON.parse(trimmed.slice(start, end + 1)); } catch {} - } - } - return null; -} - -function ciBuildBenchmarkStateDir(target: ControlPlaneTargetSpec, profile: string): string { - return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/ci-build-benchmark-${profile}`; -} - -function ciBuildBenchmarkLiveOk(job: Record, expectedServices: readonly string[], profile: CiBuildBenchmarkProfileSpec): boolean { - const pipelineRun = record(job.pipelineRun); - const pipelineStatus = renderCell(pipelineRun.status, ""); - if (pipelineStatus === "False") return false; - if (pipelineStatus !== "True") return true; - const taskRuns = ciBuildBenchmarkTaskRunRecords(job); - for (const service of expectedServices) { - if (!taskRuns.some((task) => task.pipelineTask === `build-${service}`)) return false; - } - return ciBuildBenchmarkPolicyOk(job, profile.cachePolicy); -} - -function ciBuildBenchmarkTaskRows(job: Record): Record[] { - const pipelineRun = record(job.pipelineRun); - const rows: Record[] = []; - if (Object.keys(pipelineRun).length > 0) { - rows.push({ - task: "pipeline-total", - status: ciBuildBenchmarkStatusText(pipelineRun.status), - duration: durationBetweenIso(pipelineRun.startTime, pipelineRun.completionTime), - start: shortIsoTime(pipelineRun.startTime), - end: shortIsoTime(pipelineRun.completionTime), - }); - } - const taskRuns = ciBuildBenchmarkTaskRunRecords(job).sort((left, right) => renderCell(left.startTime, "").localeCompare(renderCell(right.startTime, ""))); - for (const task of taskRuns) { - rows.push({ - task: renderCell(task.pipelineTask ?? task.name), - status: ciBuildBenchmarkStatusText(task.status), - duration: durationBetweenIso(task.startTime, task.completionTime), - start: shortIsoTime(task.startTime), - end: shortIsoTime(task.completionTime), - }); - } - return rows; -} - -function ciBuildBenchmarkServiceRows(job: Record, servicesValue: unknown): Record[] { - const services = ciBuildBenchmarkExpectedServices(servicesValue); - if (services.length === 0) return []; - const pipelineRun = record(job.pipelineRun); - const pipelineTerminal = pipelineRun.status === "True" || pipelineRun.status === "False"; - const taskRuns = ciBuildBenchmarkTaskRunRecords(job); - return services.map((service) => { - const task = taskRuns.find((item) => item.pipelineTask === `build-${service}`); - if (task === undefined) { - const status = pipelineTerminal ? "missing" : "pending"; - return { - service, - task: `build-${service}`, - status, - duration: "-", - failure: pipelineRun.status === "True" ? "cache-hit-forbidden" : "-", - }; - } - const taskStatus = ciBuildBenchmarkStatusText(task.status); - const failure = task.status === "False" ? classifyCiBuildBenchmarkFailure(`${renderCell(task.reason, "")}\n${renderCell(task.message, "")}`) : "-"; - return { - service, - task: renderCell(task.pipelineTask ?? task.name), - status: taskStatus, - duration: durationBetweenIso(task.startTime, task.completionTime), - failure, - }; - }); -} - -function ciBuildBenchmarkFailureRows(job: Record, serviceRows: readonly Record[], benchmark: Record): Record[] { - const counts = new Map(); - const add = (family: string, scope: string): void => { - if (family === "-" || family.length === 0) return; - const existing = counts.get(family) ?? { count: 0, scopes: [] }; - existing.count += 1; - if (!existing.scopes.includes(scope)) existing.scopes.push(scope); - counts.set(family, existing); - }; - for (const row of serviceRows) add(row.failure, row.service); - const pipelineRun = record(job.pipelineRun); - if (pipelineRun.status === "False") { - add(classifyCiBuildBenchmarkFailure(`${renderCell(pipelineRun.reason, "")}\n${renderCell(pipelineRun.message, "")}`), "pipeline"); - } - const cachePolicy = record(benchmark.cachePolicy); - if (cachePolicy.forbidBuildkitCache === true && ciBuildBenchmarkLogHasBuildkitCache(job)) add("cache-hit-forbidden", "buildkit-cache"); - if (cachePolicy.forbidGitopsCatalogReuse === true && ciBuildBenchmarkLogHasReuse(job)) add("cache-hit-forbidden", "artifact-reuse"); - return [...counts.entries()].map(([family, value]) => ({ family, count: String(value.count), scope: value.scopes.join(",") })); -} - -function ciBuildBenchmarkPolicyOk(job: Record, cachePolicy: CiBuildBenchmarkCachePolicy): boolean { - if (cachePolicy.forbidBuildkitCache && ciBuildBenchmarkLogHasBuildkitCache(job)) return false; - if (cachePolicy.forbidGitopsCatalogReuse && ciBuildBenchmarkLogHasReuse(job)) return false; - return true; -} - -function ciBuildBenchmarkLogHasBuildkitCache(job: Record): boolean { - const logTail = typeof job.logTail === "string" ? job.logTail : ""; - return /"buildkitCacheRef"\s*:\s*"[^"]+"|--import-cache|--export-cache|writing cache image manifest/iu.test(logTail); -} - -function ciBuildBenchmarkLogHasReuse(job: Record): boolean { - const logTail = typeof job.logTail === "string" ? job.logTail : ""; - return /"reusedFrom"\s*:\s*"(?!null)|"status"\s*:\s*"reused"/iu.test(logTail); -} - -function ciBuildBenchmarkTaskRunRecords(job: Record): Record[] { - return Array.isArray(job.taskRuns) ? job.taskRuns.map(record) : []; -} - -function ciBuildBenchmarkExpectedServices(value: unknown): string[] { - return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : []; -} - -function ciBuildBenchmarkStatusText(value: unknown): string { - if (value === "True") return "succeeded"; - if (value === "False") return "failed"; - if (value === "Unknown") return "running"; - return renderCell(value, "pending"); -} - -function classifyCiBuildBenchmarkFailure(text: string): string { - const value = text.toLowerCase(); - if (/cache-hit-forbidden|reused-from|reuse/i.test(text)) return "cache-hit-forbidden"; - if (/no such host|could not resolve|enotfound|dns/i.test(text)) return "dns"; - if (/429|rate limit|too many requests|toomanyrequests/i.test(text)) return "rate-limit"; - if (/tls|certificate|x509|timeout|timed out|i\/o timeout/i.test(text)) return "tls-timeout"; - if (/proxy|connect|connection reset|connection refused|econn/i.test(text)) return "proxy-connect"; - if (/unauthorized|authentication required|permission denied|forbidden|denied/i.test(text)) return "auth"; - if (/imagepullbackoff|errimagepull|imagepolicy|pull access denied/i.test(text)) return "image-policy"; - if (/push|registry|blob upload|manifest invalid|manifest unknown/i.test(text)) return "registry-push"; - return value.trim().length === 0 ? "unknown" : "build-script"; -} - -function durationBetweenIso(startValue: unknown, endValue: unknown): string { - if (typeof startValue !== "string" || startValue.length === 0) return "-"; - const start = Date.parse(startValue); - if (!Number.isFinite(start)) return "-"; - const end = typeof endValue === "string" && endValue.length > 0 ? Date.parse(endValue) : Date.now(); - if (!Number.isFinite(end) || end < start) return "-"; - return formatDurationMs(end - start); -} - -function formatDurationMs(ms: number): string { - const seconds = Math.round(ms / 1000); - const minutes = Math.floor(seconds / 60); - const rest = seconds % 60; - return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`; -} - -function shortIsoTime(value: unknown): string { - if (typeof value !== "string" || value.length === 0) return "-"; - return value.replace(/^\d{4}-\d{2}-\d{2}T/u, "").replace(/Z$/u, "Z"); -} - -function shortDisplay(value: string): string { - return /^[0-9a-f]{40}$/iu.test(value) ? value.slice(0, 12).toLowerCase() : value; -} - function validateBenchmarkProfileName(value: string, path: string): void { if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 63) throw new Error(`${path} must be a DNS-label style benchmark profile`); } @@ -4123,6 +2791,12 @@ function optionalStringField(obj: Record, key: string, path: st return value; } +function absoluteConfigPathField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!value.startsWith("/") || value.includes("\0") || value.includes("..")) throw new Error(`${path}.${key} must be an absolute path without '..'`); + return value; +} + function validateKubernetesName(value: string, path: string): void { if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} must be a Kubernetes resource name`); } @@ -4242,6 +2916,15 @@ function validateHttpsUrl(value: string, path: string): void { if (parsed.protocol !== "https:") throw new Error(`${path} must use https://`); } +function hostRouteProxyHostAllowed(value: string): boolean { + if (value === "127.0.0.1" || value === "localhost") return true; + const parts = value.split(".").map((part) => Number(part)); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + return parts[0] === 10 + || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) + || (parts[0] === 192 && parts[1] === 168); +} + function sha256Short(text: string): string { return `sha256:${createHash("sha256").update(text).digest("hex")}`; } diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 334f87e9..6e6f7a2c 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -307,8 +307,12 @@ export interface HwlabRuntimePublicExposureSpec { export interface HwlabRuntimeBootstrapAdminSpec { readonly username: string; readonly displayName: string; + readonly usernameSourceRef?: string; + readonly usernameSourceKey?: string; + readonly usernameSourceLine?: number; readonly passwordSourceRef: string; readonly passwordSourceKey: string; + readonly passwordSourceLine?: number; readonly passwordHashTransform: "hwlab-sha256"; readonly secretName: string; readonly secretKey: string; @@ -756,11 +760,19 @@ function bootstrapAdminConfig(value: unknown, path: string): HwlabRuntimeBootstr const transform = stringField(raw, "passwordHashTransform", path); if (transform !== "hwlab-sha256") throw new Error(`${path}.passwordHashTransform must be hwlab-sha256`); const rollout = asRecord(raw.rollout, `${path}.rollout`); + const usernameSourceRef = raw.usernameSourceRef === undefined ? undefined : sourceRefField(raw, "usernameSourceRef", path); + const usernameSourceKey = raw.usernameSourceKey === undefined ? undefined : secretKeyField(raw, "usernameSourceKey", path); + const usernameSourceLine = raw.usernameSourceLine === undefined ? undefined : numberField(raw, "usernameSourceLine", path); + const passwordSourceLine = raw.passwordSourceLine === undefined ? undefined : numberField(raw, "passwordSourceLine", path); return { username: stringField(raw, "username", path), displayName: stringField(raw, "displayName", path), + ...(usernameSourceRef === undefined ? {} : { usernameSourceRef }), + ...(usernameSourceKey === undefined ? {} : { usernameSourceKey }), + ...(usernameSourceLine === undefined ? {} : { usernameSourceLine }), passwordSourceRef: sourceRefField(raw, "passwordSourceRef", path), passwordSourceKey: secretKeyField(raw, "passwordSourceKey", path), + ...(passwordSourceLine === undefined ? {} : { passwordSourceLine }), passwordHashTransform: transform, secretName: stringField(raw, "secretName", path), secretKey: secretKeyField(raw, "secretKey", path), @@ -1044,7 +1056,7 @@ function validateConfigRef(ref: string, path: string): void { if (file.startsWith("/") || file.includes("\0") || file.includes("..") || !file.startsWith("config/") || !file.endsWith(".yaml")) { throw new Error(`${path} must reference a repo-relative config/*.yaml file without ..`); } - if (!/^[A-Za-z0-9_.\-[\]]+$/u.test(fragment)) { + if (!/^[A-Za-z0-9_.\-[\]${}]+$/u.test(fragment)) { throw new Error(`${path} has an unsupported YAML path fragment`); } } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 116f98d0..245d3175 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -13,10 +13,12 @@ import { repoRoot, rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config"; +import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; -import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact"; +import { runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; +import { remainingSeconds, runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; export type WebProbeSentinelConfigAction = "plan" | "status"; export type WebProbeSentinelImageAction = "status" | "build"; @@ -113,7 +115,7 @@ export type WebProbeSentinelOptions = readonly raw: boolean; }; -interface SentinelCicdState { +export interface SentinelCicdState { readonly spec: HwlabRuntimeLaneSpec; readonly sentinelId: string; readonly configRefs: Record; @@ -180,7 +182,7 @@ interface SentinelRemoteJobResult { readonly valuesRedacted: true; } -interface CompactCommandResult { +export interface CompactCommandResult { readonly exitCode: number | null; readonly timedOut: boolean; readonly stdoutBytes: number; @@ -189,7 +191,7 @@ interface CompactCommandResult { readonly stderrPreview: string; } -interface ChildCliResult { +export interface ChildCliResult { readonly ok: boolean; readonly parsed: Record | null; readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string }; @@ -308,19 +310,19 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; return { @@ -360,11 +362,11 @@ function resolveSourceHead(cicd: Record, timeoutSeconds: number }; } -function sentinelImagePlan(cicd: Record, sourceHead: SourceHead): SentinelImagePlan { +function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record, sourceHead: SourceHead): SentinelImagePlan { const repository = stringAt(cicd, "image.repository"); const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12); const baseImageRef = stringAt(cicd, "image.baseImageRef"); - const baseImage = stringTarget(readConfigRefTarget(baseImageRef), baseImageRef); + const baseImage = stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef); const entrypoint = stringAt(cicd, "source.entrypoint"); const dockerfile = sentinelDockerfile(baseImage, entrypoint); return { @@ -740,7 +742,7 @@ function accountSecretEnvName(sourcePurpose: string, targetKey: string): string return segment.length === 0 ? null : `HWLAB_WEB_${segment}_JSON`; } -function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record } { +export function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: NodeJS.ProcessEnv; summary: Record } { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { const purpose = stringAtNullable(source, "purpose"); @@ -1821,14 +1823,14 @@ function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: b return [`targetValidation quick verify is deferred from control-plane confirm-wait to keep CI/CD wait under ${Math.round(budgetSeconds)}s; run ${next.quickVerify}.`]; } -function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { +export function targetValidationElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; if (elapsedMs === null || elapsedMs <= budgetMs) return []; return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`]; } -function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] { +export function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] { const warnings: string[] = []; for (const item of items) { const values = Array.isArray(item) ? item : [item]; @@ -1841,7 +1843,7 @@ function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): str return warnings; } -function withWarnings(payload: Record, warnings: readonly unknown[]): Record { +export function withWarnings(payload: Record, warnings: readonly unknown[]): Record { const merged = mergeWarnings(payload.warnings, warnings); return merged.length === 0 ? payload : { ...payload, warnings: merged, valuesRedacted: true }; } @@ -1882,1909 +1884,6 @@ function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelCont }; } -function runSentinelMaintenance(state: SentinelCicdState, options: Extract): RenderedCliResult { - const command = `web-probe sentinel maintenance ${options.action}`; - const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); - if (options.action === "status") { - const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds); - const result = { - ok: serviceHealth.ok && maintenance.ok, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - serviceHealth, - maintenance, - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(result.ok, command, renderMaintenanceResult(result)); - } - if (!options.confirm) { - const result = { - ok: serviceHealth.ok, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: "dry-run", - serviceHealth, - mutation: false, - planned: { - action: options.action, - releaseId: options.releaseId, - reason: options.reason, - quickVerify: options.action === "stop" && options.quickVerify, - }, - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(result.ok, command, renderMaintenanceResult(result)); - } - if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); - if (!serviceHealth.ok) { - const result = { - ok: false, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: "confirm-wait", - mutation: false, - serviceHealth, - blocker: serviceUnavailableBlocker(state), - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(false, command, renderMaintenanceResult(result)); - } - const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true }; - const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds); - const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok - ? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds) - : null; - const result = { - ok: mutation.ok && (quickVerify === null || quickVerify.ok === true), - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: "confirm-wait", - mutation: true, - serviceHealth, - maintenance: mutation, - quickVerify, - blocker: mutation.ok ? null : serviceUnavailableBlocker(state), - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(result.ok, command, renderMaintenanceResult(result)); -} - -function runSentinelValidate(state: SentinelCicdState, options: Extract): RenderedCliResult { - const command = "web-probe sentinel validate"; - const startedAt = Date.now(); - const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); - let quickVerify: Record | null = null; - if (options.quickVerify) { - if (!options.confirm) { - const result = { - ok: initialHealth.ok, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: "dry-run", - serviceHealth: initialHealth, - planned: { quickVerify: true, waitRequired: true }, - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(result.ok, command, renderValidateResult(result)); - } - if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true); - if (!initialHealth.ok) { - const result = { - ok: false, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: "confirm-wait", - serviceHealth: initialHealth, - blocker: serviceUnavailableBlocker(state), - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(false, command, renderValidateResult(result)); - } - quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds); - } - const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20); - const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds); - const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds); - const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds); - const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health"); - const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds); - const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds); - const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds); - const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health; - const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics; - const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report; - const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds); - const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds); - if (quickVerify !== null) { - quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds")))); - } - const publicFallbackWarnings = [ - ...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []), - ...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []), - ...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []), - ]; - const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health"); - const ok = effectiveHealth.ok - && record(effectiveHealth.bodyJson).ok === true - && effectiveMetricsOk - && effectiveReport.ok - && publicExposure.ok === true - && publicDashboard.ok === true - && (quickVerify === null || quickVerify.ok === true); - const result = { - ok, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: options.quickVerify ? "confirm-wait" : "status", - serviceHealth: effectiveHealth, - metrics: effectiveMetrics, - report: effectiveReport, - internalServiceHealth: health, - internalMetrics: metrics, - internalReport: report, - publicServiceHealth: publicHealth, - publicMetrics, - publicReport, - publicExposure, - publicDashboard, - quickVerify, - warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []), - blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify), - next: sentinelP5Next(state), - valuesRedacted: true, - }; - return rendered(ok, command, renderValidateResult(result)); -} - -function runSentinelReport(state: SentinelCicdState, options: Extract): RenderedCliResult { - const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`; - const query = new URLSearchParams({ view: options.view }); - if (options.runId !== null) query.set("run", options.runId); - if (options.traceId !== null) query.set("traceId", options.traceId); - if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq)); - const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds); - const body = record(report.bodyJson); - const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true }); - const rawPayload = Object.keys(body).length > 0 ? body : report; - return rendered(report.ok && body.ok !== false, command, options.raw ? JSON.stringify(rawPayload, null, 2) : renderedText); -} - -function runSentinelDashboard(state: SentinelCicdState, options: Extract): RenderedCliResult { - const command = `web-probe sentinel dashboard ${options.action}`; - const result = probeSentinelDashboardBrowser(state, options); - return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result)); -} - -function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract): Record { - const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); - const [widthRaw, heightRaw] = options.viewport.split("x"); - const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : ""; - const script = [ - "set -eu", - `export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(`${publicBaseUrl}/`)}`, - `export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`, - `export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`, - `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`, - `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`, - `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, - `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`, - `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`, - `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`, - `export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`, - "export PLAYWRIGHT_BROWSERS_PATH=0", - "if command -v chromium >/dev/null 2>&1; then", - " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)", - "elif command -v chromium-browser >/dev/null 2>&1; then", - " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)", - "elif command -v google-chrome >/dev/null 2>&1; then", - " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)", - "else", - " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=", - "fi", - "cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'", - sentinelDashboardBrowserModule(), - "WEB_PROBE_SENTINEL_DASHBOARD_JS", - "bun \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"", - ].join("\n"); - const route = `${state.spec.nodeId}:${state.spec.workspace}`; - const job = runWebProbeRemoteArtifactJob({ - route, - localDir: options.localDir, - waitTimeoutMs: options.waitTimeoutMs, - commandTimeoutMs: options.commandTimeoutSeconds * 1000, - inactivityTimeoutMs: 30000, - runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`, - stdoutTailBytes: 32768, - }, script); - const result = job.result; - const transport = record(job.transport); - const remote = record(transport.remote); - const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : ""); - const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : []; - const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null; - const browserOk = page?.ok === true; - const screenshotOk = options.action === "verify" || screenshot !== null && screenshot.verified === true; - const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk; - return { - ok, - status: ok ? "pass" : "blocked", - command: `web-probe sentinel dashboard ${options.action}`, - node: state.spec.nodeId, - lane: state.spec.lane, - sentinelId: state.sentinelId, - publicUrl: `${publicBaseUrl}/`, - route, - viewport: options.viewport, - page, - screenshot, - artifacts, - artifactCount: artifacts.length, - remote: { - exitCode: remote.exitCode ?? null, - remoteDir: remote.remoteDir ?? null, - stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "", - stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "", - }, - transport: { - ok: transport.ok ?? null, - runId: transport.runId ?? null, - artifactCount: transport.artifactCount ?? null, - expectedArtifactCount: transport.expectedArtifactCount ?? null, - downloadFailure: transport.downloadFailure ?? null, - }, - result: compactCommand(result), - degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk), - valuesRedacted: true, - }; -} - -function sentinelDashboardBrowserModule(): string { - return String.raw`import { pathToFileURL } from "node:url"; - -const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || ""; -const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright"; -const { chromium } = await import(playwrightModuleSpecifier); - -const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL; -const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || ""; -const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1"; -const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440); -const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900); -const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000); -const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"; -const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || ""; -const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || ""; -const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/"; - -if (!url) throw new Error("missing dashboard URL"); - -const consoleMessages = []; -const pageErrors = []; -const requestFailures = []; -const browser = await chromium.launch({ - headless: true, - args: ["--disable-gpu", "--no-sandbox"], - ...(executablePath ? { executablePath } : {}), -}); -const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); -const page = await context.newPage(); -page.on("console", (message) => { - if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) }); -}); -page.on("pageerror", (error) => { - if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) }); -}); -page.on("requestfailed", (request) => { - if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null }); -}); - -let httpStatus = null; -let navigationError = null; -let navigationAttempts = 0; -const maxNavigationAttempts = 3; -const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts)); -for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) { - navigationAttempts = attempt; - navigationError = null; - try { - const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }); - httpStatus = response?.status() ?? null; - await page.waitForLoadState("networkidle", { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {}); - await page.waitForFunction(() => { - const root = document.querySelector("#monitor-web-root"); - if (!root) return false; - const ready = root.getAttribute("data-monitor-ready") === "true"; - const error = document.querySelector("#monitor-web-error"); - const runs = document.querySelectorAll(".run-list .run-row").length; - const trend = document.querySelector("[data-monitor-trend-curve]"); - return ready && (error || runs > 0 || trend); - }, null, { timeout: Math.min(15000, perAttemptTimeout) }).catch(() => {}); - await page.waitForTimeout(500); - const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false); - if (appReady || attempt === maxNavigationAttempts) break; - } catch (error) { - navigationError = String(error?.message || error).slice(0, 500); - if (attempt === maxNavigationAttempts) break; - } - await page.waitForTimeout(750 * attempt); -} - -await page.evaluate(() => { - const detailPane = document.querySelector(".workspace-grid .pane-detail"); - if (detailPane instanceof HTMLElement) detailPane.scrollTop = Math.min(96, Math.max(0, detailPane.scrollHeight - detailPane.clientHeight)); -}).catch(() => {}); -await page.waitForTimeout(150); - -const trendHoverPoint = await page.evaluate(() => { - const target = document.querySelector(".trend-dot-hit .trend-dot-red") || document.querySelector(".trend-dot-hit .trend-dot-warning"); - if (!(target instanceof SVGElement)) return null; - const rect = target.getBoundingClientRect(); - if (rect.width <= 0 || rect.height <= 0) return null; - return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; -}).catch(() => null); -if (trendHoverPoint) { - await page.mouse.move(trendHoverPoint.x, trendHoverPoint.y); - await page.waitForTimeout(250); -} - -if (captureScreenshot && screenshotPath) { - await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => { - pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) }); - }); -} - -const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId }) => { - const visible = (element) => Boolean(element && !element.hidden); - const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim(); - const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; - const errorSampleCount = (counts) => numberValue(counts?.red) + numberValue(counts?.critical) + numberValue(counts?.error); - const warningSampleCount = (counts) => numberValue(counts?.warning) + numberValue(counts?.warn) + numberValue(counts?.amber); - const allSampleCount = (counts) => Object.values(counts || {}).reduce((sum, value) => sum + numberValue(value), 0); - const root = document.querySelector("#monitor-web-root"); - const shell = document.querySelector("[data-monitor-shell='true']"); - const error = document.querySelector("#monitor-web-error"); - const trend = document.querySelector("[data-monitor-trend-curve]"); - const trendTooltip = document.querySelector("[data-monitor-trend-tooltip='true']"); - const timeline = document.querySelector("[data-monitor-timeline='true']"); - const workspace = document.querySelector("[data-monitor-independent-scroll='true']"); - const checksPanel = document.querySelector("[data-monitor-checks='true']"); - const panes = Array.from(document.querySelectorAll(".workspace-grid .pane, [data-monitor-checks='true']")); - const detailPane = document.querySelector(".workspace-grid .pane-detail"); - const detailHeader = detailPane?.querySelector(".pane-header"); - const checksHeader = checksPanel?.querySelector(".pane-header"); - const internalTextPattern = /水合|投影|Trace|trace|Shell|API|DOM|Console|console|Runner|runner|JSONL|steer|facts|分页|HTTP|http|requestfailed|pageerror|Final Response|Code Agent|web-probe|observe|analyzer|终态/u; - const checkRows = Array.from(document.querySelectorAll("[data-check-row='true']")); - const cards = checkRows.slice(0, 8).map((row) => ({ - code: String(row.querySelector(".check-code")?.textContent || "").trim(), - title: String(row.querySelector(".check-title-cell strong")?.textContent || row.querySelector("strong")?.textContent || "").trim(), - body: String(row.textContent || "").replace(/\s+/g, " ").trim().slice(0, 180), - })); - const badCardTitles = cards.filter((card) => internalTextPattern.test(card.title)); - const badCardBodies = cards.filter((card) => internalTextPattern.test(card.body)); - const legendTexts = Array.from(document.querySelectorAll(".trend-legend .legend-item")).map((item) => String(item.textContent || "").replace(/\s+/g, " ").trim()); - const legendNumber = (label) => { - const row = legendTexts.find((item) => item.includes(label)) || ""; - const match = /(\d+)/u.exec(row); - return match ? Number(match[1]) : null; - }; - const chartCounts = { - error: legendNumber("最新点错误"), - warning: legendNumber("最新点告警"), - total: legendNumber("错误+告警合计"), - }; - chartCounts.ok = typeof chartCounts.error === "number" && typeof chartCounts.warning === "number" && typeof chartCounts.total === "number" - ? chartCounts.total === chartCounts.error + chartCounts.warning - : false; - const apiUrl = (path) => { - const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; - const prefix = basePath.replace(/\/+$/u, ""); - return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; - }; - const runsPayload = await fetch(apiUrl("/api/runs?limit=30&sort=updated"), { cache: "no-store" }).then((item) => item.json()).catch(() => null); - const overviewPayload = await fetch(apiUrl("/api/overview"), { cache: "no-store" }).then((item) => item.json()).catch(() => null); - const runs = Array.isArray(runsPayload?.runs) ? runsPayload.runs : Array.isArray(runsPayload?.items) ? runsPayload.items : []; - const latestRun = runs[0] || null; - const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts) - ? latestRun.severityCounts - : {}; - const latestRunCounts = { - runId: latestRun?.id || latestRun?.runId || null, - typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count), - error: 0, - warning: 0, - total: 0, - all: 0, - errorSamples: errorSampleCount(latestCounts), - warningSamples: warningSampleCount(latestCounts), - alertSamples: errorSampleCount(latestCounts) + warningSampleCount(latestCounts), - allSamples: allSampleCount(latestCounts), - }; - const latestDetailPayload = latestRunCounts.runId - ? await fetch(apiUrl("/api/runs/" + encodeURIComponent(latestRunCounts.runId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null) - : null; - const latestDetailRows = Array.isArray(latestDetailPayload?.findings) ? latestDetailPayload.findings : []; - const rowSeverity = (row) => { - const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase(); - if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red"; - if (["warning", "warn", "amber"].includes(raw)) return "warning"; - if (["info", "notice"].includes(raw)) return "info"; - return "healthy"; - }; - const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1; - const summarizeRows = (rows) => { - const errorRows = rows.filter((row) => rowSeverity(row) === "red"); - const warningRows = rows.filter((row) => rowSeverity(row) === "warning"); - const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0); - return { - typeCount: rows.length, - errorTypeCount: errorRows.length, - warningTypeCount: warningRows.length, - alertTypeCount: errorRows.length + warningRows.length, - errorSamples: sum(errorRows), - warningSamples: sum(warningRows), - alertSamples: sum(errorRows) + sum(warningRows), - }; - }; - const latestDetailSummary = summarizeRows(latestDetailRows); - latestRunCounts.typeCount = latestDetailSummary.typeCount; - latestRunCounts.error = latestDetailSummary.errorTypeCount; - latestRunCounts.warning = latestDetailSummary.warningTypeCount; - latestRunCounts.total = latestDetailSummary.alertTypeCount; - latestRunCounts.all = latestDetailSummary.typeCount; - const workspaceRect = workspace?.getBoundingClientRect(); - const checksRect = checksPanel?.getBoundingClientRect(); - const heightSummary = (rect) => { - const viewportHeight = window.innerHeight || 1; - const heightPx = rect ? Math.round(rect.height) : null; - const ratio = rect ? Math.round((rect.height / viewportHeight) * 1000) / 1000 : null; - return { - present: Boolean(rect), - heightPx, - ratio, - targetPx: Math.round(viewportHeight * 0.8), - bounded80: Boolean(rect && rect.height <= viewportHeight * 0.82 + 1), - near80: Boolean(rect && rect.height >= viewportHeight * 0.74 && rect.height <= viewportHeight * 0.82 + 1), - }; - }; - const workspacePaneHeights = Array.from(document.querySelectorAll(".workspace-grid > .pane")).map((pane) => heightSummary(pane.getBoundingClientRect())); - const stackedWorkspace = window.matchMedia("(max-width: 1120px)").matches; - const panelHeights = { - viewportHeight: window.innerHeight, - stackedWorkspace, - workspace: heightSummary(workspaceRect), - checks: heightSummary(checksRect), - workspacePanes: workspacePaneHeights, - workspacePaneBounded: workspacePaneHeights.length >= 2 && workspacePaneHeights.every((item) => item.bounded80 === true), - workspaceOk: false, - checksOk: false, - }; - panelHeights.workspaceOk = stackedWorkspace - ? panelHeights.workspacePaneBounded === true - : panelHeights.workspace.near80 === true && panelHeights.workspacePaneBounded === true; - panelHeights.checksOk = panelHeights.checks.near80 === true; - const checkScope = { - present: Boolean(checksPanel), - scope: checksPanel?.getAttribute("data-check-scope") || null, - runId: checksPanel?.getAttribute("data-check-run-id") || null, - typeCount: numberValue(checksPanel?.getAttribute("data-check-type-count")), - errorTypeCount: numberValue(checksPanel?.getAttribute("data-check-error-type-count")), - warningTypeCount: numberValue(checksPanel?.getAttribute("data-check-warning-type-count")), - alertTypeCount: numberValue(checksPanel?.getAttribute("data-check-alert-type-count")), - errorSamples: numberValue(checksPanel?.getAttribute("data-check-error-samples")), - warningSamples: numberValue(checksPanel?.getAttribute("data-check-warning-samples")), - alertSamples: numberValue(checksPanel?.getAttribute("data-check-alert-samples")), - visibleRowCount: document.querySelectorAll("[data-check-row='true']").length, - visibleAlertSamples: numberValue(checksPanel?.getAttribute("data-visible-check-alert-samples")), - matchesLatestRun: false, - matchesRunDetail: false, - belowWorkspace: Boolean(workspaceRect && checksRect && checksRect.top >= workspaceRect.bottom - 2), - fullWidth: Boolean(workspaceRect && checksRect && checksRect.width >= workspaceRect.width - 2), - }; - const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row")) - .find((row) => row.getAttribute("data-run-id") === runId) || null; - const selectedRunRow = runListRowFor(latestRunCounts.runId) || document.querySelector(".run-list .run-row.selected"); - const selectedRunTags = { - error: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), - warning: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), - errorVisible: Boolean(selectedRunRow?.querySelector("[data-run-error-tag='true']")), - warningVisible: Boolean(selectedRunRow?.querySelector("[data-run-warning-tag='true']")), - rowSelected: Boolean(selectedRunRow?.classList.contains("selected")), - matchesRunDetail: false, - }; - selectedRunTags.matchesRunDetail = selectedRunTags.error === latestDetailSummary.errorTypeCount - && selectedRunTags.warning === latestDetailSummary.warningTypeCount - && selectedRunTags.errorVisible === (latestDetailSummary.errorTypeCount > 0) - && selectedRunTags.warningVisible === (latestDetailSummary.warningTypeCount > 0) - && selectedRunTags.rowSelected === true; - checkScope.matchesLatestRun = checkScope.present === true - && checkScope.scope === "run" - && checkScope.runId === latestRunCounts.runId - && checkScope.errorTypeCount === latestRunCounts.error - && checkScope.warningTypeCount === latestRunCounts.warning - && checkScope.alertTypeCount === latestRunCounts.total; - checkScope.matchesRunDetail = checkScope.present === true - && checkScope.typeCount === latestDetailSummary.typeCount - && checkScope.errorTypeCount === latestDetailSummary.errorTypeCount - && checkScope.warningTypeCount === latestDetailSummary.warningTypeCount - && checkScope.alertTypeCount === latestDetailSummary.alertTypeCount - && checkScope.errorSamples === latestDetailSummary.errorSamples - && checkScope.warningSamples === latestDetailSummary.warningSamples - && checkScope.alertSamples === latestDetailSummary.alertSamples - && checkScope.visibleRowCount === latestDetailSummary.alertTypeCount - && checkScope.visibleAlertSamples === latestDetailSummary.alertSamples; - const overviewCounts = overviewPayload?.severityCounts && typeof overviewPayload.severityCounts === "object" && !Array.isArray(overviewPayload.severityCounts) - ? overviewPayload.severityCounts - : {}; - const overviewSamples = { - error: errorSampleCount(overviewCounts), - warning: warningSampleCount(overviewCounts), - total: errorSampleCount(overviewCounts) + warningSampleCount(overviewCounts), - allSamples: allSampleCount(overviewCounts), - }; - chartCounts.matchesLatestRun = chartCounts.ok === true - && chartCounts.error === latestRunCounts.error - && chartCounts.warning === latestRunCounts.warning - && chartCounts.total === latestRunCounts.total; - const trendPanel = document.querySelector(".trend-panel"); - const trendLegend = document.querySelector(".trend-panel .trend-legend"); - const trendPanelRect = trendPanel?.getBoundingClientRect(); - const trendLegendRect = trendLegend?.getBoundingClientRect(); - const trendPanelCompact = { - present: Boolean(trendPanelRect && trendLegendRect), - bottomSlackPx: trendPanelRect && trendLegendRect ? Math.round(trendPanelRect.bottom - trendLegendRect.bottom) : null, - ok: Boolean(trendPanelRect && trendLegendRect && trendPanelRect.bottom - trendLegendRect.bottom <= 28), - }; - const firstCheckRow = document.querySelector("[data-check-row='true']"); - let checkDialog = { opened: false, title: "", width: null, height: null, large: false }; - if (firstCheckRow instanceof HTMLElement) { - firstCheckRow.click(); - await new Promise((resolve) => window.setTimeout(resolve, 80)); - const dialog = document.querySelector("[data-check-dialog='true'] .check-dialog"); - const rect = dialog?.getBoundingClientRect(); - checkDialog = { - opened: Boolean(dialog), - title: String(dialog?.querySelector("#check-dialog-title")?.textContent || "").trim(), - width: rect ? Math.round(rect.width) : null, - height: rect ? Math.round(rect.height) : null, - large: Boolean(rect && rect.width >= Math.min(900, window.innerWidth * 0.7) && rect.height >= Math.min(460, window.innerHeight * 0.5)), - }; - const close = dialog?.querySelector("button[aria-label='关闭监测项详情']"); - if (close instanceof HTMLElement) close.click(); - } - const datasetSentinelId = root?.getAttribute("data-sentinel-id") || ""; - const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/"; - const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/"; - const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/"); - const sentinelBoundary = { - expectedSentinelId, - expectedRoutePrefix, - datasetSentinelId, - overviewSentinelId: overviewPayload?.sentinelId || null, - runsSentinelId: runsPayload?.sentinelId || null, - finalPath, - routePrefixMatches, - datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true, - overviewMatches: expectedSentinelId ? overviewPayload?.sentinelId === expectedSentinelId : true, - runsPayloadMatches: expectedSentinelId ? runsPayload?.sentinelId === expectedSentinelId : true, - runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true, - }; - const statusText = text(".status-strip"); - const doc = document.documentElement; - const body = document.body; - const viewport = { width: window.innerWidth, height: window.innerHeight }; - const documentSize = { - width: Math.max(doc.scrollWidth, body?.scrollWidth || 0), - height: Math.max(doc.scrollHeight, body?.scrollHeight || 0), - }; - const overflow = []; - let overflowCount = 0; - for (const element of Array.from(document.querySelectorAll("body *"))) { - const rect = element.getBoundingClientRect(); - const overflowRight = rect.right - viewport.width; - const overflowLeft = -rect.left; - if (overflowRight > 1 || overflowLeft > 1) { - overflowCount += 1; - if (overflow.length < 5) { - overflow.push({ - tag: element.tagName.toLowerCase(), - className: String(element.className || "").slice(0, 40), - x: Math.round(rect.x), - y: Math.round(rect.y), - width: Math.round(rect.width), - height: Math.round(rect.height), - overflowRight: Math.max(0, Math.round(overflowRight)), - overflowLeft: Math.max(0, Math.round(overflowLeft)), - }); - } - } - } - return { - shell: Boolean(root && shell), - ready: root?.getAttribute("data-monitor-ready") === "true", - dataset: root ? { - node: root.getAttribute("data-node"), - lane: root.getAttribute("data-lane"), - sentinelId: datasetSentinelId, - basePath: root.getAttribute("data-base-path"), - contractVersion: root.getAttribute("data-contract-version"), - } : {}, - sentinelBoundary, - title: document.title, - finalUrl: window.location.href, - statusText: text(".topbar .pill"), - subtitle: text(".subtitle"), - summaryText: text(".status-strip"), - runRows: document.querySelectorAll(".run-list .run-row").length, - checkRows: document.querySelectorAll("[data-check-row='true']").length, - badCardTitleCount: badCardTitles.length, - badCardBodyCount: badCardBodies.length, - trendCurve: Boolean(trend), - trendDotCount: document.querySelectorAll(".trend-dot-hit").length, - trendTooltip: tooltipSummary(trendTooltip), - trendPanelText: text("#trend-heading"), - chartCounts, - latestRunCounts, - latestDetailSummary, - checkScope, - selectedRunTags, - trendPanelCompact, - checkDialog, - overviewSamples, - panelHeights, - scopeLabels: { - latestPointLegend: legendTexts.some((item) => item.includes("最新点")), - historicalSamples: legendTexts.some((item) => item.includes("历史样本累计")) || statusText.includes("历史错误样本"), - }, - timelineItems: document.querySelectorAll(".timeline-list .timeline-item").length, - timelineVisible: Boolean(timeline), - errorVisible: visible(error), - errorText: visible(error) ? text("#monitor-web-error").slice(0, 500) : "", - scrollModel: { - workspace: Boolean(workspace), - paneCount: panes.length, - independentScroll: panes.length >= 3 && panes.every((pane) => { - const style = window.getComputedStyle(pane); - return style.overflowY === "auto" || style.overflowY === "scroll"; - }), - stickyHeader: stickyHeaderSummary(detailPane, detailHeader), - stickyChecksHeader: stickyHeaderSummary(checksPanel, checksHeader), - }, - layout: { - viewport, - documentSize, - horizontalOverflow: documentSize.width > viewport.width + 1, - overflowCount, - overflow: overflow.slice(0, 2), - }, - }; - - function tooltipSummary(element) { - const body = String(element?.textContent || "").replace(/\s+/g, " ").trim(); - return { - visible: Boolean(element && body.length > 0), - text: body.slice(0, 240), - hasValues: /错误\s+\d+/u.test(body) && /告警\s+\d+/u.test(body) && /合计\s+\d+/u.test(body), - hasTime: /UTC/u.test(body) || /\d{4}-\d{2}-\d{2}/u.test(body), - }; - } - - function stickyHeaderSummary(pane, header) { - if (!(pane instanceof HTMLElement) || !(header instanceof HTMLElement)) { - return { present: false, coversScroll: false, backgroundOpaque: false, detailScrollTop: null }; - } - const rect = header.getBoundingClientRect(); - const style = window.getComputedStyle(header); - const sampleX = Math.round(rect.left + Math.min(32, Math.max(2, rect.width / 2))); - const sampleY = Math.round(rect.top + Math.min(12, Math.max(2, rect.height / 2))); - const topElement = document.elementFromPoint(sampleX, sampleY); - return { - present: true, - detailScrollTop: pane.scrollTop, - headerTop: Math.round(rect.top), - headerBottom: Math.round(rect.bottom), - zIndex: style.zIndex, - backgroundColor: style.backgroundColor, - coversScroll: Boolean(topElement && header.contains(topElement)), - backgroundOpaque: backgroundIsOpaque(style.backgroundColor), - topElementClass: String(topElement?.className || "").slice(0, 80), - }; - } - - function backgroundIsOpaque(value) { - const rgba = /rgba?\(([^)]+)\)/u.exec(value); - if (rgba === null) return value.length > 0 && value !== "transparent"; - const parts = rgba[1].split(",").map((part) => part.trim()); - if (parts.length < 4) return true; - return Number(parts[3]) >= 0.99; - } -}, { expectedRoutePrefix, expectedSentinelId }); - -const runFilterProbe = await page.evaluate(async ({ expectedRoutePrefix }) => { - const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; - const root = document.querySelector("#monitor-web-root"); - const apiUrl = (path) => { - const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; - const prefix = basePath.replace(/\/+$/u, ""); - return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; - }; - const rowSeverity = (row) => { - const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase(); - if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red"; - if (["warning", "warn", "amber"].includes(raw)) return "warning"; - if (["info", "notice"].includes(raw)) return "info"; - return "healthy"; - }; - const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1; - const summarizeRows = (rows) => { - const errorRows = rows.filter((row) => rowSeverity(row) === "red"); - const warningRows = rows.filter((row) => rowSeverity(row) === "warning"); - const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0); - return { - typeCount: rows.length, - errorTypeCount: errorRows.length, - warningTypeCount: warningRows.length, - alertTypeCount: errorRows.length + warningRows.length, - errorSamples: sum(errorRows), - warningSamples: sum(warningRows), - alertSamples: sum(errorRows) + sum(warningRows), - }; - }; - const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row")) - .find((row) => row.getAttribute("data-run-id") === runId) || null; - const panelCounts = () => { - const panel = document.querySelector("[data-monitor-checks='true']"); - const panelRunId = panel?.getAttribute("data-check-run-id") || null; - const selectedRunRow = runListRowFor(panelRunId) || document.querySelector(".run-list .run-row.selected"); - return { - present: Boolean(panel), - runId: panelRunId, - typeCount: numberValue(panel?.getAttribute("data-check-type-count")), - errorTypeCount: numberValue(panel?.getAttribute("data-check-error-type-count")), - warningTypeCount: numberValue(panel?.getAttribute("data-check-warning-type-count")), - alertTypeCount: numberValue(panel?.getAttribute("data-check-alert-type-count")), - errorSamples: numberValue(panel?.getAttribute("data-check-error-samples")), - warningSamples: numberValue(panel?.getAttribute("data-check-warning-samples")), - alertSamples: numberValue(panel?.getAttribute("data-check-alert-samples")), - visibleRowCount: document.querySelectorAll("[data-check-row='true']").length, - visibleAlertSamples: numberValue(panel?.getAttribute("data-visible-check-alert-samples")), - selectedRunErrorTag: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), - selectedRunWarningTag: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), - selectedRunRowSelected: Boolean(selectedRunRow?.classList.contains("selected")), - }; - }; - const waitForRun = async (runId) => { - for (let index = 0; index < 30; index += 1) { - const current = panelCounts(); - if (current.runId === runId && current.present === true) return true; - await new Promise((resolve) => window.setTimeout(resolve, 100)); - } - return false; - }; - const select = document.querySelector("select[aria-label='选择运行记录']"); - if (!(select instanceof HTMLSelectElement)) { - return { ok: false, requestedRunId: null, reason: "run-select-missing" }; - } - const options = Array.from(select.options).filter((option) => option.value); - const requestedOption = options[Math.min(2, Math.max(0, options.length - 1))] || options[0] || null; - const requestedRunId = requestedOption?.value || null; - const fallbackOption = requestedOption || null; - const targetRunId = fallbackOption?.value || requestedRunId; - if (!targetRunId) return { ok: false, requestedRunId, reason: "run-option-missing" }; - select.value = targetRunId; - select.dispatchEvent(new Event("change", { bubbles: true })); - const panelReady = await waitForRun(targetRunId); - const detailPayload = await fetch(apiUrl("/api/runs/" + encodeURIComponent(targetRunId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null); - const detailRows = Array.isArray(detailPayload?.findings) ? detailPayload.findings : []; - const expected = summarizeRows(detailRows); - const observed = panelCounts(); - const matchesRunDetail = observed.runId === targetRunId - && observed.typeCount === expected.typeCount - && observed.errorTypeCount === expected.errorTypeCount - && observed.warningTypeCount === expected.warningTypeCount - && observed.alertTypeCount === expected.alertTypeCount - && observed.errorSamples === expected.errorSamples - && observed.warningSamples === expected.warningSamples - && observed.alertSamples === expected.alertSamples - && observed.visibleRowCount === expected.alertTypeCount - && observed.visibleAlertSamples === expected.alertSamples - && observed.selectedRunErrorTag === expected.errorTypeCount - && observed.selectedRunWarningTag === expected.warningTypeCount - && observed.selectedRunRowSelected === true; - return { - ok: panelReady === true && matchesRunDetail === true, - requestedRunId, - requestedOptionPresent: Boolean(requestedOption), - targetRunId, - panelReady, - observed, - expected, - matchesRunDetail, - }; -}, { expectedRoutePrefix }); -dom.runFilterProbe = runFilterProbe; - -const consoleErrors = consoleMessages.filter((item) => item.type === "error"); -const ok = !navigationError - && httpStatus !== null - && httpStatus >= 200 - && httpStatus < 300 - && dom.shell === true - && dom.ready === true - && dom.sentinelBoundary?.datasetMatches === true - && dom.sentinelBoundary?.overviewMatches === true - && dom.sentinelBoundary?.runsPayloadMatches === true - && dom.sentinelBoundary?.runRowsMatch === true - && dom.sentinelBoundary?.routePrefixMatches === true - && dom.errorVisible !== true - && dom.trendCurve === true - && dom.chartCounts?.ok === true - && dom.chartCounts?.matchesLatestRun === true - && dom.checkScope?.matchesLatestRun === true - && dom.checkScope?.matchesRunDetail === true - && dom.selectedRunTags?.matchesRunDetail === true - && dom.runFilterProbe?.ok === true - && dom.runFilterProbe?.requestedOptionPresent === true - && dom.checkScope?.belowWorkspace === true - && dom.checkScope?.fullWidth === true - && dom.scopeLabels?.latestPointLegend === true - && dom.scopeLabels?.historicalSamples === true - && (dom.trendDotCount === 0 || (dom.trendTooltip?.visible === true && dom.trendTooltip?.hasValues === true && dom.trendTooltip?.hasTime === true)) - && dom.trendPanelCompact?.ok === true - && dom.checkRows > 0 - && dom.checkDialog?.opened === true - && dom.checkDialog?.large === true - && dom.badCardTitleCount === 0 - && dom.badCardBodyCount === 0 - && dom.timelineVisible === true - && dom.scrollModel?.independentScroll === true - && dom.scrollModel?.stickyHeader?.present === true - && dom.scrollModel?.stickyHeader?.backgroundOpaque === true - && dom.scrollModel?.stickyChecksHeader?.present === true - && dom.scrollModel?.stickyChecksHeader?.backgroundOpaque === true - && dom.panelHeights?.workspaceOk === true - && dom.panelHeights?.checksOk === true - && dom.layout?.horizontalOverflow !== true - && pageErrors.length === 0; - -console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ - ok, - url, - httpStatus, - navigationError, - navigationAttempts, - executablePath: executablePath || null, - viewport: { width, height }, - screenshotPath: captureScreenshot ? screenshotPath : null, - dom, - consoleCount: consoleMessages.length, - consoleErrorCount: consoleErrors.length, - pageErrorCount: pageErrors.length, - requestFailureCount: requestFailures.length, - consoleMessages: consoleMessages.slice(0, 8), - pageErrors: pageErrors.slice(0, 8), - requestFailures: requestFailures.slice(0, 8), - valuesRedacted: true, -})); - -await context.close().catch(() => {}); -await browser.close().catch(() => {}); -`; -} - -function parseDashboardBrowserPayload(textValue: string): Record | null { - const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__"; - const index = textValue.lastIndexOf(marker); - if (index < 0) return null; - try { - return record(JSON.parse(textValue.slice(index + marker.length).trim())); - } catch { - return null; - } -} - -function dashboardScreenshotName(options: Extract, state: SentinelCicdState): string { - const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`; - const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120); - return safe.endsWith(".png") ? safe : `${safe}.png`; -} - -function compactDashboardArtifact(artifact: Record): Record { - const transfer = record(artifact.transfer); - return { - remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null, - localPath: typeof artifact.localPath === "string" ? artifact.localPath : null, - bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null, - sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null, - verified: artifact.verified === true, - transfer: Object.keys(transfer).length === 0 ? null : { - strategy: transfer.strategy ?? null, - transport: transfer.transport ?? null, - chunks: transfer.chunks ?? null, - elapsedMs: transfer.elapsedMs ?? null, - throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null, - }, - }; -} - -function dashboardDegradedReason(result: CommandResult, transport: Record, page: Record | null, screenshotOk: boolean): string { - if (result.timedOut) return "sentinel-dashboard-command-timeout"; - if (transport.ok !== true) return "sentinel-dashboard-transport-failed"; - if (page === null) return "sentinel-dashboard-browser-output-missing"; - if (page.ok !== true) return "sentinel-dashboard-render-failed"; - if (!screenshotOk) return "sentinel-dashboard-screenshot-missing"; - return "sentinel-dashboard-unknown"; -} - -function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult { - const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; - if (releaseId !== null) args.push("--release-id", releaseId); - if (reason !== null) args.push("--reason", reason); - if (quickVerify) args.push("--quick-verify"); - const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`); - const command = `web-probe sentinel ${subcommand.join(" ")}`; - return rendered(true, command, renderAsyncJobResult({ - ok: true, - command, - node: state.spec.nodeId, - lane: state.spec.lane, - mode: "async-job", - mutation: true, - job, - next: { - status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, - wait: ["bun", "scripts/cli.ts", ...args].join(" "), - }, - valuesRedacted: true, - })); -} - -function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { - const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { - if (typeof value === "string") return [key, short(value)]; - if (Array.isArray(value)) return [key, value.slice(0, 8)]; - return [key, value]; - })); - process.stdout.write(`${JSON.stringify({ - event: "sentinel.quick-verify.progress", - at: new Date().toISOString(), - node: state.spec.nodeId, - lane: state.spec.lane, - sentinelId: state.sentinelId, - runId, - ...compactExtra, - phase, - status, - valuesRedacted: true, - })}\n`); -} - -function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record { - const startedAt = Date.now(); - const elapsedMs = () => Date.now() - startedAt; - const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId"); - const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); - const scenario = findScenario(state, scenarioId); - if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; - const commandSequence = arrayAt(scenario, "commandSequence").map(record); - const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt"); - const prompts = needsPromptSet - ? readPromptSetForScenario(scenario) - : { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } }; - if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true }; - const accountEnv = quickVerifyAccountEnv(state); - if (!accountEnv.ok) { - const findings = [{ - id: "quick-verify-account-secret-missing", - severity: "red", - count: arrayAt(accountEnv.summary, "missing").length || 1, - summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.", - missing: arrayAt(accountEnv.summary, "missing"), - valuesRedacted: true, - }]; - return recordQuickVerify(state, { - ok: false, - runId: `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`, - scenarioId, - reason, - status: "blocked", - observerId: null, - elapsedMs: 0, - businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds), - steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }], - failure: "quick-verify-account-secret-missing", - findingCount: findings.length, - findings, - promptSource: prompts.summary, - accountEnv: accountEnv.summary, - views: { - summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, - "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, - }, - warnings: [], - valuesRedacted: true, - }); - } - const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs"); - const warningBudgetSeconds = maxSeconds; - const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds"))); - const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds); - const deadline = Date.now() + hardBudgetSeconds * 1000; - const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; - printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds }); - const steps: Record[] = []; - const startArgs = [ - "web-probe", "observe", "start", - "--node", state.spec.nodeId, - "--lane", state.spec.lane, - "--target-path", stringAt(scenario, "observeTargetPath"), - "--sample-interval-ms", String(sampleIntervalMs), - "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), - "--command-timeout-seconds", "55", - ]; - const viewport = stringAtNullable(scenario, "viewport"); - if (viewport !== null) startArgs.push("--viewport", viewport); - printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) }); - const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); - steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); - const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); - printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); - if (!started.ok || observerId === null) { - const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null); - return recordQuickVerify(state, { - ok: false, - runId, - scenarioId, - reason, - status: "blocked", - observerId, - elapsedMs: elapsedMs(), - businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds), - steps, - failure: "observe-start-failed", - findingCount: findings.length, - findings, - warnings: elapsedWarnings(), - valuesRedacted: true, - }); - } - printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) }); - const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds); - steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady }); - printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() }); - if (startupReady.ok !== true) { - return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { - runId, - scenarioId, - reason, - observerId, - promptIndex: 0, - steps, - failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"), - elapsedMs: elapsedMs(), - warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()), - promptSource: prompts.summary, - })); - } - let promptIndex = 0; - const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario); - const nonBlockingCanaryWarnings: string[] = []; - for (const item of commandSequence) { - const type = stringAt(item, "type"); - const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); - for (let index = 0; index < repeat; index += 1) { - if (Date.now() >= deadline) { - printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds }); - return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { - runId, - scenarioId, - reason, - observerId, - promptIndex, - steps, - failure: "quick-verify-timeout-over-budget", - elapsedMs: elapsedMs(), - warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()), - promptSource: prompts.summary, - })); - } - const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000)); - const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55)); - const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55)); - const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))]; - if (type === "selectProvider") args.push("--provider", stringAt(item, "provider")); - if (type === "loginAccount" || type === "listSessions" || type === "logout") { - const accountId = stringAtNullable(item, "accountId"); - if (accountId !== null) args.push("--account-id", accountId); - } - if (type === "switchSessions") { - const fromAccountId = stringAtNullable(item, "fromAccountId"); - const toAccountId = stringAtNullable(item, "toAccountId"); - if (fromAccountId !== null) args.push("--from-account-id", fromAccountId); - if (toAccountId !== null) args.push("--to-account-id", toAccountId); - } - if (type === "sendPrompt") { - args.push("--text", prompts.prompts[promptIndex % prompts.prompts.length] ?? ""); - args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000)); - promptIndex += 1; - } - appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" }); - printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat }); - const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10))); - steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); - printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() }); - if (!commandResult.ok) { - if (type === "sendPrompt") { - printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); - const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); - steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult }); - printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() }); - if (delayedWaitResult.ok === true) { - const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); - nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); - printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); - if (invariantResult.ok !== true) { - return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { - runId, - scenarioId, - reason, - observerId, - promptIndex, - steps, - failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"), - promptSource: prompts.summary, - elapsedMs: elapsedMs(), - warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()), - })); - } - continue; - } - } - return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { - runId, - scenarioId, - reason, - observerId, - promptIndex, - steps, - failure: `observe-command-${type}-failed`, - elapsedMs: elapsedMs(), - warnings: elapsedWarnings(), - promptSource: prompts.summary, - })); - } - if (type === "sendPrompt") { - printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); - const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); - steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult }); - printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() }); - if (waitResult.ok !== true) { - return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { - runId, - scenarioId, - reason, - observerId, - promptIndex, - steps, - failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"), - promptSource: prompts.summary, - elapsedMs: elapsedMs(), - warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()), - })); - } - const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); - nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); - printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); - if (invariantResult.ok !== true) { - return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { - runId, - scenarioId, - reason, - observerId, - promptIndex, - steps, - failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"), - promptSource: prompts.summary, - elapsedMs: elapsedMs(), - warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()), - })); - } - } - } - } - printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) }); - const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120)); - steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); - printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); - const indexEntry = readLocalObserveIndex(observerId); - const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30)); - const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); - const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); - const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame); - const artifactSummaryRecord = record(artifactSummary); - const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; - const findings = mergeFindingRecords(artifactFindings, controlFindings); - const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); - const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."]; - const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; - const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds); - printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() }); - return recordQuickVerify(state, { - ok, - runId, - scenarioId, - reason, - status: ok ? "analyzed" : "blocked", - observerId, - elapsedMs: elapsedMs(), - businessStatus, - stateDir: indexEntry?.stateDir ?? null, - reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), - findingCount: findings.length, - artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, - failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null, - promptSource: prompts.summary, - accountEnv: accountEnv.summary, - steps, - analysis: artifactSummary, - views: { - summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, - "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, - "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, - "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, - }, - findings, - screenshot: record(artifactSummary).screenshot, - publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), - warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, elapsedWarnings()), - valuesRedacted: true, - }); -} - -function sessionInvarianceChecksByRound(scenario: Record): Map[]> { - const checks = new Map[]>(); - const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : []; - for (const item of items) { - const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null; - if (afterRound === null || afterRound < 0) continue; - const list = checks.get(afterRound) ?? []; - list.push(item); - checks.set(afterRound, list); - } - return checks; -} - -function runQuickVerifySessionInvarianceChecks( - state: SentinelCicdState, - observerId: string, - checks: readonly Record[], - deadline: number, - promptIndex: number, - steps: Record[], -): Record { - const warnings: string[] = []; - for (const check of checks) { - const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`; - const blocking = check.blocking === true; - const commands: { readonly type: string; readonly enabled: boolean }[] = [ - { type: "refreshCurrentSession", enabled: check.refreshCurrent === true }, - { type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true }, - { type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false }, - ]; - for (const command of commands) { - if (!command.enabled) continue; - const args = [ - "web-probe", "observe", "command", observerId, - "--node", state.spec.nodeId, - "--lane", state.spec.lane, - "--type", command.type, - "--after-round", String(promptIndex), - "--wait-ms", "55000", - "--command-timeout-seconds", String(remainingSeconds(deadline, 55)), - ]; - const severity = nonEmptyString(check.severity); - const findingId = nonEmptyString(check.findingId); - const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange); - const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy); - if (severity !== null) args.push("--severity", severity); - if (findingId !== null) args.push("--finding-id", findingId); - if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange); - if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy); - if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready"); - args.push(check.blocking === true ? "--blocking" : "--non-blocking"); - const result = runChildCli(args, remainingSeconds(deadline, 60)); - steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result }); - if (!result.ok) { - if (!blocking) { - warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`); - continue; - } - return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true }; - } - } - } - return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true }; -} - -function appendScenarioObserveCommandArgs(args: string[], item: Record, options: { readonly skipText?: boolean } = {}): void { - const mappings: readonly (readonly [string, string])[] = [ - ["path", "--path"], - ["label", "--label"], - ["sessionId", "--session-id"], - ["provider", "--provider"], - ["accountId", "--account-id"], - ["fromAccountId", "--from-account-id"], - ["toAccountId", "--to-account-id"], - ["sourceId", "--source-id"], - ["fileRef", "--file-ref"], - ["filename", "--filename"], - ["taskRef", "--task-ref"], - ["taskId", "--task-id"], - ["task", "--task"], - ["field", "--field"], - ["link", "--link"], - ["title", "--title"], - ["body", "--body"], - ["status", "--status"], - ["hwpodId", "--hwpod-id"], - ["nodeId", "--node-id"], - ["workspaceRoot", "--workspace-root"], - ["root", "--root"], - ]; - for (const [key, flag] of mappings) { - if (args.includes(flag)) continue; - const value = stringAtNullable(item, key); - if (value !== null) args.push(flag, value); - } - if (options.skipText !== true && !args.includes("--text")) { - const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value"); - if (text !== null) args.push("--text", text); - } - if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready"); -} - -function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { - readonly runId: string; - readonly scenarioId: string; - readonly reason: string; - readonly observerId: string; - readonly promptIndex: number; - readonly steps: readonly Record[]; - readonly failure: string; - readonly promptSource?: Record; - readonly elapsedMs?: number; - readonly warnings?: readonly unknown[]; -}): Record { - const cleanupSteps: Record[] = []; - if (input.promptIndex > 0) { - const cancel = runChildCli([ - "web-probe", "observe", "command", input.observerId, - "--node", state.spec.nodeId, - "--lane", state.spec.lane, - "--type", "cancel", - "--wait-ms", "55000", - "--command-timeout-seconds", "55", - ], 60); - cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result }); - } - const stop = runChildCli([ - "web-probe", "observe", "stop", input.observerId, - "--node", state.spec.nodeId, - "--lane", state.spec.lane, - "--force", - "--command-timeout-seconds", "55", - ], 30); - cleanupSteps.push({ phase: "observe-stop-after-failure", ok: stop.ok, result: stop.result }); - const analysis = runChildCli([ - "web-probe", "observe", "analyze", input.observerId, - "--node", state.spec.nodeId, - "--lane", state.spec.lane, - "--command-timeout-seconds", "55", - ], 60); - cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result }); - const indexEntry = readLocalObserveIndex(input.observerId); - const artifactSummary = indexEntry === null - ? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true } - : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30); - const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30); - const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30); - const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame); - const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame); - const artifactSummaryRecord = record(artifactSummary); - const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; - const findings = mergeFindingRecords(artifactFindings, controlFindings); - const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); - const recoveredWaitFailure = durableBusinessTurn - && isRecoverableQuickVerifyWaitFailure(input.failure) - && record(artifactSummary).ok === true - && controlFindings.length === 0 - && blockingFindings.length === 0; - const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, numberAt(state.cicd, "targetValidation.maxSeconds")); - return { - ok: recoveredWaitFailure, - runId: input.runId, - scenarioId: input.scenarioId, - reason: input.reason, - status: recoveredWaitFailure ? "analyzed" : "blocked", - observerId: input.observerId, - elapsedMs: input.elapsedMs ?? null, - businessStatus, - stateDir: indexEntry?.stateDir ?? null, - reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), - findingCount: findings.length, - artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, - failure: recoveredWaitFailure ? null : input.failure, - promptSource: input.promptSource, - steps: [...input.steps, ...cleanupSteps], - analysis: artifactSummary, - views: { - summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, - "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, - "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, - "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, - }, - findings, - screenshot: record(artifactSummary).screenshot, - publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), - warnings: mergeWarnings( - Array.isArray(input.warnings) ? input.warnings : [], - recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [], - targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", numberAt(state.cicd, "targetValidation.maxSeconds")), - ), - valuesRedacted: true, - }; -} - -function recordQuickVerify(state: SentinelCicdState, payload: Record): Record { - const views = compactQuickVerifyRecordViews(record(payload.views)); - const summary = { - reason: payload.reason, - status: payload.status, - businessStatus: payload.businessStatus ?? null, - elapsedMs: payload.elapsedMs, - failure: payload.failure, - warnings: Array.isArray(payload.warnings) ? payload.warnings : [], - analysis: compactQuickVerifyRecordAnalysis(payload.analysis), - promptSource: payload.promptSource, - steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [], - valuesRedacted: true, - }; - const recordResult = callSentinelService(state, "POST", "/api/runs/record", { - runId: payload.runId, - scenarioId: payload.scenarioId, - status: payload.status, - observerId: payload.observerId, - stateDir: payload.stateDir, - reportJsonSha256: payload.reportJsonSha256, - findingCount: payload.findingCount, - artifactCount: payload.artifactCount, - summary, - businessStatus: payload.businessStatus ?? null, - findings: payload.findings, - views, - screenshot: payload.screenshot, - publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"), - maintenance: payload.reason === "maintenance-stop", - valuesRedacted: true, - }, 60); - return withWarnings({ ...payload, views, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : ["quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]); -} - -function compactQuickVerifyRecordViews(views: Record): Record { - const compacted: Record = {}; - for (const [key, value] of Object.entries(views)) { - const item = record(value); - const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000; - compacted[key] = { - ...item, - renderedText: boundQuickVerifyRecordText(item.renderedText, limit), - valuesRedacted: true, - }; - } - return compacted; -} - -function compactQuickVerifyRecordAnalysis(value: unknown): Record | null { - const item = record(value); - if (Object.keys(item).length === 0) return null; - return { - ok: item.ok === true ? true : item.ok === false ? false : null, - reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null, - reason: stringAtNullable(item, "reason"), - stateDir: stringAtNullable(item, "stateDir"), - reportJsonSha256: stringAtNullable(item, "reportJsonSha256"), - reportMdSha256: stringAtNullable(item, "reportMdSha256"), - findingCount: numberAtNullable(item, "findingCount"), - artifactCount: numberAtNullable(item, "artifactCount"), - counts: compactQuickVerifyRecordCounts(record(item.counts)), - screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)), - findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [], - pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [], - valuesRedacted: true, - }; -} - -function compactQuickVerifyRecordCounts(value: Record): Record { - return { - samples: numberAtNullable(value, "samples"), - control: numberAtNullable(value, "control"), - network: numberAtNullable(value, "network"), - console: numberAtNullable(value, "console"), - errors: numberAtNullable(value, "errors"), - artifacts: numberAtNullable(value, "artifacts"), - valuesRedacted: true, - }; -} - -function compactQuickVerifyRecordScreenshot(value: Record): Record | null { - if (Object.keys(value).length === 0) return null; - return { - path: stringAtNullable(value, "path"), - sha256: stringAtNullable(value, "sha256"), - bytes: numberAtNullable(value, "bytes"), - valuesRedacted: true, - }; -} - -function compactQuickVerifyRecordFinding(value: unknown): Record { - const item = record(value); - return { - id: stringAtNullable(item, "id"), - kind: stringAtNullable(item, "kind"), - code: stringAtNullable(item, "code"), - severity: stringAtNullable(item, "severity"), - level: stringAtNullable(item, "level"), - count: numberAtNullable(item, "count"), - summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220), - rootCause: boundQuickVerifyRecordText(item.rootCause, 140), - rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90), - rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40), - nextAction: boundQuickVerifyRecordText(item.nextAction, 240), - evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence), - timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100), - timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60), - timingAlert: item.timingAlert === true, - blocking: item.blocking === true, - valuesRedacted: true, - }; -} - -function compactQuickVerifyFindingEvidence(value: unknown): string | null { - const item = record(value); - if (Object.keys(item).length === 0) return null; - const keys = [ - "http404Count", - "responseErrorCount", - "requestFailedCount", - "statuses", - "afterProjectedSeqs", - "sinceSeqs", - "traceIds", - "maxFallbackRatio", - "maxFallbackTitleCount", - "overThresholdSampleCount", - "majorityFallbackSampleCount", - ]; - const compact: Record = {}; - for (const key of keys) { - const raw = item[key]; - if (raw === null || raw === undefined) continue; - compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw; - } - return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240); -} - -function compactQuickVerifyRecordStep(value: unknown): Record { - const item = record(value); - return { - phase: stringAtNullable(item, "phase"), - ok: item.ok === true ? true : item.ok === false ? false : null, - promptIndex: numberAtNullable(item, "promptIndex"), - checkId: stringAtNullable(item, "checkId"), - failure: stringAtNullable(item, "failure"), - result: compactQuickVerifyRecordStepResult(record(item.result)), - valuesRedacted: true, - }; -} - -function compactQuickVerifyRecordStepResult(value: Record): Record { - return { - ok: value.ok === true ? true : value.ok === false ? false : null, - status: stringAtNullable(value, "status"), - view: stringAtNullable(value, "view"), - exitCode: numberAtNullable(value, "exitCode"), - timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, - stdoutBytes: numberAtNullable(value, "stdoutBytes"), - stderrBytes: numberAtNullable(value, "stderrBytes"), - stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240), - stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240), - valuesRedacted: true, - }; -} - -function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null { - if (typeof value !== "string") return null; - if (value.length <= maxChars) return value; - return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`; -} - -function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { - const namespace = stringAt(state.runtime, "namespace"); - const serviceName = stringAt(state.runtime, "serviceName"); - const servicePort = numberAt(state.runtime, "servicePort"); - const deploymentName = stringAt(state.runtime, "deploymentName"); - const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`; - if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") { - return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url); - } - const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; - const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); - const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); - const postScript = [ - "const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');", - "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", - `const url = 'http://127.0.0.1:${servicePort}' + path;`, - "fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {", - " const text = await response.text();", - " process.stdout.write(text);", - " if (!response.ok) process.exit(22);", - "}).catch((error) => {", - " console.error(error && error.stack ? error.stack : String(error));", - " process.exit(23);", - "});", - ].join(" "); - const script = method === "GET" - ? `kubectl get --raw ${shellQuote(proxyPath)}` - : [ - "set -eu", - `kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`, - ].join("\n"); - const maxAttempts = method === "GET" ? 3 : 1; - const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); - const attempts: Record[] = []; - let result: CommandResult | null = null; - let parsed: Record | null = null; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); - parsed = parseJsonObject(result.stdout); - attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true }); - if (result.exitCode === 0) break; - } - const compactBodyJson = compactSentinelServiceBodyJson(parsed); - return { - ok: result?.exitCode === 0, - method, - path: pathWithQuery, - internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, - httpStatus: result?.exitCode === 0 ? 200 : null, - bodyJson: record(compactBodyJson), - bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "", - bodyBytes: Buffer.byteLength(result?.stdout ?? ""), - error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000), - proxyPath, - result: result === null ? null : compactCommand(result), - attempts, - valuesRedacted: true, - }; -} - -function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number, url: string): Record { - const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); - const fetchScript = [ - "const method = process.env.SENTINEL_METHOD || 'GET';", - "const url = process.env.SENTINEL_URL || '';", - "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", - "const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);", - "const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);", - "const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;", - "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", - "(async () => {", - " let lastError = null;", - " for (let attempt = 1; attempt <= attempts; attempt += 1) {", - " try {", - " const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });", - " const text = await response.text();", - " process.stdout.write(text);", - " if (!response.ok) process.exit(22);", - " process.exit(0);", - " } catch (error) {", - " lastError = error;", - " console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));", - " if (attempt < attempts && delayMs > 0) await sleep(delayMs);", - " }", - " }", - " console.error(lastError && lastError.stack ? lastError.stack : String(lastError));", - " process.exit(23);", - "})().catch((error) => {", - " console.error(error && error.stack ? error.stack : String(error));", - " process.exit(24);", - "});", - ].join(" "); - const attempts = method === "GET" ? 6 : 3; - const retryDelayMs = 1000; - const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70)); - const result = runCommand(["node", "-e", fetchScript], repoRoot, { - timeoutMs: attemptTimeoutSeconds * 1000, - env: { - ...process.env, - SENTINEL_METHOD: method, - SENTINEL_URL: url, - SENTINEL_BODY_B64: bodyB64, - SENTINEL_ATTEMPTS: String(attempts), - SENTINEL_RETRY_DELAY_MS: String(retryDelayMs), - }, - }); - const parsed = parseJsonObject(result.stdout); - const compactBodyJson = compactSentinelServiceBodyJson(parsed); - return { - ok: result.exitCode === 0, - method, - path: pathWithQuery, - internalUrl: url, - httpStatus: result.exitCode === 0 ? 200 : null, - bodyJson: record(compactBodyJson), - bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "", - bodyBytes: Buffer.byteLength(result.stdout), - error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000), - proxyPath: null, - result: compactCommand(result), - attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }], - transport: "direct-service", - valuesRedacted: true, - }; -} - -function compactSentinelServiceBodyJson(value: Record | null): unknown { - if (value === null || typeof value.renderedText !== "string") return value; - return { - ...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]), - run: pickFields(record(value.run), ["id", "scenario_id", "status", "node", "lane", "observer_id", "state_dir", "report_json_sha256", "finding_count", "artifact_count", "maintenance", "created_at", "updated_at"]), - summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]), - findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [], - renderedText: value.renderedText, - valuesRedacted: true, - }; -} - -function pickFields(value: Record, keys: readonly string[]): Record { - const picked: Record = {}; - for (const key of keys) { - if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key]; - } - return picked; -} - -function clipTail(value: string, maxChars: number): string { - return value.length <= maxChars ? value : value.slice(-maxChars); -} - -function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record { - const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); - const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`; - const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000)); - const js = [ - "const url=process.env.REQ_URL||'';", - "const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);", - "let out;", - "try{", - " const controller=new AbortController();", - " const timer=setTimeout(()=>controller.abort(), timeoutMs);", - " const started=Date.now();", - " const res=await fetch(url,{signal:controller.signal});", - " const text=await res.text();", - " clearTimeout(timer);", - " let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}", - " out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};", - "}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}", - "console.log(JSON.stringify(out));", - ].join(""); - const result = runCommand(["bun", "-e", js], repoRoot, { - timeoutMs: timeoutMs + 2000, - env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) }, - }); - const parsed = parseJsonObject(result.stdout); - return { - ok: result.exitCode === 0 && parsed?.ok === true, - method: "GET", - path: pathWithQuery, - publicUrl: url, - httpStatus: parsed?.httpStatus ?? null, - bodyJson: record(parsed?.bodyJson), - bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "", - bodyBytes: parsed?.bodyBytes ?? null, - error: parsed?.error ?? null, - result: compactCommand(result), - valuesRedacted: true, - }; -} - -function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { - const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl"); - const hostname = stringAt(state.publicExposure, "hostname"); - const expectedA = stringAt(state.publicExposure, "expectedA"); - const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`; - const script = [ - "set +e", - `host=${shellQuote(hostname)}`, - `expected=${shellQuote(expectedA)}`, - `url=${shellQuote(probeUrl)}`, - "dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)", - "headers=$(mktemp)", - "body=$(mktemp)", - "writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)", - "curl_rc=$?", - "body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')", - "node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'", - "const fs=require('node:fs');", - "const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);", - "const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);", - "const status=Number(statusRaw||0);", - "const ssl=Number(sslRaw||-1);", - "const addrs=dns?dns.split(',').filter(Boolean):[];", - "const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();", - "const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');", - "let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}", - "const authCovered=status===401||status===403||status>=200&&status<300;", - "const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;", - "const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));", - "const dnsMatches=addrs.includes(expected);", - "console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));", - "NODE", - ].join("\n"); - const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; -} - -function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record { - const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); - const rootUrl = `${publicBaseUrl}/`; - const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`; - const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`; - const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`; - const script = [ - "set +e", - `root_url=${shellQuote(rootUrl)}`, - `css_url=${shellQuote(cssUrl)}`, - `js_url=${shellQuote(jsUrl)}`, - `vue_url=${shellQuote(vueUrl)}`, - "root_body=$(mktemp)", - "css_body=$(mktemp)", - "js_body=$(mktemp)", - "vue_body=$(mktemp)", - "root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?", - "css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?", - "js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?", - "vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?", - "node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'", - "const fs=require('node:fs');", - "const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);", - "function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}", - "const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);", - "const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&root.includes('monitor-web-bootstrap');", - "const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.includes('monitor-shell')&&css.includes('workspace-grid')&&css.includes('trend-stage')&&css.length>1000;", - "const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.includes('createApp')&&js.includes('/api/overview')&&js.includes('data-monitor-trend-curve')&&js.length>1000;", - "const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.includes('createApp')&&vue.length>80000;", - "console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contract:root.includes('draft-2026-06-27-p11-monitor-web-observability-dashboard')},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css),workspaceGrid:css.includes('workspace-grid'),trendStage:css.includes('trend-stage')},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js),vueApp:js.includes('createApp'),apiClient:js.includes('/api/overview'),trend:js.includes('data-monitor-trend-curve')},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue),runtime:vue.includes('createApp')},valuesRedacted:true}));", - "NODE", - ].join("\n"); - const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; -} - function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { @@ -3890,22 +1989,36 @@ function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: n function readSentinelSecretSourceValue(source: Record): Record { const sourceRef = stringAt(source, "sourceRef"); const sourceKey = stringAt(source, "sourceKey"); + const sourceLine = numberAtNullable(source, "sourceLine"); const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); if (!existsSync(sourcePath)) return { ok: false, error: "secret-source-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; - const values = parseEnvFile(readFileSync(sourcePath, "utf8")); - const value = values[sourceKey]; - if (value === undefined || value.length === 0) return { ok: false, error: "secret-source-key-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + const textValue = readFileSync(sourcePath, "utf8"); + const value = sourceLine === null ? parseEnvFile(textValue)[sourceKey] : textValue.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, ""); + if (value === undefined || value.length === 0) return { ok: false, error: sourceLine === null ? "secret-source-key-missing" : "secret-source-line-missing", sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; const format = stringAtNullable(source, "format"); - if (format === null) return { ok: true, value, sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + if (format === null) return { ok: true, value, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; if (format === "web-account-json") { - const username = stringAtNullable(source, "username"); - if (username === null) return { ok: false, error: "web-account-json-username-missing", sourceRef, sourceKey, sourcePath: displayPath(sourcePath), valuesRedacted: true }; - return { ok: true, value: JSON.stringify({ username, password: value }), sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + const username = readSentinelWebAccountUsername(source); + if (!username.ok) return { ok: false, error: username.error, sourceRef, sourceKey, sourceLine, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + return { ok: true, value: JSON.stringify({ username: username.value, password: value }), sourceRef, sourceKey, sourceLine, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; } return { ok: false, error: "unsupported-secret-source-format", sourceRef, sourceKey, format, sourcePath: displayPath(sourcePath), valuesRedacted: true }; } +function readSentinelWebAccountUsername(source: Record): { ok: true; value: string } | { ok: false; error: string } { + const username = stringAtNullable(source, "username"); + if (username !== null) return { ok: true, value: username }; + const sourceRef = stringAtNullable(source, "usernameSourceRef"); + const sourceLine = numberAtNullable(source, "usernameSourceLine"); + if (sourceRef === null || sourceLine === null) return { ok: false, error: "web-account-json-username-missing" }; + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + if (!existsSync(sourcePath)) return { ok: false, error: "web-account-json-username-source-missing" }; + const value = readFileSync(sourcePath, "utf8").split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? ""; + return value.length === 0 ? { ok: false, error: "web-account-json-username-line-missing" } : { ok: true, value }; +} + function applySentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { const material = readSentinelFrpcMaterial(state); if (!material.ok) return { ok: false, hostname: stringAt(state.publicExposure, "hostname"), material, valuesRedacted: true }; @@ -4185,503 +2298,8 @@ function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: numbe return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; } -function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record { - if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true }; - const script = [ - "set -eu", - `state_dir=${shellQuote(stateDir)}`, - "node - \"$state_dir\" <<'NODE'", - "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", - "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", - "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", - "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", - "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", - "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", - "let artifactCount=0; let screenshot=null;", - "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", - "walk(stateDir);", - "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", - "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", - "console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", - "NODE", - ].join("\n"); - const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; -} - -function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: string, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record { - const observations: Record[] = []; - const indexEntry = readLocalObserveIndex(observerId); - if (indexEntry === null) { - return { - ok: false, - failure: "observe-index-entry-missing", - observerId, - valuesRedacted: true, - }; - } - const pollSleepMs = Math.max(250, Math.min(500, Math.trunc(pollIntervalMs / 2) || 250)); - while (Date.now() < deadline) { - const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now())); - const script = quickVerifyObserverStartupWaitScript(indexEntry.stateDir, waitMs, pollSleepMs); - const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 }); - const payload = parseJsonObject(result.stdout); - if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); - const terminalPayload = { - observerId, - stateDir: indexEntry.stateDir, - status: typeof payload?.status === "string" ? payload.status : null, - heartbeatStatus: typeof payload?.heartbeatStatus === "string" ? payload.heartbeatStatus : null, - startup: record(payload?.startup), - observations: observations.slice(-6), - waitResult: compactCommand(result), - valuesRedacted: true, - }; - if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-startup-wait-chunk-timeout") { - return { - ok: false, - failure: text(payload?.failure ?? "quick-verify-startup-artifact-wait-failed"), - ...terminalPayload, - }; - } - if (payload.ok === true) return { ok: true, ...terminalPayload }; - } - return { - ok: false, - failure: "quick-verify-timeout-over-budget", - observerId, - stateDir: indexEntry.stateDir, - observations: observations.slice(-6), - warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for the observe runner startup to finish before sending the first command.`], - valuesRedacted: true, - }; -} - -function quickVerifyObserverStartupWaitScript(stateDir: string, timeoutMs: number, pollSleepMs: number): string { - return [ - "set -eu", - `state_dir=${shellQuote(stateDir)}`, - `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, - `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, - "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", - "node - \"$state_dir\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", - "const fs = require('node:fs');", - "const path = require('node:path');", - "const dir = process.argv[2];", - "const timeoutMs = Number(process.argv[3]);", - "const pollMs = Number(process.argv[4]);", - "const startedAt = Date.now();", - "const startupIds = ['startup-login', 'startup-goto', 'startup-observer-goto'];", - "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", - "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", - "const clip = (value, limit = 160) => value == null ? null : String(value).replace(/\\s+/gu, ' ').trim().slice(0, limit);", - "const norm = (value) => String(value || '').trim().toLowerCase().replace(/_/gu, '-');", - "const terminal = new Set(['failed', 'force-stopped', 'stopped', 'abandoned', 'completed']);", - "function commandEvents(control, id) { return control.filter((item) => item && item.commandId === id); }", - "function lastPhase(control, id) { return commandEvents(control, id).filter((item) => typeof item.phase === 'string').slice(-1)[0]?.phase || null; }", - "function firstFailedStartup(control) { return control.filter((item) => item && startupIds.includes(item.commandId) && item.phase === 'failed').slice(-1)[0] || null; }", - "function rowFor() {", - " const heartbeat = readJson('heartbeat.json') || {};", - " const manifest = readJson('manifest.json') || {};", - " const control = readJsonl('control.jsonl');", - " const phases = Object.fromEntries(startupIds.map((id) => [id, lastPhase(control, id)]));", - " const failed = firstFailedStartup(control);", - " const heartbeatStatus = norm(heartbeat.status || manifest.status);", - " const ready = startupIds.every((id) => phases[id] === 'completed') && heartbeatStatus === 'running';", - " const terminalBeforeReady = !ready && terminal.has(heartbeatStatus);", - " const degraded = control.filter((item) => item && item.type === 'observer-startup-degraded').slice(-1)[0] || null;", - " return {", - " ok: ready,", - " status: ready ? 'startup-ready' : terminalBeforeReady ? 'startup-terminal' : 'startup-waiting',", - " heartbeatStatus,", - " startup: { phases, failedCommandId: failed?.commandId || null, failedType: failed?.type || null, failedMessage: clip(failed?.detail?.error?.message || failed?.detail?.error || failed?.error?.message), observerStartupDegraded: !!degraded, degradedReason: clip(degraded?.reason || degraded?.result?.failureKind || degraded?.result?.reason), sampleSeq: heartbeat.sampleSeq ?? null, commandSeq: heartbeat.commandSeq ?? null, currentUrl: clip(heartbeat.currentUrl, 180), observerUrl: clip(heartbeat.observerUrl, 180), valuesRedacted: true },", - " valuesRedacted: true", - " };", - "}", - "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", - "(async () => {", - " const observations = [];", - " while (Date.now() - startedAt <= timeoutMs) {", - " const row = rowFor();", - " observations.push(row);", - " if (row.ok === true) { console.log(JSON.stringify({ ok: true, ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", - " if (row.startup.failedCommandId) { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-command-failed', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", - " if (row.status === 'startup-terminal') { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-terminal', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", - " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", - " }", - " const row = rowFor();", - " observations.push(row);", - " console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-chunk-timeout', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", - "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", - "NODE", - ].join("\n"); -} - -function collectObserveView(state: SentinelCicdState, observerId: string, view: "turn-summary" | "trace-frame", turn: number | null, timeoutSeconds: number): Record { - const args = ["web-probe", "observe", "collect", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--view", view, "--command-timeout-seconds", String(Math.max(5, Math.min(timeoutSeconds, 55))), "--raw", "--compact-raw"]; - if (turn !== null) args.push("--turn", String(turn)); - const result = runChildCli(args, timeoutSeconds); - const payload = cliDataPayload(result.parsed); - const collect = record(payload.collect); - return { - ok: result.ok && result.parsed !== null && payload.ok !== false && collect.ok !== false, - view, - renderedText: typeof collect.renderedText === "string" ? collect.renderedText : typeof payload.renderedText === "string" ? payload.renderedText : String(record(result.result).stdoutTail ?? record(result.result).stdoutPreview ?? ""), - collect, - payload, - result: result.result, - valuesRedacted: true, - }; -} - -function runChildCli(args: string[], timeoutSeconds: number, input?: string, env?: NodeJS.ProcessEnv): ChildCliResult { - const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { - input, - env: env === undefined ? undefined : { ...process.env, ...env }, - timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000, - }); - return { - ok: result.exitCode === 0 && !result.timedOut, - parsed: parseJsonObject(result.stdout), - result: compactCommandWithTail(result), - }; -} - -function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number, chunkSeconds = 45): Record { - const observations: Record[] = []; - const indexEntry = readLocalObserveIndex(observerId); - if (indexEntry === null) { - return { - ok: false, - failure: "observe-index-entry-missing", - round: promptIndex, - observerId, - valuesRedacted: true, - }; - } - const pollSleepMs = Math.max(1000, Math.min(3000, Math.trunc(pollIntervalMs * 2) || 1000)); - while (Date.now() < deadline) { - const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now())); - const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs); - const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 }); - const payload = parseJsonObject(result.stdout); - if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); - const status = typeof payload?.status === "string" ? payload.status : null; - const terminalPayload = { - round: promptIndex, - status, - traceId: payload?.traceId ?? null, - finalResponseEmpty: payload?.finalResponseEmpty === true, - composerReadyForTurn: payload?.composerReadyForTurn === true, - composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null, - observations: observations.slice(-6), - waitResult: compactCommand(result), - valuesRedacted: true, - }; - if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") { - const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex); - if (fallback.ok === true) { - return { - ok: true, - ...terminalPayload, - status: stringAtNullable(fallback, "status") ?? status, - traceId: stringAtNullable(fallback, "traceId") ?? payload?.traceId ?? null, - finalResponseEmpty: false, - fallback, - warnings: ["quick verify artifact wait command failed, but bounded turn-summary artifacts show this round completed; continuing validation."], - }; - } - return { - ok: false, - failure: text(payload?.failure ?? "quick-verify-artifact-wait-failed"), - ...terminalPayload, - fallback, - }; - } - if (payload.ok === false && payload.failure === "quick-verify-wait-chunk-timeout") { - const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex); - if (fallback.ok === true) { - return { - ok: true, - ...terminalPayload, - status: stringAtNullable(fallback, "status") ?? status, - traceId: stringAtNullable(fallback, "traceId") ?? payload.traceId ?? null, - finalResponseEmpty: false, - fallback, - warnings: ["quick verify wait chunk timed out, but bounded turn-summary artifacts show this round completed; continuing validation."], - }; - } - } - if (payload.ok === true) return { ok: true, ...terminalPayload }; - if (isQuickVerifyTurnSuccessful(status)) { - if (payload?.finalResponseEmpty !== true) return { ok: true, ...terminalPayload }; - continue; - } - if (isQuickVerifyTurnTerminal(status)) { - return { - ok: false, - failure: "observe-turn-terminal-non-success", - ...terminalPayload, - }; - } - } - return { - ok: false, - failure: "quick-verify-timeout-over-budget", - round: promptIndex, - observations: observations.slice(-6), - warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for a submitted turn to become terminal; investigate Code Agent multi-round continuity before retrying.`], - valuesRedacted: true, - }; -} - -function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, timeoutMs: number, pollSleepMs: number): string { - return [ - "set -eu", - `state_dir=${shellQuote(stateDir)}`, - `prompt_index=${shellQuote(String(promptIndex))}`, - `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, - `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, - "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", - "node - \"$state_dir\" \"$prompt_index\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", - "const fs = require('node:fs');", - "const path = require('node:path');", - "const dir = process.argv[2];", - "const promptIndex = Number(process.argv[3]);", - "const timeoutMs = Number(process.argv[4]);", - "const pollMs = Number(process.argv[5]);", - "const startedAt = Date.now();", - "const short = (value, limit = 160) => String(value || '').replace(/\\s+/gu, ' ').trim().slice(0, limit);", - "const textOf = (value) => String(value?.text || value?.textPreview || value?.preview || '');", - "const arr = (value) => Array.isArray(value) ? value : [];", - "const unique = (values) => Array.from(new Set(values.filter(Boolean)));", - "const numOrNull = (value) => { const n = Number(value); return Number.isFinite(n) ? n : null; };", - "const tsMs = (value) => { const ms = Date.parse(String(value || '')); return Number.isFinite(ms) ? ms : null; };", - "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", - "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", - "const readJsonlTail = (rel, maxBytes = 2000000) => {", - " try {", - " const file = path.join(dir, rel);", - " const stat = fs.statSync(file);", - " const start = Math.max(0, stat.size - maxBytes);", - " const length = stat.size - start;", - " const fd = fs.openSync(file, 'r');", - " try {", - " const buffer = Buffer.alloc(length);", - " fs.readSync(fd, buffer, 0, length, start);", - " const lines = buffer.toString('utf8').split(/\\r?\\n/u);", - " if (start > 0) lines.shift();", - " return lines.filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);", - " } finally {", - " fs.closeSync(fd);", - " }", - " } catch {", - " return [];", - " }", - "};", - "const readDone = (id) => id ? readJson(path.join('commands', 'done', `${id}.json`)) : null;", - "const readFailed = (id) => id ? readJson(path.join('commands', 'failed', `${id}.json`)) : null;", - "function sessionIdFromUrl(value) { const match = String(value || '').match(/\\/workbench\\/sessions\\/(ses_[A-Za-z0-9_-]+)/u); return match ? match[1] : null; }", - "function commandSessionId(item) { const done = readDone(item?.commandId); return item?.sessionId || item?.detail?.sessionId || item?.input?.sessionId || item?.result?.sessionId || done?.result?.sessionId || done?.result?.observer?.sessionId || sessionIdFromUrl(item?.afterUrl) || sessionIdFromUrl(item?.detail?.afterUrl) || sessionIdFromUrl(done?.result?.afterUrl) || null; }", - "function commandTextHash(item) { const done = readDone(item?.commandId); return item?.detail?.textHash || item?.input?.textHash || item?.result?.textHash || done?.result?.textHash || null; }", - "function firstTraceId(value) { const match = String(value || '').match(/\\btrc_[A-Za-z0-9_-]+\\b/u); return match ? match[0] : null; }", - "function commandTraceId(item) { const done = readDone(item?.commandId); return item?.traceId || item?.detail?.chatSubmit?.traceId || item?.detail?.traceId || item?.input?.traceId || item?.result?.chatSubmit?.traceId || done?.result?.chatSubmit?.traceId || done?.result?.traceId || null; }", - "function itemTraceId(item) { return item?.traceId || firstTraceId(textOf(item)) || null; }", - "function completedNewSessionIdsBefore(control, ts) { const limit = tsMs(ts); return control.filter((item) => item.type === 'newSession' && item.phase === 'completed').filter((item) => limit === null || tsMs(item.ts) === null || tsMs(item.ts) <= limit).map(commandSessionId).filter(Boolean); }", - "function authoritativeSessionIdForPrompts(control, prompts) { const ids = completedNewSessionIdsBefore(control, prompts[0]?.firstTs || null); return ids.slice(-1)[0] || unique(prompts.map((item) => item.sessionId))[0] || null; }", - "function promptCommands(control) {", - " const map = new Map();", - " for (const item of control) {", - " if (item.type !== 'sendPrompt' || !['started', 'completed', 'failed'].includes(item.phase)) continue;", - " const id = item.commandId || item.seq || String(map.size + 1);", - " const existing = map.get(id) || {};", - " map.set(id, { ...existing, ...item, input: { ...(existing.input || {}), ...(item.input || {}) }, sessionId: existing.sessionId || commandSessionId(item), traceId: existing.traceId || commandTraceId(item), textHash: existing.textHash || commandTextHash(item), firstTs: existing.firstTs || item.ts, lastTs: item.ts });", - " }", - " const prompts = Array.from(map.values()).filter((item) => tsMs(item.firstTs) !== null).sort((a, b) => tsMs(a.firstTs) - tsMs(b.firstTs));", - " const sessionId = authoritativeSessionIdForPrompts(control, prompts);", - " if (!sessionId) return prompts;", - " const scoped = prompts.filter((item) => item.sessionId === sessionId);", - " return scoped.length > 0 ? scoped : prompts;", - "}", - "function segmentFor(samples, prompts, index) { const start = tsMs(prompts[index]?.firstTs); const end = index + 1 < prompts.length ? tsMs(prompts[index + 1].firstTs) : Infinity; return samples.filter((sample) => { const ms = tsMs(sample.ts); return ms !== null && ms >= start && ms < end; }); }", - "function entryGroups(sample) { return [...arr(sample.turns).map((item) => ({ group: 'turn', item })), ...arr(sample.traceRows).map((item) => ({ group: 'traceRow', item })), ...arr(sample.messages).map((item) => ({ group: 'message', item }))]; }", - "function traceIdsFromSamples(items) { const ids = []; for (const sample of items) for (const entry of entryGroups(sample)) { const id = itemTraceId(entry.item); if (id) ids.push(id); } return unique(ids); }", - "function chooseTraceId(segment, prompt) { const promptTraceId = commandTraceId(prompt) || prompt?.traceId || null; const ids = traceIdsFromSamples(segment); if (promptTraceId && (ids.length === 0 || ids.includes(promptTraceId))) return promptTraceId; return ids.slice(-1)[0] || promptTraceId || null; }", - "function traceIdForPromptUserMessage(items, prompt) {", - " const hash = prompt?.textHash || commandTextHash(prompt);", - " if (!hash) return null;", - " for (const sample of items) {", - " for (const message of arr(sample.messages)) {", - " const role = String(message?.role || message?.dataRole || message?.messageRole || '').toLowerCase();", - " if (role && !/user/u.test(role)) continue;", - " if (message?.textHash === hash) {", - " const id = itemTraceId(message);", - " if (id) return id;", - " }", - " }", - " }", - " return null;", - "}", - "function traceEntries(items, traceId) { const entries = []; for (const sample of items) for (const entry of entryGroups(sample)) { const text = textOf(entry.item); const id = itemTraceId(entry.item); if (!traceId || id === traceId || text.includes(traceId)) entries.push({ ...entry, sample, text }); } return entries; }", - "function normalizeLifecycleStatus(value) { const raw = String(value || '').trim().toLowerCase(); if (/^(canceled|cancelled)$/u.test(raw)) return 'canceled'; if (/^(failed|failure|error)$/u.test(raw)) return 'failed'; if (/^(completed|complete|succeeded|success|terminal)$/u.test(raw)) return 'completed'; if (/^(running|admitting|queued|pending|in_progress|in-progress)$/u.test(raw)) return 'running'; return null; }", - "function statusFor(items, traceId) { const entries = traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) }))); const lastTurn = entries.filter((entry) => entry.group === 'turn').slice(-1)[0]?.item || null; const turnStatus = normalizeLifecycleStatus(lastTurn?.status); if (turnStatus) return turnStatus; const lastMessage = entries.filter((entry) => entry.group === 'message').slice(-1)[0]?.item || null; return normalizeLifecycleStatus(lastMessage?.status) || 'unknown'; }", - "function cleanFinalResponseText(value) { const raw = String(value || '').trim(); if (!raw) return ''; if (/^(completed|failed|canceled|cancelled|轮次完成|轮次失败|轮次取消|已记录)$/iu.test(raw.replace(/\\s+/gu, ' '))) return ''; if (/^(admitted|run|ok|error)\\s+/iu.test(raw)) return ''; return raw; }", - "function finalResponseTextFromEntry(entry) {", - " const explicit = cleanFinalResponseText(entry.item?.finalResponse?.text || entry.item?.finalResponse?.preview || '');", - " if (explicit && !/^Code Agent\\s*耗时/iu.test(explicit)) return explicit;", - " if (entry.group !== 'message') return '';", - " const role = String(entry.item?.role || entry.item?.dataRole || entry.item?.messageRole || '').toLowerCase();", - " if (role && !/assistant|agent|system/u.test(role)) return '';", - " const text = cleanFinalResponseText(entry.text);", - " return text && !/^Code Agent\\s*耗时/iu.test(text) ? text : '';", - "}", - "function finalResponseEmpty(items, traceId) { if (!/^(completed|failed|canceled)$/u.test(statusFor(items, traceId))) return true; const entries = (traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) })))).slice().reverse(); for (const entry of entries) { if (finalResponseTextFromEntry(entry)) return false; } return true; }", - "function rowFor() {", - " const control = readJsonl('control.jsonl');", - " const samples = readJsonlTail('samples.jsonl');", - " const prompts = promptCommands(control);", - " const prompt = prompts[promptIndex - 1] || null;", - " if (!prompt) return { ok: true, round: promptIndex, status: 'command-pending', traceId: null, finalResponseEmpty: true, lastSeq: null, lastTs: null, promptMissing: true, valuesRedacted: true };", - " const done = readDone(prompt.commandId);", - " const failed = readFailed(prompt.commandId);", - " if (failed) return { ok: false, failure: 'observe-command-sendPrompt-failed', round: promptIndex, status: 'command-failed', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandFailure: short(failed.error?.message || failed.failure || failed.status || 'command failed'), valuesRedacted: true };", - " const promptTraceId = commandTraceId(prompt);", - " if (!done) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: promptTraceId || null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: !promptTraceId, valuesRedacted: true };", - " const segment = segmentFor(samples, prompts, promptIndex - 1);", - " const controlSegment = segment.filter((sample) => sample.pageRole === 'control');", - " const traceId = promptTraceId || traceIdForPromptUserMessage(segment, prompt) || chooseTraceId(controlSegment, prompt) || chooseTraceId(segment, prompt);", - " if (!traceId) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: true, segmentSampleCount: segment.length, valuesRedacted: true };", - " const controlTraceSegment = traceId ? controlSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)) : [];", - " const statusSegment = controlTraceSegment.length > 0 ? controlSegment : segment;", - " const status = statusFor(statusSegment, traceId);", - " const sampleForTrace = traceId ? statusSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;", - " const lastSample = sampleForTrace || segment.slice(-1)[0] || null;", - " const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;", - " const composer = composerSample?.composer || {};", - " const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';", - " return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(statusSegment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, sampleScope: controlSegment.length > 0 ? 'control' : 'all', segmentSampleCount: segment.length, statusSampleCount: statusSegment.length, source: 'observe-artifact-wait-script', valuesRedacted: true };", - "}", - "function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }", - "function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }", - "function terminal(value) { return ['completed', 'succeeded', 'success', 'failed', 'error', 'blocked', 'timeout', 'canceled', 'cancelled', 'terminal'].includes(norm(value)); }", - "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", - "(async () => {", - " const observations = [];", - " while (Date.now() - startedAt <= timeoutMs) {", - " const row = rowFor();", - " observations.push(row);", - " if (row.ok === false) { console.log(JSON.stringify({ ...row, ok: false, failure: row.failure || 'quick-verify-artifact-row-failed', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", - " if (successful(row.status) && row.finalResponseEmpty !== true) { console.log(JSON.stringify({ ...row, ok: true, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", - " if (!successful(row.status) && terminal(row.status)) { console.log(JSON.stringify({ ...row, ok: false, failure: 'observe-turn-terminal-non-success', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", - " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", - " }", - " const row = rowFor();", - " observations.push(row);", - " console.log(JSON.stringify({ ...row, ok: false, failure: 'quick-verify-wait-chunk-timeout', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", - "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-artifact-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", - "NODE", - ].join("\n"); -} - -function isQuickVerifyTurnSuccessful(value: string | null): boolean { - const status = normalizeQuickVerifyStatus(value); - return status === "completed" || status === "succeeded" || status === "success"; -} - -function isQuickVerifyTurnTerminal(value: string | null): boolean { - const status = normalizeQuickVerifyStatus(value); - return status === "completed" - || status === "succeeded" - || status === "success" - || status === "failed" - || status === "error" - || status === "blocked" - || status === "timeout" - || status === "canceled" - || status === "cancelled" - || status === "terminal"; -} - -function normalizeQuickVerifyStatus(value: string | null): string { - return String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); -} - -function cliDataPayload(parsed: Record | null): Record { - const root = record(parsed); - const payload = isRecord(root.data) ? root.data : root; - return cliDumpPayload(payload) ?? payload; -} - -function cliDumpPayload(payload: Record): Record | null { - if (payload.outputTruncated !== true) return null; - const dumpPath = stringAtNullable(record(payload.dump), "path"); - if (dumpPath === null || !existsSync(dumpPath)) return null; - const dumped = parseJsonObject(readFileSync(dumpPath, "utf8")); - if (dumped === null) return null; - const dumpedRoot = record(dumped); - return isRecord(dumpedRoot.data) ? dumpedRoot.data : dumpedRoot; -} - -function findScenario(state: SentinelCicdState, scenarioId: string): Record | null { - const scenarios = readConfigRefTarget(state.configRefs.scenarios); - const items = Array.isArray(scenarios) ? scenarios : isRecord(scenarios) ? [scenarios] : []; - return items.map(record).find((item) => item.id === scenarioId) ?? null; -} - -function readPromptSetForScenario(scenario: Record): { ok: true; prompts: string[]; summary: Record } | { ok: false; error: string; summary: Record } { - const promptSet = recordTarget(readConfigRefTarget(stringAt(scenario, "promptSetRef")), stringAt(scenario, "promptSetRef")); - const sourceRef = stringAt(promptSet, "promptSourceRef"); - const key = stringAt(promptSet, "promptSourceKey"); - const paths = secretSourcePaths(sourceRef); - const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); - const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true }; - const runtimeRaw = process.env[key]; - const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; - const raw = values[key] ?? runtimeRaw; - const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null; - if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary }; - if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary }; - const parsed = parsePromptJson(raw); - if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary }; - return { - ok: true, - prompts: parsed, - summary: { - ...summary, - sourceMode, - promptCount: parsed.length, - promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))), - promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`), - promptTextBytes: parsed.map((item) => Buffer.byteLength(item)), - valuesRedacted: true, - }, - }; -} - -function parsePromptJson(raw: string): string[] { - try { - const parsed = JSON.parse(raw) as unknown; - if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string" && item.length > 0); - const recordValue = record(parsed); - if (Array.isArray(recordValue.prompts)) return recordValue.prompts.filter((item): item is string => typeof item === "string" && item.length > 0); - if (typeof recordValue.prompt === "string" && recordValue.prompt.length > 0) return [recordValue.prompt]; - } catch { - if (raw.trim().length > 0) return [raw]; - } - return []; -} - -function readLocalObserveIndex(observerId: string): { stateDir: string } | null { - const path = rootPath(".state/web-observe/index.json"); - if (!existsSync(path)) return null; - const parsed = parseJsonObject(readFileSync(path, "utf8")); - const entry = record(parsed?.[observerId]); - const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : null; - return stateDir === null ? null : { stateDir }; -} - -function secretSourcePaths(sourceRef: string): string[] { +export function secretSourcePaths(sourceRef: string): string[] { + if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); @@ -4689,7 +2307,15 @@ function secretSourcePaths(sourceRef: string): string[] { return [...new Set(paths)]; } -function parseEnvFile(textValue: string): Record { +function ownerFileSourcePaths(sourceRef: string): string[] { + if (sourceRef.includes("..") || sourceRef.includes("\0")) return []; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot]; + return [...new Set(roots.map((root) => join(root, sourceRef)))]; +} + +export function parseEnvFile(textValue: string): Record { const values: Record = {}; for (const rawLine of textValue.split(/\r?\n/u)) { const line = rawLine.trim(); @@ -4704,68 +2330,17 @@ function parseEnvFile(textValue: string): Record { return values; } -function observerIdFromText(textValue: string): string | null { - return /\bwebobs-[a-z0-9-]+\b/iu.exec(textValue)?.[0] ?? null; -} - -function remainingSeconds(deadline: number, cap: number): number { - return Math.max(5, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); -} - -function metricNames(textValue: unknown): string[] { - if (typeof textValue !== "string") return []; - return textValue.split(/\r?\n/u).map((line) => /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line)?.[1]).filter((item): item is string => typeof item === "string"); -} - -function validationBlocker(health: Record, metrics: Record, report: Record, publicExposure: Record, publicDashboard: Record, quickVerify: Record | null): Record { - const blockers = []; - if (!health.ok || record(health.bodyJson).ok !== true) blockers.push("health"); - if (!metrics.ok || !metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health")) blockers.push("metrics"); - if (!report.ok) blockers.push("recent-report"); - if (publicExposure.ok !== true) blockers.push("public-exposure"); - if (publicDashboard.ok !== true) blockers.push("public-dashboard"); - if (quickVerify !== null && quickVerify.ok !== true) blockers.push("quick-verify"); - return { code: "sentinel-validation-failed", blockers, valuesRedacted: true }; -} - -function serviceUnavailableBlocker(state: SentinelCicdState): Record { - return { - code: "sentinel-service-unavailable", - policy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"), - reason: "sentinel service must be reachable through k3s internal Service DNS before quick verify can run; no public/fallback path is used.", - retry: `bun scripts/cli.ts web-probe sentinel validate --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, - valuesRedacted: true, - }; -} - -function sentinelP5Next(state: SentinelCicdState): Record { - const node = state.spec.nodeId; - const lane = state.spec.lane; - const suffix = sentinelCliSuffix(state); - return { - validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`, - quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`, - maintenanceStart: `bun scripts/cli.ts web-probe sentinel maintenance start --node ${node} --lane ${lane}${suffix} --confirm --wait`, - maintenanceStop: `bun scripts/cli.ts web-probe sentinel maintenance stop --node ${node} --lane ${lane}${suffix} --confirm --wait`, - report: `bun scripts/cli.ts web-probe sentinel report --node ${node} --lane ${lane}${suffix} --view summary`, - }; -} - -function isSafeRelativeStateDir(value: string): boolean { - return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes(".."); -} - -function stringAtNullable(value: unknown, path: string): string | null { +export function stringAtNullable(value: unknown, path: string): string | null { const found = valueAtPath(value, path); return typeof found === "string" && found.length > 0 ? found : null; } -function numberAtNullable(value: unknown, path: string): number | null { +export function numberAtNullable(value: unknown, path: string): number | null { const found = valueAtPath(value, path); return typeof found === "number" && Number.isFinite(found) ? found : null; } -function displayPath(pathValue: string): string { +export function displayPath(pathValue: string): string { if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1); const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); @@ -4776,479 +2351,16 @@ function displayPath(pathValue: string): string { return pathValue; } -function mergeFindingRecords(primary: readonly Record[], extra: readonly Record[]): Record[] { - const merged: Record[] = []; - const seen = new Set(); - for (const item of [...primary, ...extra]) { - const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? stringAtNullable(item, "finding_id") ?? "finding"; - const severity = stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "unknown"; - const key = `${id}\0${severity}`; - if (seen.has(key)) continue; - seen.add(key); - merged.push(item); - } - return merged; -} - -function isQuickVerifyBlockingFinding(item: Record): boolean { - const severity = (stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "").toLowerCase(); - if (!["critical", "red", "fatal", "error", "failed", "blocked"].includes(severity)) return false; - const id = (stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? "").toLowerCase(); - if (id === "observer-command-failed") return observerCommandFailureBlocks(item); - return [ - "quick-verify-no-business-turn", - "quick-verify-command-sequence-failed", - "quick-verify-observer-start-failed", - "quick-verify-account-secret-missing", - "prompt-chat-submit-failed", - "route-active-session-mismatch", - "final-response-flicker", - "round-completion-final-response-missing", - "turn-trace-id-missing", - "no-samples", - "jsonl-read-issues", - ].includes(id); -} - -function observerCommandFailureBlocks(item: Record): boolean { - const commands = Array.isArray(item.commands) ? item.commands.map(record) : []; - if (commands.length === 0) return true; - return commands.some((command) => { - const type = (stringAtNullable(command, "type") ?? "").toLowerCase(); - if (["stop", "cancel", "mark", "screenshot"].includes(type)) return false; - return true; - }); -} - -function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): Record[] { - if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return []; - const rendered = [ - typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "", - typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "", - ].join("\n"); - const noPromptScenario = promptIndex <= 0; - if (noPromptScenario && failure === null) return []; - if (noPromptScenario && failure !== null) { - const observerStartFailure = failure === "observe-start-failed"; - return [{ - id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed", - severity: "red", - count: 1, - summary: observerStartFailure - ? "quick verify observer failed to start before the no-prompt scenario could run." - : "quick verify no-prompt command sequence failed before the account/session workflow completed.", - failure, - promptIndex, - valuesRedacted: true, - }]; - } - const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered); - const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered); - if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return []; - return [{ - id: "quick-verify-no-business-turn", - severity: "red", - count: 1, - summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.", - failure: failure ?? null, - promptIndex, - valuesRedacted: true, - }]; -} - -function quickVerifyCompletedTurnSummaryRow(promptIndex: number, turnSummary: Record | null): Record | null { - const rows = Array.isArray(record(turnSummary?.collect).rows) ? record(turnSummary?.collect).rows.map(record) : []; - const scopedRows = promptIndex > 0 ? rows.filter((row) => numberAtNullable(row, "round") === promptIndex) : rows; - return scopedRows.find((row) => { - const finalResponse = record(row.finalResponse); - return isQuickVerifyTurnSuccessful(stringAtNullable(row, "status")) - && stringAtNullable(row, "traceId") !== null - && finalResponse.empty !== true; - }) ?? null; -} - -function quickVerifyTurnSummaryFallback(state: SentinelCicdState, observerId: string, promptIndex: number): Record { - const turnSummary = collectObserveView(state, observerId, "turn-summary", null, 25); - const row = quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary); - const rows = Array.isArray(record(turnSummary.collect).rows) ? record(turnSummary.collect).rows.map(record) : []; - if (row === null) { - return { - ok: false, - source: "turn-summary-fallback", - collectOk: turnSummary.ok === true, - rowCount: rows.length, - promptIndex, - result: turnSummary.result ?? null, - valuesRedacted: true, - }; - } - const finalResponse = record(row.finalResponse); - return { - ok: true, - source: "turn-summary-fallback", - collectOk: turnSummary.ok === true, - rowCount: rows.length, - promptIndex, - status: stringAtNullable(row, "status"), - traceId: stringAtNullable(row, "traceId"), - finalResponseEmpty: finalResponse.empty === true, - finalResponseBytes: numberAtNullable(finalResponse, "textBytes"), - result: turnSummary.result ?? null, - valuesRedacted: true, - }; -} - -function quickVerifyHasDurableBusinessTurn(promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): boolean { - if (quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary) !== null) return true; - const renderedTrace = typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : ""; - if (!renderedTrace) return false; - if (/Final Response\s*\n\s*\(空内容\)/iu.test(renderedTrace)) return false; - return /Code Agent[^\n]*completed|轮次完成（总耗时/iu.test(renderedTrace) - && /Final Response\s*\n\s*\S/iu.test(renderedTrace) - && !/无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(renderedTrace); -} - -function quickVerifyBusinessStatus( - failure: string | null, - promptIndex: number, - turnSummary: Record | null, - traceFrame: Record | null, - elapsedMs: unknown, - budgetSeconds: number, -): Record { - const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame); - const elapsed = typeof elapsedMs === "number" && Number.isFinite(elapsedMs) ? elapsedMs : null; - const budgetMs = Math.max(0, budgetSeconds) * 1000; - const budgetExceeded = elapsed !== null && budgetMs > 0 && elapsed > budgetMs; - const observerTimeout = budgetExceeded || (failure !== null && isRecoverableQuickVerifyWaitFailure(failure)); - const status = durableBusinessTurn - ? "business-turn-completed" - : observerTimeout - ? "observer-timeout" - : "scenario-incomplete"; - return { - status, - durableBusinessTurn, - observerTimeout, - scenarioComplete: durableBusinessTurn, - failure: failure ?? null, - promptIndex, - elapsedMs: elapsed, - budgetSeconds, - sourceOfTruth: durableBusinessTurn ? "turn-summary-or-trace-frame" : observerTimeout ? "runner-wait-budget" : "control-findings", - valuesRedacted: true, - }; -} - -function isRecoverableQuickVerifyWaitFailure(failure: string): boolean { - return failure === "quick-verify-wait-chunk-timeout" - || failure === "quick-verify-timeout-over-budget" - || failure === "observe-turn-terminal-wait-failed"; -} - -function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } { - return { - ...compactCommand(result), - stdoutPreview: result.stdout.trim().slice(0, 1200), - stderrPreview: result.stderr.trim().slice(0, 1200), - stdoutTail: result.stdout.trim().slice(-4000), - stderrTail: result.stderr.trim().slice(-4000), - }; -} - -function renderQuickVerifySummary(input: Record): string { - const artifact = record(input.artifactSummary); - const findings = Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; - return [ - "Web Probe Sentinel Quick Verify", - "=======================================================", - `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, - `report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${artifact.findingCount ?? findings.length}`, - `publicOrigin=${input.publicOrigin ?? "-"}`, - "", - "Findings", - findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"), - ].join("\n"); -} - -function renderAuthSessionSwitchQuickVerifySummary(input: Record): string { - const artifact = record(input.artifactSummary); - const accountEnv = record(input.accountEnv); - const findingRows = Array.isArray(input.findings) - ? input.findings.map(record).slice(0, 8) - : Array.isArray(artifact.findings) - ? artifact.findings.map(record).slice(0, 8) - : []; - const steps = Array.isArray(input.steps) ? input.steps.map(record) : []; - const commandSteps = steps - .filter((step) => { - const phase = stringAtNullable(step, "phase") ?? ""; - return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env"; - }) - .map((step) => { - const phase = stringAtNullable(step, "phase") ?? "-"; - const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-"; - const result = record(step.result); - const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-"; - return `${phase} ${ok} ${status}`; - }); - return [ - "Auth Session Switch Quick Verify", - "=======================================================", - `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, - `status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`, - `accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`, - "", - "Command Sequence", - commandSteps.length === 0 ? "-" : commandSteps.join("\n"), - "", - "Findings", - findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"), - ].join("\n"); -} - -function formatQuickVerifyTimingSuffix(item: Record): string { - const status = stringAtNullable(item, "timingStatus"); - const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind"); - if (status === null && source === null) return ""; - return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`; -} - -function renderMaintenanceResult(result: Record): string { - const serviceHealth = record(result.serviceHealth); - const maintenance = record(result.maintenance); - const quickVerify = record(result.quickVerify); - const quickVerifyBusiness = record(quickVerify.businessStatus); - const planned = record(result.planned); - const blocker = record(result.blocker); - const next = record(result.next); - const maintenanceBody = record(maintenance.bodyJson); - const state = record(maintenanceBody.maintenance); - return [ - String(result.command), - "", - table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]), - "", - table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]), - "", - Object.keys(state).length > 0 - ? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]]) - : Object.keys(planned).length > 0 - ? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]]) - : "MAINTENANCE\n-", - "", - Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), - "", - Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), - "", - "NEXT", - ` validate: ${next.validate ?? "-"}`, - ` report: ${next.report ?? "-"}`, - ` maintenance-stop: ${next.maintenanceStop ?? "-"}`, - "", - "DISCLOSURE", - " maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.", - ].join("\n"); -} - -function renderValidateResult(result: Record): string { - const health = record(result.serviceHealth); - const metrics = record(result.metrics); - const report = record(result.report); - const publicExposure = record(result.publicExposure); - const publicDashboard = record(result.publicDashboard); - const quickVerify = record(result.quickVerify); - const blocker = record(result.blocker); - const next = record(result.next); - const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []); - return [ - String(result.command), - "", - table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]), - "", - table(["CHECK", "OK", "DETAIL"], [ - ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`], - ["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`], - ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], - ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], - ["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`], - ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`], - ]), - "", - warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), - "", - Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]), - "", - "NEXT", - ` quick-verify: ${next.quickVerify ?? "-"}`, - ` report: ${next.report ?? "-"}`, - ` maintenance-start: ${next.maintenanceStart ?? "-"}`, - "", - "DISCLOSURE", - " validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.", - ].join("\n"); -} - -function renderDashboardResult(result: Record): string { - const page = record(result.page); - const dom = record(page.dom); - const dataset = record(dom.dataset); - const layout = record(dom.layout); - const chartCounts = record(dom.chartCounts); - const latestRunCounts = record(dom.latestRunCounts); - const latestDetailSummary = record(dom.latestDetailSummary); - const checkScope = record(dom.checkScope); - const selectedRunTags = record(dom.selectedRunTags); - const trendPanelCompact = record(dom.trendPanelCompact); - const checkDialog = record(dom.checkDialog); - const runFilterProbe = record(dom.runFilterProbe); - const runFilterObserved = record(runFilterProbe.observed); - const runFilterExpected = record(runFilterProbe.expected); - const overviewSamples = record(dom.overviewSamples); - const panelHeights = record(dom.panelHeights); - const workspaceHeight = record(panelHeights.workspace); - const checksHeight = record(panelHeights.checks); - const screenshot = record(result.screenshot); - const remote = record(result.remote); - const transport = record(result.transport); - const degradedReason = result.degradedReason ?? null; - return [ - String(result.command), - "", - table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]), - "", - table(["HTTP", "SHELL", "RUN_ROWS", "CHECK_ROWS", "TABS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[ - page.httpStatus ?? "-", - dom.shell, - dom.runRows, - dom.checkRows, - dom.detailTabs, - page.pageErrorCount, - page.consoleErrorCount, - page.requestFailureCount, - ]]), - "", - table(["TITLE", "STATUS_TEXT", "CONTRACT", "BASE_PATH"], [[dom.title, dom.statusText, dataset.contractVersion, dataset.basePath ?? "-"]]), - "", - table(["TREND_ERR_TYPES", "TREND_ALERT_TYPES", "TREND_TOTAL_TYPES", "TREND_EXACT", "MATCH_LATEST", "BAD_TITLE", "BAD_BODY"], [[ - chartCounts.error ?? "-", - chartCounts.warning ?? "-", - chartCounts.total ?? "-", - chartCounts.ok ?? "-", - chartCounts.matchesLatestRun ?? "-", - dom.badCardTitleCount ?? "-", - dom.badCardBodyCount ?? "-", - ]]), - "", - table(["LATEST_RUN", "TYPE_COUNT", "ERR_TYPES", "ALERT_TYPES", "TOTAL_TYPES", "SAMPLE_TOTAL", "HIST_ERR", "HIST_ALERT"], [[ - latestRunCounts.runId ?? "-", - latestRunCounts.typeCount ?? "-", - latestRunCounts.error ?? "-", - latestRunCounts.warning ?? "-", - latestRunCounts.total ?? "-", - latestRunCounts.alertSamples ?? "-", - overviewSamples.error ?? "-", - overviewSamples.warning ?? "-", - ]]), - "", - table(["CHECK_SCOPE", "CHECK_RUN", "CHECK_TYPES", "CHECK_ERR_TYPES", "CHECK_ALERT_TYPES", "SAMPLE_ERR", "SAMPLE_ALERT", "CHECK_MATCH_LATEST", "CHECK_MATCH_DETAIL"], [[ - checkScope.scope ?? "-", - checkScope.runId ?? "-", - `${checkScope.typeCount ?? "-"}/${latestDetailSummary.typeCount ?? "-"}`, - `${checkScope.errorTypeCount ?? "-"}/${latestDetailSummary.errorTypeCount ?? "-"}`, - `${checkScope.alertTypeCount ?? "-"}/${latestDetailSummary.alertTypeCount ?? "-"}`, - checkScope.errorSamples ?? "-", - checkScope.alertSamples ?? "-", - checkScope.matchesLatestRun ?? "-", - checkScope.matchesRunDetail ?? "-", - ]]), - "", - table(["CHECK_VISIBLE_ROWS", "CHECK_VISIBLE_ALERT", "RUN_TAG_ERR", "RUN_TAG_ALERT", "RUN_TAG_MATCH", "BELOW_WORKSPACE", "FULL_WIDTH"], [[ - checkScope.visibleRowCount ?? "-", - checkScope.visibleAlertSamples ?? "-", - selectedRunTags.error ?? "-", - selectedRunTags.warning ?? "-", - selectedRunTags.matchesRunDetail ?? "-", - checkScope.belowWorkspace ?? "-", - checkScope.fullWidth ?? "-", - ]]), - "", - table(["TREND_PANEL_SLACK", "TREND_PANEL_COMPACT", "DETAIL_DIALOG", "DIALOG_LARGE"], [[ - trendPanelCompact.bottomSlackPx ?? "-", - trendPanelCompact.ok ?? "-", - checkDialog.opened ?? "-", - checkDialog.large ?? "-", - ]]), - "", - table(["WORKSPACE_H", "WORKSPACE_RATIO", "WORKSPACE_80", "CHECKS_H", "CHECKS_RATIO", "CHECKS_80", "PANES_80"], [[ - `${workspaceHeight.heightPx ?? "-"}/${workspaceHeight.targetPx ?? "-"}`, - workspaceHeight.ratio ?? "-", - panelHeights.workspaceOk ?? "-", - `${checksHeight.heightPx ?? "-"}/${checksHeight.targetPx ?? "-"}`, - checksHeight.ratio ?? "-", - panelHeights.checksOk ?? "-", - panelHeights.workspacePaneBounded ?? "-", - ]]), - "", - table(["FILTER_RUN", "FILTER_OPTION", "FILTER_TYPES", "FILTER_ERR_TYPES", "FILTER_ALERT_TYPES", "FILTER_SAMPLE_ERR", "FILTER_SAMPLE_ALERT", "FILTER_MATCH_DETAIL"], [[ - runFilterProbe.targetRunId ?? "-", - runFilterProbe.requestedOptionPresent ?? "-", - `${runFilterObserved.typeCount ?? "-"}/${runFilterExpected.typeCount ?? "-"}`, - `${runFilterObserved.errorTypeCount ?? "-"}/${runFilterExpected.errorTypeCount ?? "-"}`, - `${runFilterObserved.alertTypeCount ?? "-"}/${runFilterExpected.alertTypeCount ?? "-"}`, - runFilterObserved.errorSamples ?? "-", - runFilterObserved.alertSamples ?? "-", - runFilterProbe.matchesRunDetail ?? "-", - ]]), - "", - table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[ - result.viewport, - `${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`, - layout.horizontalOverflow, - layout.overflowCount, - ]]), - "", - Object.keys(screenshot).length === 0 - ? "SCREENSHOT\n-" - : table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]), - "", - degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT"], [[degradedReason, remote.exitCode, transport.ok]]), - "", - "NEXT", - ` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, - ` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, - "", - "DISCLOSURE", - " dashboard verify uses the YAML publicExposure URL and remote browser execution; it does not start a sentinel run or inspect provider payloads.", - ].join("\n"); -} - -function renderReportResult(result: Record): string { - const report = record(result.report); - const body = record(report.bodyJson); - const run = record(body.run); - return [ - String(result.command), - "", - table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]), - "", - table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]), - "", - "DISCLOSURE", - " report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.", - ].join("\n"); -} - function sentinelPipelineRunName(state: SentinelCicdState): string { const commit = state.sourceHead.commit ?? "source"; return `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`; } -function sentinelCliSuffix(state: SentinelCicdState): string { +export function sentinelCliSuffix(state: SentinelCicdState): string { return ` --sentinel ${state.sentinelId}`; } -function safeJobSegment(value: string): string { +export function safeJobSegment(value: string): string { return value.replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "").slice(0, 48) || "sentinel"; } @@ -5419,7 +2531,7 @@ function observedDetail(name: string, item: Record): string { return "-"; } -function renderAsyncJobResult(result: Record): string { +export function renderAsyncJobResult(result: Record): string { const job = record(result.job); const next = record(result.next); return [ @@ -5438,16 +2550,10 @@ function renderAsyncJobResult(result: Record): string { ].join("\n"); } -function rendered(ok: boolean, command: string, text: string): RenderedCliResult { +export function rendered(ok: boolean, command: string, text: string): RenderedCliResult { return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" }; } -function readConfigRefTarget(ref: string): unknown { - const file = configRefFile(ref); - const path = configRefPath(ref); - return valueAtPath(readConfigFile(file), path); -} - function readConfigFile(file: string): unknown { if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`); const abs = rootPath(file); @@ -5461,12 +2567,6 @@ function configRefFile(ref: string): string { return file; } -function configRefPath(ref: string): string { - const [, path] = ref.split("#"); - if (path === undefined || path.length === 0) throw new Error(`invalid configRef: ${ref}`); - return path; -} - function valueAtPath(value: unknown, path: string): unknown { let current: unknown = value; for (const segment of path.split(".")) { @@ -5484,7 +2584,7 @@ function valueAtPath(value: unknown, path: string): unknown { return current; } -function stringAt(value: unknown, path: string): string { +export function stringAt(value: unknown, path: string): string { const found = valueAtPath(value, path); if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`); return found; @@ -5503,28 +2603,28 @@ function stringTarget(value: unknown, label: string): string { return value; } -function numberAt(value: unknown, path: string): number { +export function numberAt(value: unknown, path: string): number { const found = valueAtPath(value, path); if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`); return found; } -function arrayAt(value: unknown, path: string): unknown[] { +export function arrayAt(value: unknown, path: string): unknown[] { const found = valueAtPath(value, path); if (!Array.isArray(found)) throw new Error(`${path} must be an array`); return found; } -function recordTarget(value: unknown, label: string): Record { +export function recordTarget(value: unknown, label: string): Record { if (!isRecord(value)) throw new Error(`${label} must resolve to an object`); return value; } -function record(value: unknown): Record { +export function record(value: unknown): Record { return isRecord(value) ? value : {}; } -function isRecord(value: unknown): value is Record { +export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -5536,7 +2636,7 @@ function manifestObjectSummary(items: readonly Record[]): reado })); } -function compactCommand(result: CommandResult): CompactCommandResult { +export function compactCommand(result: CommandResult): CompactCommandResult { return { exitCode: result.exitCode, timedOut: result.timedOut, @@ -5547,7 +2647,7 @@ function compactCommand(result: CommandResult): CompactCommandResult { }; } -function parseJsonObject(text: string): Record | null { +export function parseJsonObject(text: string): Record | null { const trimmed = text.trim(); if (trimmed.length === 0) return null; try { @@ -5558,19 +2658,19 @@ function parseJsonObject(text: string): Record | null { } } -function table(headers: string[], rows: unknown[][]): string { +export function table(headers: string[], rows: unknown[][]): string { const normalized = [headers, ...rows.map((row) => row.map(text))]; const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length))); return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n"); } -function text(value: unknown): string { +export function text(value: unknown): string { if (value === undefined || value === null || value === "") return "-"; if (typeof value === "boolean") return value ? "true" : "false"; return String(value).replace(/\s+/gu, " ").trim(); } -function short(value: unknown): string { +export function short(value: unknown): string { const raw = text(value); if (raw === "-") return raw; if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`; @@ -5582,7 +2682,7 @@ function sha256(textValue: string): string { return `sha256:${createHash("sha256").update(textValue).digest("hex")}`; } -function shellQuote(value: string): string { +export function shellQuote(value: string): string { return `'${value.replace(/'/gu, "'\\''")}'`; } diff --git a/scripts/src/hwlab-node-web-sentinel-config-ref.ts b/scripts/src/hwlab-node-web-sentinel-config-ref.ts new file mode 100644 index 00000000..4820a227 --- /dev/null +++ b/scripts/src/hwlab-node-web-sentinel-config-ref.ts @@ -0,0 +1,209 @@ +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-29-p14-yaml-template-refs. +// Responsibility: Render YAML-first web-probe sentinel configRefs with node/lane variables and selector paths. +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { rootPath } from "./config"; + +export interface WebProbeSentinelTemplateContext { + readonly nodeId: string; + readonly lane: string; + readonly runtimeNamespace?: string; +} + +type TemplateVarValue = string | number | boolean; + +export interface WebProbeSentinelRenderedConfigRef { + readonly originalRef: string; + readonly ref: string; + readonly file: string; + readonly path: string; + readonly target: unknown; + readonly sha256: string; + readonly byteCount: number; + readonly variableNames: readonly string[]; +} + +export function readWebProbeSentinelConfigRefTarget(context: WebProbeSentinelTemplateContext, ref: string): unknown { + return readWebProbeSentinelConfigRef(context, ref).target; +} + +export function readWebProbeSentinelConfigRef(context: WebProbeSentinelTemplateContext, ref: string): WebProbeSentinelRenderedConfigRef { + const builtinVars = templateContextVars(context); + const renderedRef = renderTemplateString(ref, builtinVars, "configRef"); + const parsed = parseRenderedConfigRef(renderedRef); + const absPath = rootPath(parsed.file); + if (!existsSync(absPath)) throw new Error(`${parsed.file} does not exist`); + const text = readFileSync(absPath, "utf8"); + const doc = Bun.YAML.parse(text) as unknown; + const vars = { ...builtinVars, ...documentVars(doc, builtinVars) }; + const path = renderTemplateString(parsed.path, vars, `${parsed.file}#path`); + const target = valueAtConfigPath(doc, path); + if (target === undefined) throw new Error(`${parsed.file}#${path} is missing`); + return { + originalRef: ref, + ref: `${parsed.file}#${path}`, + file: parsed.file, + path, + target: renderTemplateValue(target, vars), + sha256: `sha256:${createHash("sha256").update(text).digest("hex")}`, + byteCount: Buffer.byteLength(text), + variableNames: Object.keys(vars).sort(), + }; +} + +export function renderWebProbeSentinelConfigRefString(context: WebProbeSentinelTemplateContext, ref: string): string { + return renderTemplateString(ref, templateContextVars(context), "configRef"); +} + +function templateContextVars(context: WebProbeSentinelTemplateContext): Record { + const node = context.nodeId; + const nodeLower = node.toLowerCase(); + const lane = context.lane; + const laneMinor = lane.replace(/^v/u, ""); + return { + NODE: node, + node, + nodeLower, + NODE_LOWER: nodeLower, + LANE: lane, + lane, + laneMinor, + LANE_MINOR: laneMinor, + nodeLane: `${nodeLower}-${lane}`, + NODE_LANE: `${node}-${lane}`, + runtimeNamespace: context.runtimeNamespace ?? "", + ...contextTemplateVars(context), + }; +} + +function contextTemplateVars(context: WebProbeSentinelTemplateContext): Record { + const observability = isRecord((context as { readonly observability?: unknown }).observability) + ? (context as { readonly observability: Record }).observability + : {}; + const webProbe = isRecord(observability.webProbe) ? observability.webProbe : {}; + const templateVars = isRecord(webProbe.templateVars) ? webProbe.templateVars : {}; + const result: Record = {}; + for (const [key, value] of Object.entries(templateVars)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`observability.webProbe.templateVars.${key} is not a supported variable name`); + if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") { + throw new Error(`observability.webProbe.templateVars.${key} must be a scalar`); + } + result[key] = value; + } + return result; +} + +function documentVars(doc: unknown, vars: Record): Record { + if (!isRecord(doc) || !isRecord(doc.vars)) return {}; + const result: Record = {}; + for (const [key, value] of Object.entries(doc.vars)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`vars.${key} is not a supported variable name`); + if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") { + throw new Error(`vars.${key} must be a scalar`); + } + result[key] = typeof value === "string" + ? renderTemplateString(value, { ...vars, ...result }, `vars.${key}`) + : value; + } + return result; +} + +function renderTemplateValue(value: unknown, vars: Record): unknown { + if (typeof value === "string") return renderTemplateScalar(value, vars, "YAML string"); + if (Array.isArray(value)) return value.map((item) => renderTemplateValue(item, vars)); + if (!isRecord(value)) return value; + const localVars = localTemplateVars(value, vars); + const rendered: Record = {}; + for (const [key, item] of Object.entries(value)) { + if (key === "vars") continue; + rendered[key] = renderTemplateValue(item, localVars); + } + return rendered; +} + +function localTemplateVars(value: Record, parentVars: Record): Record { + if (!isRecord(value.vars)) return parentVars; + const local: Record = {}; + for (const [key, item] of Object.entries(value.vars)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`vars.${key} is not a supported variable name`); + if (typeof item !== "string" && typeof item !== "number" && typeof item !== "boolean") throw new Error(`vars.${key} must be a scalar`); + local[key] = typeof item === "string" + ? renderTemplateString(item, { ...parentVars, ...local }, `vars.${key}`) + : item; + } + return { ...parentVars, ...local }; +} + +function renderTemplateScalar(value: string, vars: Record, label: string): TemplateVarValue { + const whole = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/u.exec(value); + if (whole !== null) { + const rendered = vars[whole[1]]; + if (rendered === undefined) throw new Error(`${label} references undefined variable ${whole[1]}`); + return rendered; + } + return renderTemplateString(value, vars, label); +} + +function renderTemplateString(value: string, vars: Record, label: string): string { + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/gu, (_match, key: string) => { + const rendered = vars[key]; + if (rendered === undefined) throw new Error(`${label} references undefined variable ${key}`); + return String(rendered); + }); +} + +function parseRenderedConfigRef(ref: string): { readonly file: string; readonly path: string } { + const [file, path, extra] = ref.split("#"); + if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) { + throw new Error(`${ref} must use path/to/file.yaml#object.path syntax`); + } + if (file.startsWith("/") || file.includes("\0") || file.includes("..") || !file.startsWith("config/") || !file.endsWith(".yaml")) { + throw new Error(`${ref} must reference a repo-relative config/*.yaml file without ..`); + } + return { file, path }; +} + +function valueAtConfigPath(value: unknown, path: string): unknown { + let current = value; + for (const segment of path.split(".")) { + if (segment.length === 0) return undefined; + current = valueAtSegment(current, segment); + if (current === undefined) return undefined; + } + return current; +} + +function valueAtSegment(value: unknown, segment: string): unknown { + let rest = segment; + let current = value; + const key = /^[A-Za-z0-9_-]+/u.exec(rest)?.[0] ?? null; + if (key !== null) { + if (!isRecord(current)) return undefined; + current = current[key]; + rest = rest.slice(key.length); + } + if (key === null && !rest.startsWith("[")) return undefined; + while (rest.length > 0) { + const match = /^\[([^\]]+)\]/u.exec(rest); + if (match === null) return undefined; + current = valueAtBracket(current, match[1]); + if (current === undefined) return undefined; + rest = rest.slice(match[0].length); + } + return current; +} + +function valueAtBracket(value: unknown, selector: string): unknown { + if (/^\d+$/u.test(selector)) { + if (!Array.isArray(value)) return undefined; + return value[Number(selector)]; + } + const match = /^([A-Za-z0-9_-]+)=([A-Za-z0-9_.:/@+-]+)$/u.exec(selector); + if (match === null || !Array.isArray(value)) return undefined; + const [, key, expected] = match; + return value.find((item) => isRecord(item) && item[key] === expected); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/scripts/src/hwlab-node-web-sentinel-config.ts b/scripts/src/hwlab-node-web-sentinel-config.ts index 3186d814..3a74d8ed 100644 --- a/scripts/src/hwlab-node-web-sentinel-config.ts +++ b/scripts/src/hwlab-node-web-sentinel-config.ts @@ -1,9 +1,7 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel. // Responsibility: Redacted YAML configRef graph for web-probe sentinel plan/status. -import { createHash } from "node:crypto"; -import { existsSync, readFileSync } from "node:fs"; -import { rootPath } from "./config"; +import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref"; import { HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS, type HwlabRuntimeLaneSpec, type HwlabRuntimeWebProbeSentinelConfigRefKey } from "./hwlab-node-lanes"; import { effectiveWebProbeSentinelPublicExposure, resolveWebProbeSentinel, webProbeSentinelRegistryRows, type WebProbeSentinelRegistryRow } from "./hwlab-node-web-sentinel-resolver"; import type { RenderedCliResult } from "./output"; @@ -212,7 +210,7 @@ export function webProbeSentinelConfigPlan(spec: HwlabRuntimeLaneSpec, action: W const selected = resolveWebProbeSentinel(spec, sentinelId); const refs = HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS - .map((key) => readSentinelConfigRef(key, selected.configRefs[key])) + .map((key) => readSentinelConfigRef(spec, key, selected.configRefs[key])) .map((ref) => effectiveConfigRefStatus(spec, selected.id, ref)); const conflicts = selected.enabled ? crossReferenceConflicts(spec, refs) : []; const refBlocked = refs.some((ref) => !ref.present || !ref.targetPresent || ref.missingFields.length > 0 || ref.conflicts.length > 0 || ref.error !== null); @@ -255,28 +253,22 @@ export function withWebProbeSentinelConfigRendered(value: WebProbeSentinelConfig }; } -function readSentinelConfigRef(key: HwlabRuntimeWebProbeSentinelConfigRefKey, ref: string): InternalConfigRefStatus { - const parsed = parseConfigRef(ref); - if (parsed.error !== null) return emptyRefStatus(key, ref, parsed.file, parsed.path, parsed.error); - const absPath = rootPath(parsed.file); - if (!existsSync(absPath)) return emptyRefStatus(key, ref, parsed.file, parsed.path, `${parsed.file} does not exist`); +function readSentinelConfigRef(spec: HwlabRuntimeLaneSpec, key: HwlabRuntimeWebProbeSentinelConfigRefKey, ref: string): InternalConfigRefStatus { try { - const text = readFileSync(absPath, "utf8"); - const sha256 = `sha256:${createHash("sha256").update(text).digest("hex")}`; - const doc = Bun.YAML.parse(text) as unknown; - const target = valueAtPath(doc, parsed.path); + const resolved = readWebProbeSentinelConfigRef(spec, ref); + const target = resolved.target; const targetKind = target === undefined ? "missing" : targetKindOf(target); const missingFields = target === undefined ? ["target"] : missingFieldsForTarget(key, target); return { key, ref, - file: parsed.file, - path: parsed.path, + file: resolved.file, + path: resolved.path, present: true, targetPresent: target !== undefined, targetKind, - sha256, - byteCount: Buffer.byteLength(text), + sha256: resolved.sha256, + byteCount: resolved.byteCount, missingFields, conflicts: [], summary: summarizeTarget(key, target), @@ -285,21 +277,10 @@ function readSentinelConfigRef(key: HwlabRuntimeWebProbeSentinelConfigRefKey, re }; } catch (error) { const message = error instanceof Error ? error.message : String(error); - return emptyRefStatus(key, ref, parsed.file, parsed.path, message); + return emptyRefStatus(key, ref, "", "", message); } } -function parseConfigRef(ref: string): { readonly file: string; readonly path: string; readonly error: string | null } { - const [file, path, extra] = ref.split("#"); - if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) { - return { file: file ?? "", path: path ?? "", error: "configRef must use path/to/file.yaml#object.path" }; - } - if (file.startsWith("/") || file.includes("\0") || file.includes("..") || !file.startsWith("config/") || !file.endsWith(".yaml")) { - return { file, path, error: "configRef file must be repo-relative config/*.yaml without .." }; - } - return { file, path, error: null }; -} - function emptyRefStatus(key: HwlabRuntimeWebProbeSentinelConfigRefKey, ref: string, file: string, path: string, error: string): InternalConfigRefStatus { return { key, @@ -348,8 +329,6 @@ function crossReferenceConflicts(spec: HwlabRuntimeLaneSpec, refs: readonly Inte requireEquals(conflicts, byKey.get("runtime"), "namespace", spec.runtimeNamespace, `selected namespace ${spec.runtimeNamespace}`); } - const promptSetRef = byKey.get("promptSet")?.ref ?? null; - const reportViewsRef = byKey.get("reportViews")?.ref ?? null; const scenarioIds = new Set(); const scenarioProviders = new Set(); for (const [index, scenario] of scenarios.entries()) { @@ -357,8 +336,8 @@ function crossReferenceConflicts(spec: HwlabRuntimeLaneSpec, refs: readonly Inte if (id !== null) scenarioIds.add(id); const provider = stringAt(scenario, "providerProfile"); if (provider !== null) scenarioProviders.add(provider); - if (promptSetRef !== null) requireEquals(conflicts, byKey.get("scenarios"), `[${index}].promptSetRef`, promptSetRef, "promptSet configRef"); - if (reportViewsRef !== null) requireEquals(conflicts, byKey.get("scenarios"), `[${index}].reportViewRef`, reportViewsRef, "reportViews configRef"); + requireReadableConfigRef(spec, conflicts, byKey.get("scenarios"), `[${index}].promptSetRef`, scenario.promptSetRef); + requireReadableConfigRef(spec, conflicts, byKey.get("scenarios"), `[${index}].reportViewRef`, scenario.reportViewRef); } const promptProvider = promptSet === null ? null : stringAt(promptSet, "providerProfile"); @@ -384,6 +363,15 @@ function crossReferenceConflicts(spec: HwlabRuntimeLaneSpec, refs: readonly Inte return conflicts; } +function requireReadableConfigRef(spec: HwlabRuntimeLaneSpec, conflicts: string[], ref: InternalConfigRefStatus | undefined, path: string, value: unknown): void { + if (typeof value !== "string" || value.length === 0) return; + try { + readWebProbeSentinelConfigRef(spec, value); + } catch (error) { + conflicts.push(`${ref?.file ?? "-"}#${ref?.path ?? "-"}.${path}=${value} is not readable: ${error instanceof Error ? error.message : String(error)}`); + } +} + function requireEquals(conflicts: string[], ref: InternalConfigRefStatus | undefined, path: string, expected: string, expectedLabel: string): void { if (ref === undefined) return; const actual = stringAt(ref.target, path); diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts new file mode 100644 index 00000000..3068a615 --- /dev/null +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -0,0 +1,1484 @@ +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. +// Responsibility: Quick-verify observe orchestration and artifact interpretation for web-probe sentinel P5 validation. +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { CommandResult } from "./command"; +import { runCommand } from "./command"; +import { repoRoot, rootPath } from "./config"; +import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; +import type { ChildCliResult, CompactCommandResult, SentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; +import { + arrayAt, + compactCommand, + displayPath, + isRecord, + numberAt, + numberAtNullable, + parseEnvFile, + parseJsonObject, + quickVerifyAccountEnv, + record, + recordTarget, + secretSourcePaths, + shellQuote, + short, + stringAt, + stringAtNullable, + text, +} from "./hwlab-node-web-sentinel-cicd"; + +function printQuickVerifyProgress(state: SentinelCicdState, runId: string | null, phase: string, status: string, extra: Record = {}): void { + const compactExtra = Object.fromEntries(Object.entries(extra).map(([key, value]) => { + if (typeof value === "string") return [key, short(value)]; + if (Array.isArray(value)) return [key, value.slice(0, 8)]; + return [key, value]; + })); + process.stdout.write(`${JSON.stringify({ + event: "sentinel.quick-verify.progress", + at: new Date().toISOString(), + node: state.spec.nodeId, + lane: state.spec.lane, + sentinelId: state.sentinelId, + runId, + ...compactExtra, + phase, + status, + valuesRedacted: true, + })}\n`); +} + +export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, timeoutSeconds: number): Record { + const startedAt = Date.now(); + const elapsedMs = () => Date.now() - startedAt; + const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId"); + const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); + const scenario = findScenario(state, scenarioId); + if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; + const commandSequence = arrayAt(scenario, "commandSequence").map(record); + const needsPromptSet = commandSequence.some((item) => stringAt(item, "type") === "sendPrompt"); + const prompts = needsPromptSet + ? readPromptSetForScenario(state, scenario) + : { ok: true as const, prompts: [], summary: { source: "not-required", promptCount: 0, valuesRedacted: true } }; + if (!prompts.ok) return { ok: false, status: "blocked", reason: "prompt-source-unavailable", promptSource: prompts, valuesRedacted: true }; + const accountEnv = quickVerifyAccountEnv(state); + if (!accountEnv.ok) { + const findings = [{ + id: "quick-verify-account-secret-missing", + severity: "red", + count: arrayAt(accountEnv.summary, "missing").length || 1, + summary: "quick verify could not materialize YAML-declared web account credentials for the observer runner.", + missing: arrayAt(accountEnv.summary, "missing"), + valuesRedacted: true, + }]; + return recordQuickVerify(state, { + ok: false, + runId: `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`, + scenarioId, + reason, + status: "blocked", + observerId: null, + elapsedMs: 0, + businessStatus: quickVerifyBusinessStatus("quick-verify-account-secret-missing", 0, null, null, 0, maxSeconds), + steps: [{ phase: "quick-verify-account-env", ok: false, result: accountEnv.summary }], + failure: "quick-verify-account-secret-missing", + findingCount: findings.length, + findings, + promptSource: prompts.summary, + accountEnv: accountEnv.summary, + views: { + summary: { renderedText: renderQuickVerifySummary({ scenarioId, artifactSummary: { ok: false, findings, findingCount: findings.length }, steps: [], publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ scenarioId, steps: [], findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + }, + warnings: [], + valuesRedacted: true, + }); + } + const sampleIntervalMs = numberAt(scenario, "sampleIntervalMs"); + const warningBudgetSeconds = maxSeconds; + const hardBudgetSeconds = Math.min(timeoutSeconds, Math.max(maxSeconds, numberAt(scenario, "maxRunSeconds"))); + const elapsedWarnings = () => targetValidationElapsedWarnings(elapsedMs(), "quick verify confirm-wait", warningBudgetSeconds); + const deadline = Date.now() + hardBudgetSeconds * 1000; + const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; + printQuickVerifyProgress(state, runId, "start", "running", { scenarioId, reason, warningBudgetSeconds, hardBudgetSeconds, timeoutSeconds }); + const steps: Record[] = []; + const startArgs = [ + "web-probe", "observe", "start", + "--node", state.spec.nodeId, + "--lane", state.spec.lane, + "--target-path", stringAt(scenario, "observeTargetPath"), + "--sample-interval-ms", String(sampleIntervalMs), + "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), + "--command-timeout-seconds", "55", + ]; + const viewport = stringAtNullable(scenario, "viewport"); + if (viewport !== null) startArgs.push("--viewport", viewport); + printQuickVerifyProgress(state, runId, "observe-start", "running", { targetPath: stringAt(scenario, "observeTargetPath"), remainingSeconds: remainingSeconds(deadline, 55) }); + const started = runChildCli(startArgs, remainingSeconds(deadline, 55), undefined, accountEnv.env); + steps.push({ phase: "observe-start", ok: started.ok, result: started.result }); + const observerId = observerIdFromText(String(record(started.result).stdoutPreview ?? "")); + printQuickVerifyProgress(state, runId, "observe-start", started.ok && observerId !== null ? "succeeded" : "failed", { observerId, exitCode: record(started.result).exitCode ?? null, timedOut: record(started.result).timedOut === true, elapsedMs: elapsedMs() }); + if (!started.ok || observerId === null) { + const findings = quickVerifyControlFindings("observe-start-failed", 0, null, null); + return recordQuickVerify(state, { + ok: false, + runId, + scenarioId, + reason, + status: "blocked", + observerId, + elapsedMs: elapsedMs(), + businessStatus: quickVerifyBusinessStatus("observe-start-failed", 0, null, null, elapsedMs(), maxSeconds), + steps, + failure: "observe-start-failed", + findingCount: findings.length, + findings, + warnings: elapsedWarnings(), + valuesRedacted: true, + }); + } + printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 55) }); + const startupReady = waitForQuickVerifyObserverStartup(state, observerId, deadline, sampleIntervalMs, warningBudgetSeconds); + steps.push({ phase: "observe-wait-startup-ready", ok: startupReady.ok, result: startupReady }); + printQuickVerifyProgress(state, runId, "observe-wait-startup-ready", startupReady.ok === true ? "succeeded" : "failed", { observerId, failure: startupReady.failure ?? null, status: startupReady.status ?? null, heartbeatStatus: startupReady.heartbeatStatus ?? null, elapsedMs: elapsedMs() }); + if (startupReady.ok !== true) { + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex: 0, + steps, + failure: text(startupReady.failure ?? "observe-startup-ready-wait-failed"), + elapsedMs: elapsedMs(), + warnings: mergeWarnings(Array.isArray(startupReady.warnings) ? startupReady.warnings : [], elapsedWarnings()), + promptSource: prompts.summary, + })); + } + let promptIndex = 0; + const sessionInvarianceChecks = sessionInvarianceChecksByRound(scenario); + const nonBlockingCanaryWarnings: string[] = []; + for (const item of commandSequence) { + const type = stringAt(item, "type"); + const repeat = Math.max(1, typeof item.repeat === "number" && Number.isFinite(item.repeat) ? Math.trunc(item.repeat) : 1); + for (let index = 0; index < repeat; index += 1) { + if (Date.now() >= deadline) { + printQuickVerifyProgress(state, runId, "timeout", "failed", { observerId, promptIndex, elapsedMs: elapsedMs(), hardBudgetSeconds }); + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex, + steps, + failure: "quick-verify-timeout-over-budget", + elapsedMs: elapsedMs(), + warnings: mergeWarnings(`quick verify exceeded the hard ${hardBudgetSeconds}s execution budget after the configured ${warningBudgetSeconds}s targetValidation warning budget.`, elapsedWarnings()), + promptSource: prompts.summary, + })); + } + const commandWaitMs = Math.max(1000, Math.trunc(numberAtNullable(item, "commandWaitMs") ?? numberAtNullable(scenario, "commandWaitMs") ?? 55_000)); + const commandTimeoutSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "commandTimeoutSeconds") ?? numberAtNullable(scenario, "commandTimeoutSeconds") ?? 55)); + const turnWaitChunkSeconds = Math.max(1, Math.trunc(numberAtNullable(item, "turnWaitChunkSeconds") ?? numberAtNullable(scenario, "turnWaitChunkSeconds") ?? 55)); + const args = ["web-probe", "observe", "command", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--type", type, "--wait-ms", String(commandWaitMs), "--command-timeout-seconds", String(remainingSeconds(deadline, commandTimeoutSeconds))]; + if (type === "selectProvider") args.push("--provider", stringAt(item, "provider")); + if (type === "loginAccount" || type === "listSessions" || type === "logout") { + const accountId = stringAtNullable(item, "accountId"); + if (accountId !== null) args.push("--account-id", accountId); + } + if (type === "switchSessions") { + const fromAccountId = stringAtNullable(item, "fromAccountId"); + const toAccountId = stringAtNullable(item, "toAccountId"); + if (fromAccountId !== null) args.push("--from-account-id", fromAccountId); + if (toAccountId !== null) args.push("--to-account-id", toAccountId); + } + if (type === "sendPrompt") { + args.push("--text", prompts.prompts[promptIndex % prompts.prompts.length] ?? ""); + args.push("--expected-action-wait-ms", String(numberAtNullable(item, "expectedActionWaitMs") ?? 45000)); + promptIndex += 1; + } + appendScenarioObserveCommandArgs(args, item, { skipText: type === "sendPrompt" }); + printQuickVerifyProgress(state, runId, `observe-command-${type}`, "running", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, repeatIndex: index + 1, repeat }); + const commandResult = runChildCli(args, remainingSeconds(deadline, Math.max(60, commandTimeoutSeconds + 10))); + steps.push({ phase: `observe-command-${type}`, ok: commandResult.ok, promptIndex: type === "sendPrompt" ? promptIndex : null, result: commandResult.result }); + printQuickVerifyProgress(state, runId, `observe-command-${type}`, commandResult.ok ? "succeeded" : "failed", { observerId, promptIndex: type === "sendPrompt" ? promptIndex : null, exitCode: record(commandResult.result).exitCode ?? null, timedOut: record(commandResult.result).timedOut === true, elapsedMs: elapsedMs() }); + if (!commandResult.ok) { + if (type === "sendPrompt") { + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); + const delayedWaitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); + steps.push({ phase: "observe-wait-turn-terminal-after-command-timeout", ok: delayedWaitResult.ok, promptIndex, result: delayedWaitResult }); + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal-after-command-timeout", delayedWaitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: delayedWaitResult.failure ?? null, status: delayedWaitResult.status ?? null, traceId: delayedWaitResult.traceId ?? null, elapsedMs: elapsedMs() }); + if (delayedWaitResult.ok === true) { + const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); + nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); + printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); + if (invariantResult.ok !== true) { + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex, + steps, + failure: text(record(invariantResult).failure ?? "observe-session-invariance-failed"), + promptSource: prompts.summary, + elapsedMs: elapsedMs(), + warnings: mergeWarnings(record(invariantResult).warnings, elapsedWarnings()), + })); + } + continue; + } + } + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex, + steps, + failure: `observe-command-${type}-failed`, + elapsedMs: elapsedMs(), + warnings: elapsedWarnings(), + promptSource: prompts.summary, + })); + } + if (type === "sendPrompt") { + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", "running", { observerId, promptIndex, remainingSeconds: remainingSeconds(deadline, turnWaitChunkSeconds) }); + const waitResult = waitForQuickVerifyPromptTurn(state, observerId, promptIndex, deadline, sampleIntervalMs, warningBudgetSeconds, turnWaitChunkSeconds); + steps.push({ phase: "observe-wait-turn-terminal", ok: waitResult.ok, promptIndex, result: waitResult }); + printQuickVerifyProgress(state, runId, "observe-wait-turn-terminal", waitResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: waitResult.failure ?? null, status: waitResult.status ?? null, traceId: waitResult.traceId ?? null, elapsedMs: elapsedMs() }); + if (waitResult.ok !== true) { + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex, + steps, + failure: text(waitResult.failure ?? "observe-turn-terminal-wait-failed"), + promptSource: prompts.summary, + elapsedMs: elapsedMs(), + warnings: mergeWarnings(Array.isArray(waitResult.warnings) ? waitResult.warnings : [], elapsedWarnings()), + })); + } + const invariantResult = runQuickVerifySessionInvarianceChecks(state, observerId, sessionInvarianceChecks.get(promptIndex) ?? [], deadline, promptIndex, steps); + nonBlockingCanaryWarnings.push(...mergeWarnings(record(invariantResult).warnings)); + printQuickVerifyProgress(state, runId, "observe-session-invariance", invariantResult.ok === true ? "succeeded" : "failed", { observerId, promptIndex, failure: record(invariantResult).failure ?? null, elapsedMs: elapsedMs() }); + if (invariantResult.ok !== true) { + return recordQuickVerify(state, finalizeQuickVerifyFailure(state, { + runId, + scenarioId, + reason, + observerId, + promptIndex, + steps, + failure: text(invariantResult.failure ?? "observe-session-invariance-check-failed"), + promptSource: prompts.summary, + elapsedMs: elapsedMs(), + warnings: mergeWarnings(nonBlockingCanaryWarnings, elapsedWarnings()), + })); + } + } + } + } + printQuickVerifyProgress(state, runId, "observe-analyze", "running", { observerId, remainingSeconds: remainingSeconds(deadline, 120) }); + const analysis = runChildCli(["web-probe", "observe", "analyze", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--command-timeout-seconds", String(remainingSeconds(deadline, 120))], remainingSeconds(deadline, 120)); + steps.push({ phase: "observe-analyze", ok: analysis.ok, result: analysis.result }); + printQuickVerifyProgress(state, runId, "observe-analyze", analysis.ok ? "succeeded" : "failed", { observerId, exitCode: record(analysis.result).exitCode ?? null, timedOut: record(analysis.result).timedOut === true, elapsedMs: elapsedMs() }); + const indexEntry = readLocalObserveIndex(observerId); + const artifactSummary = indexEntry === null ? { ok: false, reason: "observe-index-entry-missing", observerId, valuesRedacted: true } : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, remainingSeconds(deadline, 30)); + const turnSummary = collectObserveView(state, observerId, "turn-summary", null, remainingSeconds(deadline, 30)); + const traceFrame = collectObserveView(state, observerId, "trace-frame", promptIndex > 0 ? promptIndex : null, remainingSeconds(deadline, 30)); + const controlFindings = quickVerifyControlFindings(null, promptIndex, turnSummary, traceFrame); + const artifactSummaryRecord = record(artifactSummary); + const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; + const findings = mergeFindingRecords(artifactFindings, controlFindings); + const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); + const analysisWarnings = analysis.ok ? [] : ["quick verify analyze command returned non-zero but a readable analysis artifact was produced; targetValidation is using artifact severity plus control blockers."]; + const ok = record(artifactSummary).ok === true && controlFindings.length === 0 && blockingFindings.length === 0; + const businessStatus = quickVerifyBusinessStatus(null, promptIndex, turnSummary, traceFrame, elapsedMs(), maxSeconds); + printQuickVerifyProgress(state, runId, "record-report", ok ? "succeeded" : "blocked", { observerId, reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), findingCount: findings.length, blockingFindingCount: blockingFindings.length, controlFindingCount: controlFindings.length, elapsedMs: elapsedMs() }); + return recordQuickVerify(state, { + ok, + runId, + scenarioId, + reason, + status: ok ? "analyzed" : "blocked", + observerId, + elapsedMs: elapsedMs(), + businessStatus, + stateDir: indexEntry?.stateDir ?? null, + reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), + findingCount: findings.length, + artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, + failure: controlFindings.length > 0 ? "quick-verify-no-business-turn" : blockingFindings.length > 0 ? "quick-verify-blocking-findings" : null, + promptSource: prompts.summary, + accountEnv: accountEnv.summary, + steps, + analysis: artifactSummary, + views: { + summary: { renderedText: renderQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId, scenarioId, observerId, artifactSummary, steps, findings, accountEnv: accountEnv.summary, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, + "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, + }, + findings, + screenshot: record(artifactSummary).screenshot, + publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), + warnings: mergeWarnings(analysisWarnings, nonBlockingCanaryWarnings, elapsedWarnings()), + valuesRedacted: true, + }); +} + +function sessionInvarianceChecksByRound(scenario: Record): Map[]> { + const checks = new Map[]>(); + const items = Array.isArray(scenario.sessionInvarianceChecks) ? scenario.sessionInvarianceChecks.map(record) : []; + for (const item of items) { + const afterRound = typeof item.afterRound === "number" && Number.isInteger(item.afterRound) ? item.afterRound : null; + if (afterRound === null || afterRound < 0) continue; + const list = checks.get(afterRound) ?? []; + list.push(item); + checks.set(afterRound, list); + } + return checks; +} + +function runQuickVerifySessionInvarianceChecks( + state: SentinelCicdState, + observerId: string, + checks: readonly Record[], + deadline: number, + promptIndex: number, + steps: Record[], +): Record { + const warnings: string[] = []; + for (const check of checks) { + const checkId = nonEmptyString(check.id) ?? `after-round-${promptIndex}`; + const blocking = check.blocking === true; + const commands: { readonly type: string; readonly enabled: boolean }[] = [ + { type: "refreshCurrentSession", enabled: check.refreshCurrent === true }, + { type: "switchAwayAndBack", enabled: check.switchAwayAndBack === true }, + { type: "assertSessionInvariant", enabled: check.assertSessionInvariant !== false }, + ]; + for (const command of commands) { + if (!command.enabled) continue; + const args = [ + "web-probe", "observe", "command", observerId, + "--node", state.spec.nodeId, + "--lane", state.spec.lane, + "--type", command.type, + "--after-round", String(promptIndex), + "--wait-ms", "55000", + "--command-timeout-seconds", String(remainingSeconds(deadline, 55)), + ]; + const severity = nonEmptyString(check.severity); + const findingId = nonEmptyString(check.findingId); + const expectedSentinelRange = nonEmptyString(check.expectedSentinelRange); + const alternateSessionStrategy = nonEmptyString(check.alternateSessionStrategy); + if (severity !== null) args.push("--severity", severity); + if (findingId !== null) args.push("--finding-id", findingId); + if (expectedSentinelRange !== null) args.push("--expected-sentinel-range", expectedSentinelRange); + if (command.type === "switchAwayAndBack" && alternateSessionStrategy !== null) args.push("--alternate-session-strategy", alternateSessionStrategy); + if (command.type === "assertSessionInvariant" && check.requireComposerReady === true) args.push("--require-composer-ready"); + args.push(check.blocking === true ? "--blocking" : "--non-blocking"); + const result = runChildCli(args, remainingSeconds(deadline, 60)); + steps.push({ phase: `observe-session-invariance-${command.type}`, ok: result.ok, promptIndex, checkId, result: result.result }); + if (!result.ok) { + if (!blocking) { + warnings.push(`non-blocking session invariance canary ${checkId}/${command.type} failed after round ${promptIndex}; continuing Code Agent multi-round quick verify because scenario marks this check blocking=false.`); + continue; + } + return { ok: false, failure: `observe-session-invariance-${command.type}-failed`, checkId, promptIndex, valuesRedacted: true }; + } + } + } + return { ok: true, promptIndex, checkCount: checks.length, warnings, valuesRedacted: true }; +} + +function appendScenarioObserveCommandArgs(args: string[], item: Record, options: { readonly skipText?: boolean } = {}): void { + const mappings: readonly (readonly [string, string])[] = [ + ["path", "--path"], + ["label", "--label"], + ["sessionId", "--session-id"], + ["provider", "--provider"], + ["accountId", "--account-id"], + ["fromAccountId", "--from-account-id"], + ["toAccountId", "--to-account-id"], + ["sourceId", "--source-id"], + ["fileRef", "--file-ref"], + ["filename", "--filename"], + ["taskRef", "--task-ref"], + ["taskId", "--task-id"], + ["task", "--task"], + ["field", "--field"], + ["link", "--link"], + ["title", "--title"], + ["body", "--body"], + ["status", "--status"], + ["hwpodId", "--hwpod-id"], + ["nodeId", "--node-id"], + ["workspaceRoot", "--workspace-root"], + ["root", "--root"], + ]; + for (const [key, flag] of mappings) { + if (args.includes(flag)) continue; + const value = stringAtNullable(item, key); + if (value !== null) args.push(flag, value); + } + if (options.skipText !== true && !args.includes("--text")) { + const text = stringAtNullable(item, "text") ?? stringAtNullable(item, "value"); + if (text !== null) args.push("--text", text); + } + if (item.waitProjectManagementReady === true && !args.includes("--wait-project-management-ready")) args.push("--wait-project-management-ready"); +} + +function finalizeQuickVerifyFailure(state: SentinelCicdState, input: { + readonly runId: string; + readonly scenarioId: string; + readonly reason: string; + readonly observerId: string; + readonly promptIndex: number; + readonly steps: readonly Record[]; + readonly failure: string; + readonly promptSource?: Record; + readonly elapsedMs?: number; + readonly warnings?: readonly unknown[]; +}): Record { + const cleanupSteps: Record[] = []; + if (input.promptIndex > 0) { + const cancel = runChildCli([ + "web-probe", "observe", "command", input.observerId, + "--node", state.spec.nodeId, + "--lane", state.spec.lane, + "--type", "cancel", + "--wait-ms", "55000", + "--command-timeout-seconds", "55", + ], 60); + cleanupSteps.push({ phase: "observe-cancel-after-failure", ok: cancel.ok, result: cancel.result }); + } + const stop = runChildCli([ + "web-probe", "observe", "stop", input.observerId, + "--node", state.spec.nodeId, + "--lane", state.spec.lane, + "--force", + "--command-timeout-seconds", "55", + ], 30); + cleanupSteps.push({ phase: "observe-stop-after-failure", ok: stop.ok, result: stop.result }); + const analysis = runChildCli([ + "web-probe", "observe", "analyze", input.observerId, + "--node", state.spec.nodeId, + "--lane", state.spec.lane, + "--command-timeout-seconds", "55", + ], 60); + cleanupSteps.push({ phase: "observe-analyze-after-failure", ok: analysis.ok, result: analysis.result }); + const indexEntry = readLocalObserveIndex(input.observerId); + const artifactSummary = indexEntry === null + ? { ok: false, reason: "observe-index-entry-missing", observerId: input.observerId, valuesRedacted: true } + : readAnalysisSummaryFromWorkspace(state, indexEntry.stateDir, 30); + const turnSummary = collectObserveView(state, input.observerId, "turn-summary", null, 30); + const traceFrame = collectObserveView(state, input.observerId, "trace-frame", input.promptIndex > 0 ? input.promptIndex : null, 30); + const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(input.promptIndex, turnSummary, traceFrame); + const controlFindings = quickVerifyControlFindings(input.failure, input.promptIndex, turnSummary, traceFrame); + const artifactSummaryRecord = record(artifactSummary); + const artifactFindings = Array.isArray(artifactSummaryRecord.findings) ? artifactSummaryRecord.findings.map(record) : []; + const findings = mergeFindingRecords(artifactFindings, controlFindings); + const blockingFindings = findings.filter(isQuickVerifyBlockingFinding); + const recoveredWaitFailure = durableBusinessTurn + && isRecoverableQuickVerifyWaitFailure(input.failure) + && record(artifactSummary).ok === true + && controlFindings.length === 0 + && blockingFindings.length === 0; + const businessStatus = quickVerifyBusinessStatus(input.failure, input.promptIndex, turnSummary, traceFrame, input.elapsedMs ?? null, numberAt(state.cicd, "targetValidation.maxSeconds")); + return { + ok: recoveredWaitFailure, + runId: input.runId, + scenarioId: input.scenarioId, + reason: input.reason, + status: recoveredWaitFailure ? "analyzed" : "blocked", + observerId: input.observerId, + elapsedMs: input.elapsedMs ?? null, + businessStatus, + stateDir: indexEntry?.stateDir ?? null, + reportJsonSha256: stringAtNullable(artifactSummary, "reportJsonSha256"), + findingCount: findings.length, + artifactCount: numberAtNullable(artifactSummary, "artifactCount") ?? 0, + failure: recoveredWaitFailure ? null : input.failure, + promptSource: input.promptSource, + steps: [...input.steps, ...cleanupSteps], + analysis: artifactSummary, + views: { + summary: { renderedText: renderQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "auth-session-switch-summary": { renderedText: renderAuthSessionSwitchQuickVerifySummary({ runId: input.runId, scenarioId: input.scenarioId, observerId: input.observerId, artifactSummary, steps: input.steps, findings, publicOrigin: stringAt(state.publicExposure, "publicBaseUrl") }) }, + "turn-summary": { renderedText: typeof turnSummary.renderedText === "string" ? turnSummary.renderedText : null, ok: turnSummary.ok }, + "trace-frame": { renderedText: typeof traceFrame.renderedText === "string" ? traceFrame.renderedText : null, ok: traceFrame.ok }, + }, + findings, + screenshot: record(artifactSummary).screenshot, + publicOrigin: stringAt(state.publicExposure, "publicBaseUrl"), + warnings: mergeWarnings( + Array.isArray(input.warnings) ? input.warnings : [], + recoveredWaitFailure ? ["quick verify wait command timed out, but collected turn-summary/trace-frame artifacts show a durable completed business turn; treating the wait timeout as a non-blocking tool finding."] : [], + targetValidationElapsedWarnings(input.elapsedMs ?? null, "quick verify confirm-wait", numberAt(state.cicd, "targetValidation.maxSeconds")), + ), + valuesRedacted: true, + }; +} + +function recordQuickVerify(state: SentinelCicdState, payload: Record): Record { + const views = compactQuickVerifyRecordViews(record(payload.views)); + const summary = { + reason: payload.reason, + status: payload.status, + businessStatus: payload.businessStatus ?? null, + elapsedMs: payload.elapsedMs, + failure: payload.failure, + warnings: Array.isArray(payload.warnings) ? payload.warnings : [], + analysis: compactQuickVerifyRecordAnalysis(payload.analysis), + promptSource: payload.promptSource, + steps: Array.isArray(payload.steps) ? payload.steps.map(compactQuickVerifyRecordStep) : [], + valuesRedacted: true, + }; + const recordResult = callSentinelService(state, "POST", "/api/runs/record", { + runId: payload.runId, + scenarioId: payload.scenarioId, + status: payload.status, + observerId: payload.observerId, + stateDir: payload.stateDir, + reportJsonSha256: payload.reportJsonSha256, + findingCount: payload.findingCount, + artifactCount: payload.artifactCount, + summary, + businessStatus: payload.businessStatus ?? null, + findings: payload.findings, + views, + screenshot: payload.screenshot, + publicOrigin: payload.publicOrigin ?? stringAt(state.publicExposure, "publicBaseUrl"), + maintenance: payload.reason === "maintenance-stop", + valuesRedacted: true, + }, 60); + return withWarnings({ ...payload, views, recordResult, valuesRedacted: true }, recordResult.ok === true ? [] : ["quick verify completed but sentinel report index record failed; report/dashboard may lag until record payload is reduced or retried."]); +} + +function compactQuickVerifyRecordViews(views: Record): Record { + const compacted: Record = {}; + for (const [key, value] of Object.entries(views)) { + const item = record(value); + const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000; + compacted[key] = { + ...item, + renderedText: boundQuickVerifyRecordText(item.renderedText, limit), + valuesRedacted: true, + }; + } + return compacted; +} + +function compactQuickVerifyRecordAnalysis(value: unknown): Record | null { + const item = record(value); + if (Object.keys(item).length === 0) return null; + return { + ok: item.ok === true ? true : item.ok === false ? false : null, + reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null, + reason: stringAtNullable(item, "reason"), + stateDir: stringAtNullable(item, "stateDir"), + reportJsonSha256: stringAtNullable(item, "reportJsonSha256"), + reportMdSha256: stringAtNullable(item, "reportMdSha256"), + findingCount: numberAtNullable(item, "findingCount"), + artifactCount: numberAtNullable(item, "artifactCount"), + counts: compactQuickVerifyRecordCounts(record(item.counts)), + screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)), + findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [], + pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [], + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordCounts(value: Record): Record { + return { + samples: numberAtNullable(value, "samples"), + control: numberAtNullable(value, "control"), + network: numberAtNullable(value, "network"), + console: numberAtNullable(value, "console"), + errors: numberAtNullable(value, "errors"), + artifacts: numberAtNullable(value, "artifacts"), + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordScreenshot(value: Record): Record | null { + if (Object.keys(value).length === 0) return null; + return { + path: stringAtNullable(value, "path"), + sha256: stringAtNullable(value, "sha256"), + bytes: numberAtNullable(value, "bytes"), + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordFinding(value: unknown): Record { + const item = record(value); + return { + id: stringAtNullable(item, "id"), + kind: stringAtNullable(item, "kind"), + code: stringAtNullable(item, "code"), + severity: stringAtNullable(item, "severity"), + level: stringAtNullable(item, "level"), + count: numberAtNullable(item, "count"), + summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220), + rootCause: boundQuickVerifyRecordText(item.rootCause, 140), + rootCauseStatus: boundQuickVerifyRecordText(item.rootCauseStatus, 90), + rootCauseConfidence: boundQuickVerifyRecordText(item.rootCauseConfidence, 40), + nextAction: boundQuickVerifyRecordText(item.nextAction, 240), + evidenceSummary: stringAtNullable(item, "evidenceSummary") ?? compactQuickVerifyFindingEvidence(item.evidence), + timingSourceOfTruth: boundQuickVerifyRecordText(item.timingSourceOfTruth ?? item.expectedElapsedSource ?? item.evidenceKind, 100), + timingStatus: boundQuickVerifyRecordText(item.timingStatus, 60), + timingAlert: item.timingAlert === true, + blocking: item.blocking === true, + valuesRedacted: true, + }; +} + +function compactQuickVerifyFindingEvidence(value: unknown): string | null { + const item = record(value); + if (Object.keys(item).length === 0) return null; + const keys = [ + "http404Count", + "responseErrorCount", + "requestFailedCount", + "statuses", + "afterProjectedSeqs", + "sinceSeqs", + "traceIds", + "maxFallbackRatio", + "maxFallbackTitleCount", + "overThresholdSampleCount", + "majorityFallbackSampleCount", + ]; + const compact: Record = {}; + for (const key of keys) { + const raw = item[key]; + if (raw === null || raw === undefined) continue; + compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw; + } + return Object.keys(compact).length === 0 ? null : boundQuickVerifyRecordText(JSON.stringify(compact), 240); +} + +function compactQuickVerifyRecordStep(value: unknown): Record { + const item = record(value); + return { + phase: stringAtNullable(item, "phase"), + ok: item.ok === true ? true : item.ok === false ? false : null, + promptIndex: numberAtNullable(item, "promptIndex"), + checkId: stringAtNullable(item, "checkId"), + failure: stringAtNullable(item, "failure"), + result: compactQuickVerifyRecordStepResult(record(item.result)), + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordStepResult(value: Record): Record { + return { + ok: value.ok === true ? true : value.ok === false ? false : null, + status: stringAtNullable(value, "status"), + view: stringAtNullable(value, "view"), + exitCode: numberAtNullable(value, "exitCode"), + timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, + stdoutBytes: numberAtNullable(value, "stdoutBytes"), + stderrBytes: numberAtNullable(value, "stderrBytes"), + stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240), + stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240), + valuesRedacted: true, + }; +} + +function boundQuickVerifyRecordText(value: unknown, maxChars: number): string | null { + if (typeof value !== "string") return null; + if (value.length <= maxChars) return value; + return `${value.slice(0, maxChars)}\n[truncated ${value.length - maxChars} chars]`; +} + +function readAnalysisSummaryFromWorkspace(state: SentinelCicdState, stateDir: string, timeoutSeconds: number): Record { + if (!isSafeRelativeStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, valuesRedacted: true }; + const script = [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + "node - \"$state_dir\" <<'NODE'", + "const fs=require('node:fs'); const path=require('node:path'); const crypto=require('node:crypto');", + "const stateDir=process.argv[2]; const reportPath=path.join(stateDir,'analysis','report.json'); const reportMdPath=path.join(stateDir,'analysis','report.md');", + "const read=(p)=>{try{return fs.readFileSync(p)}catch{return null}}; const jsonBuf=read(reportPath);", + "const sha=(buf)=>buf?`sha256:${crypto.createHash('sha256').update(buf).digest('hex')}`:null;", + "const rec=(v)=>v&&typeof v==='object'&&!Array.isArray(v)?v:{}; const arr=(v)=>Array.isArray(v)?v:[]; const clip=(v,n=180)=>v==null?null:String(v).slice(0,n);", + "let report=null; try{report=jsonBuf?JSON.parse(jsonBuf.toString('utf8')):null}catch{}", + "let artifactCount=0; let screenshot=null;", + "function walk(dir){let entries=[]; try{entries=fs.readdirSync(dir,{withFileTypes:true})}catch{return}; for(const e of entries){const p=path.join(dir,e.name); if(e.isDirectory()) walk(p); else { artifactCount++; if(/\\.png$/i.test(e.name)){const b=read(p); screenshot={path:p,sha256:sha(b),bytes:b?b.length:0}; } } }}", + "walk(stateDir);", + "const findings=arr(report?.findings ?? report?.archiveSummary?.redFindings).slice(0,20).map((item)=>{const v=rec(item); return {id:clip(v.id??v.kind??v.code,80),kind:clip(v.kind??v.id??v.code,80),code:clip(v.code??v.kind??v.id,80),severity:clip(v.severity??v.level,32),level:clip(v.level??v.severity,32),count:Number(v.count??v.sampleCount??1),summary:clip(v.summary??v.message,220),message:clip(v.message??v.summary,220),rootCause:clip(v.rootCause,140),rootCauseStatus:clip(v.rootCauseStatus,90),rootCauseConfidence:clip(v.rootCauseConfidence,40),nextAction:clip(v.nextAction,240),evidenceSummary:v.evidence?clip(JSON.stringify(rec(v.evidence)),220):clip(v.evidenceSummary,220),timingSourceOfTruth:clip(v.timingSourceOfTruth??v.expectedElapsedSource??v.evidenceKind,100),timingStatus:clip(v.timingStatus,60),timingAlert:v.timingAlert===true,blocking:v.blocking===true,afterRound:v.afterRound??null,canarySessionId:clip(v.canarySessionId,80),routeSessionId:clip(v.routeSessionId,80),activeSessionId:clip(v.activeSessionId,80),consecutiveUserMessageCount:v.consecutiveUserMessageCount??null,sentinelRange:clip(v.sentinelRange,80),sampleSeq:v.sampleSeq??null,traceIds:arr(v.traceIds).slice(0,8).map((x)=>clip(x,80)),pageRole:clip(v.pageRole,32),pageId:clip(v.pageId,80),observerId:clip(v.observerId,80),stateDir:clip(v.stateDir,160),commandId:clip(v.commandId,80),valuesRedacted:true};});", + "const slow=arr(report?.pagePerformanceSlowApi ?? report?.archivePagePerformanceSlowApi).slice(0,8).map((item)=>{const v=rec(item); return {path:clip(v.path??v.route,120),sampleCount:v.sampleCount??null,p95Ms:v.p95Ms??null,maxMs:v.maxMs??null,overFiveSecondCount:v.overFiveSecondCount??null};});", + "console.log(JSON.stringify({ok:!!report,reportOk:!!report&&report.ok!==false,stateDir,reportJsonPath:reportPath,reportJsonSha256:sha(jsonBuf),reportMdPath,reportMdSha256:sha(read(reportMdPath)),findingCount:Number(report?.findingCount??findings.length),artifactCount,screenshot,findings,counts:rec(report?.counts),analysisWindow:rec(report?.analysisWindow??report?.windows?.recent?.summary),pagePerformanceSlowApi:slow,valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function waitForQuickVerifyObserverStartup(state: SentinelCicdState, observerId: string, deadline: number, pollIntervalMs: number, budgetSeconds: number): Record { + const observations: Record[] = []; + const indexEntry = readLocalObserveIndex(observerId); + if (indexEntry === null) { + return { + ok: false, + failure: "observe-index-entry-missing", + observerId, + valuesRedacted: true, + }; + } + const pollSleepMs = Math.max(250, Math.min(500, Math.trunc(pollIntervalMs / 2) || 250)); + while (Date.now() < deadline) { + const waitMs = Math.max(1000, Math.min(55_000, deadline - Date.now())); + const script = quickVerifyObserverStartupWaitScript(indexEntry.stateDir, waitMs, pollSleepMs); + const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 5000 }); + const payload = parseJsonObject(result.stdout); + if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); + const terminalPayload = { + observerId, + stateDir: indexEntry.stateDir, + status: typeof payload?.status === "string" ? payload.status : null, + heartbeatStatus: typeof payload?.heartbeatStatus === "string" ? payload.heartbeatStatus : null, + startup: record(payload?.startup), + observations: observations.slice(-6), + waitResult: compactCommand(result), + valuesRedacted: true, + }; + if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-startup-wait-chunk-timeout") { + return { + ok: false, + failure: text(payload?.failure ?? "quick-verify-startup-artifact-wait-failed"), + ...terminalPayload, + }; + } + if (payload.ok === true) return { ok: true, ...terminalPayload }; + } + return { + ok: false, + failure: "quick-verify-timeout-over-budget", + observerId, + stateDir: indexEntry.stateDir, + observations: observations.slice(-6), + warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for the observe runner startup to finish before sending the first command.`], + valuesRedacted: true, + }; +} + +function quickVerifyObserverStartupWaitScript(stateDir: string, timeoutMs: number, pollSleepMs: number): string { + return [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, + `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, + "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", + "node - \"$state_dir\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", + "const fs = require('node:fs');", + "const path = require('node:path');", + "const dir = process.argv[2];", + "const timeoutMs = Number(process.argv[3]);", + "const pollMs = Number(process.argv[4]);", + "const startedAt = Date.now();", + "const startupIds = ['startup-login', 'startup-goto', 'startup-observer-goto'];", + "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", + "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", + "const clip = (value, limit = 160) => value == null ? null : String(value).replace(/\\s+/gu, ' ').trim().slice(0, limit);", + "const norm = (value) => String(value || '').trim().toLowerCase().replace(/_/gu, '-');", + "const terminal = new Set(['failed', 'force-stopped', 'stopped', 'abandoned', 'completed']);", + "function commandEvents(control, id) { return control.filter((item) => item && item.commandId === id); }", + "function lastPhase(control, id) { return commandEvents(control, id).filter((item) => typeof item.phase === 'string').slice(-1)[0]?.phase || null; }", + "function firstFailedStartup(control) { return control.filter((item) => item && startupIds.includes(item.commandId) && item.phase === 'failed').slice(-1)[0] || null; }", + "function rowFor() {", + " const heartbeat = readJson('heartbeat.json') || {};", + " const manifest = readJson('manifest.json') || {};", + " const control = readJsonl('control.jsonl');", + " const phases = Object.fromEntries(startupIds.map((id) => [id, lastPhase(control, id)]));", + " const failed = firstFailedStartup(control);", + " const heartbeatStatus = norm(heartbeat.status || manifest.status);", + " const ready = startupIds.every((id) => phases[id] === 'completed') && heartbeatStatus === 'running';", + " const terminalBeforeReady = !ready && terminal.has(heartbeatStatus);", + " const degraded = control.filter((item) => item && item.type === 'observer-startup-degraded').slice(-1)[0] || null;", + " return {", + " ok: ready,", + " status: ready ? 'startup-ready' : terminalBeforeReady ? 'startup-terminal' : 'startup-waiting',", + " heartbeatStatus,", + " startup: { phases, failedCommandId: failed?.commandId || null, failedType: failed?.type || null, failedMessage: clip(failed?.detail?.error?.message || failed?.detail?.error || failed?.error?.message), observerStartupDegraded: !!degraded, degradedReason: clip(degraded?.reason || degraded?.result?.failureKind || degraded?.result?.reason), sampleSeq: heartbeat.sampleSeq ?? null, commandSeq: heartbeat.commandSeq ?? null, currentUrl: clip(heartbeat.currentUrl, 180), observerUrl: clip(heartbeat.observerUrl, 180), valuesRedacted: true },", + " valuesRedacted: true", + " };", + "}", + "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", + "(async () => {", + " const observations = [];", + " while (Date.now() - startedAt <= timeoutMs) {", + " const row = rowFor();", + " observations.push(row);", + " if (row.ok === true) { console.log(JSON.stringify({ ok: true, ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " if (row.startup.failedCommandId) { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-command-failed', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " if (row.status === 'startup-terminal') { console.log(JSON.stringify({ ok: false, failure: 'observer-startup-terminal', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", + " }", + " const row = rowFor();", + " observations.push(row);", + " console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-chunk-timeout', ...row, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", + "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-startup-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", + "NODE", + ].join("\n"); +} + +function collectObserveView(state: SentinelCicdState, observerId: string, view: "turn-summary" | "trace-frame", turn: number | null, timeoutSeconds: number): Record { + const args = ["web-probe", "observe", "collect", observerId, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--view", view, "--command-timeout-seconds", String(Math.max(5, Math.min(timeoutSeconds, 55))), "--raw", "--compact-raw"]; + if (turn !== null) args.push("--turn", String(turn)); + const result = runChildCli(args, timeoutSeconds); + const payload = cliDataPayload(result.parsed); + const collect = record(payload.collect); + return { + ok: result.ok && result.parsed !== null && payload.ok !== false && collect.ok !== false, + view, + renderedText: typeof collect.renderedText === "string" ? collect.renderedText : typeof payload.renderedText === "string" ? payload.renderedText : String(record(result.result).stdoutTail ?? record(result.result).stdoutPreview ?? ""), + collect, + payload, + result: result.result, + valuesRedacted: true, + }; +} + +export function runChildCli(args: string[], timeoutSeconds: number, input?: string, env?: NodeJS.ProcessEnv): ChildCliResult { + const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { + input, + env: env === undefined ? undefined : { ...process.env, ...env }, + timeoutMs: Math.max(5, Math.min(timeoutSeconds, 120)) * 1000, + }); + return { + ok: result.exitCode === 0 && !result.timedOut, + parsed: parseJsonObject(result.stdout), + result: compactCommandWithTail(result), + }; +} + +function waitForQuickVerifyPromptTurn(state: SentinelCicdState, observerId: string, promptIndex: number, deadline: number, pollIntervalMs: number, budgetSeconds: number, chunkSeconds = 45): Record { + const observations: Record[] = []; + const indexEntry = readLocalObserveIndex(observerId); + if (indexEntry === null) { + return { + ok: false, + failure: "observe-index-entry-missing", + round: promptIndex, + observerId, + valuesRedacted: true, + }; + } + const pollSleepMs = Math.max(1000, Math.min(3000, Math.trunc(pollIntervalMs * 2) || 1000)); + while (Date.now() < deadline) { + const waitMs = Math.max(1000, Math.min(Math.max(1000, Math.trunc(chunkSeconds * 1000)), deadline - Date.now())); + const script = quickVerifyPromptWaitScript(indexEntry.stateDir, promptIndex, waitMs, pollSleepMs); + const result = runCommand(["trans", `${state.spec.nodeId}:${state.spec.workspace}`, "sh"], repoRoot, { input: script, timeoutMs: waitMs + 8000 }); + const payload = parseJsonObject(result.stdout); + if (Array.isArray(payload?.observations)) observations.push(...payload.observations.map(record)); + const status = typeof payload?.status === "string" ? payload.status : null; + const terminalPayload = { + round: promptIndex, + status, + traceId: payload?.traceId ?? null, + finalResponseEmpty: payload?.finalResponseEmpty === true, + composerReadyForTurn: payload?.composerReadyForTurn === true, + composerAction: typeof payload?.composerAction === "string" ? payload.composerAction : null, + observations: observations.slice(-6), + waitResult: compactCommand(result), + valuesRedacted: true, + }; + if (result.exitCode !== 0 || payload === null || payload.ok === false && payload.failure !== "quick-verify-wait-chunk-timeout") { + const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex); + if (fallback.ok === true) { + return { + ok: true, + ...terminalPayload, + status: stringAtNullable(fallback, "status") ?? status, + traceId: stringAtNullable(fallback, "traceId") ?? payload?.traceId ?? null, + finalResponseEmpty: false, + fallback, + warnings: ["quick verify artifact wait command failed, but bounded turn-summary artifacts show this round completed; continuing validation."], + }; + } + return { + ok: false, + failure: text(payload?.failure ?? "quick-verify-artifact-wait-failed"), + ...terminalPayload, + fallback, + }; + } + if (payload.ok === false && payload.failure === "quick-verify-wait-chunk-timeout") { + const fallback = quickVerifyTurnSummaryFallback(state, observerId, promptIndex); + if (fallback.ok === true) { + return { + ok: true, + ...terminalPayload, + status: stringAtNullable(fallback, "status") ?? status, + traceId: stringAtNullable(fallback, "traceId") ?? payload.traceId ?? null, + finalResponseEmpty: false, + fallback, + warnings: ["quick verify wait chunk timed out, but bounded turn-summary artifacts show this round completed; continuing validation."], + }; + } + } + if (payload.ok === true) return { ok: true, ...terminalPayload }; + if (isQuickVerifyTurnSuccessful(status)) { + if (payload?.finalResponseEmpty !== true) return { ok: true, ...terminalPayload }; + continue; + } + if (isQuickVerifyTurnTerminal(status)) { + return { + ok: false, + failure: "observe-turn-terminal-non-success", + ...terminalPayload, + }; + } + } + return { + ok: false, + failure: "quick-verify-timeout-over-budget", + round: promptIndex, + observations: observations.slice(-6), + warnings: [`quick verify exceeded the configured ${budgetSeconds}s targetValidation budget while waiting for a submitted turn to become terminal; investigate Code Agent multi-round continuity before retrying.`], + valuesRedacted: true, + }; +} + +function quickVerifyPromptWaitScript(stateDir: string, promptIndex: number, timeoutMs: number, pollSleepMs: number): string { + return [ + "set -eu", + `state_dir=${shellQuote(stateDir)}`, + `prompt_index=${shellQuote(String(promptIndex))}`, + `timeout_ms=${shellQuote(String(Math.max(1, Math.trunc(timeoutMs))))}`, + `poll_ms=${shellQuote(String(Math.max(250, Math.trunc(pollSleepMs))))}`, + "test -d \"$state_dir\" || { printf '{\"ok\":false,\"failure\":\"state-dir-missing\",\"stateDir\":\"%s\",\"valuesRedacted\":true}\\n' \"$state_dir\"; exit 0; }", + "node - \"$state_dir\" \"$prompt_index\" \"$timeout_ms\" \"$poll_ms\" <<'NODE'", + "const fs = require('node:fs');", + "const path = require('node:path');", + "const dir = process.argv[2];", + "const promptIndex = Number(process.argv[3]);", + "const timeoutMs = Number(process.argv[4]);", + "const pollMs = Number(process.argv[5]);", + "const startedAt = Date.now();", + "const short = (value, limit = 160) => String(value || '').replace(/\\s+/gu, ' ').trim().slice(0, limit);", + "const textOf = (value) => String(value?.text || value?.textPreview || value?.preview || '');", + "const arr = (value) => Array.isArray(value) ? value : [];", + "const unique = (values) => Array.from(new Set(values.filter(Boolean)));", + "const numOrNull = (value) => { const n = Number(value); return Number.isFinite(n) ? n : null; };", + "const tsMs = (value) => { const ms = Date.parse(String(value || '')); return Number.isFinite(ms) ? ms : null; };", + "const readJson = (rel) => { try { return JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); } catch { return null; } };", + "const readJsonl = (rel) => { try { return fs.readFileSync(path.join(dir, rel), 'utf8').split(/\\r?\\n/u).filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean); } catch { return []; } };", + "const readJsonlTail = (rel, maxBytes = 2000000) => {", + " try {", + " const file = path.join(dir, rel);", + " const stat = fs.statSync(file);", + " const start = Math.max(0, stat.size - maxBytes);", + " const length = stat.size - start;", + " const fd = fs.openSync(file, 'r');", + " try {", + " const buffer = Buffer.alloc(length);", + " fs.readSync(fd, buffer, 0, length, start);", + " const lines = buffer.toString('utf8').split(/\\r?\\n/u);", + " if (start > 0) lines.shift();", + " return lines.filter(Boolean).map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);", + " } finally {", + " fs.closeSync(fd);", + " }", + " } catch {", + " return [];", + " }", + "};", + "const readDone = (id) => id ? readJson(path.join('commands', 'done', `${id}.json`)) : null;", + "const readFailed = (id) => id ? readJson(path.join('commands', 'failed', `${id}.json`)) : null;", + "function sessionIdFromUrl(value) { const match = String(value || '').match(/\\/workbench\\/sessions\\/(ses_[A-Za-z0-9_-]+)/u); return match ? match[1] : null; }", + "function commandSessionId(item) { const done = readDone(item?.commandId); return item?.sessionId || item?.detail?.sessionId || item?.input?.sessionId || item?.result?.sessionId || done?.result?.sessionId || done?.result?.observer?.sessionId || sessionIdFromUrl(item?.afterUrl) || sessionIdFromUrl(item?.detail?.afterUrl) || sessionIdFromUrl(done?.result?.afterUrl) || null; }", + "function commandTextHash(item) { const done = readDone(item?.commandId); return item?.detail?.textHash || item?.input?.textHash || item?.result?.textHash || done?.result?.textHash || null; }", + "function firstTraceId(value) { const match = String(value || '').match(/\\btrc_[A-Za-z0-9_-]+\\b/u); return match ? match[0] : null; }", + "function commandTraceId(item) { const done = readDone(item?.commandId); return item?.traceId || item?.detail?.chatSubmit?.traceId || item?.detail?.traceId || item?.input?.traceId || item?.result?.chatSubmit?.traceId || done?.result?.chatSubmit?.traceId || done?.result?.traceId || null; }", + "function itemTraceId(item) { return item?.traceId || firstTraceId(textOf(item)) || null; }", + "function completedNewSessionIdsBefore(control, ts) { const limit = tsMs(ts); return control.filter((item) => item.type === 'newSession' && item.phase === 'completed').filter((item) => limit === null || tsMs(item.ts) === null || tsMs(item.ts) <= limit).map(commandSessionId).filter(Boolean); }", + "function authoritativeSessionIdForPrompts(control, prompts) { const ids = completedNewSessionIdsBefore(control, prompts[0]?.firstTs || null); return ids.slice(-1)[0] || unique(prompts.map((item) => item.sessionId))[0] || null; }", + "function promptCommands(control) {", + " const map = new Map();", + " for (const item of control) {", + " if (item.type !== 'sendPrompt' || !['started', 'completed', 'failed'].includes(item.phase)) continue;", + " const id = item.commandId || item.seq || String(map.size + 1);", + " const existing = map.get(id) || {};", + " map.set(id, { ...existing, ...item, input: { ...(existing.input || {}), ...(item.input || {}) }, sessionId: existing.sessionId || commandSessionId(item), traceId: existing.traceId || commandTraceId(item), textHash: existing.textHash || commandTextHash(item), firstTs: existing.firstTs || item.ts, lastTs: item.ts });", + " }", + " const prompts = Array.from(map.values()).filter((item) => tsMs(item.firstTs) !== null).sort((a, b) => tsMs(a.firstTs) - tsMs(b.firstTs));", + " const sessionId = authoritativeSessionIdForPrompts(control, prompts);", + " if (!sessionId) return prompts;", + " const scoped = prompts.filter((item) => item.sessionId === sessionId);", + " return scoped.length > 0 ? scoped : prompts;", + "}", + "function segmentFor(samples, prompts, index) { const start = tsMs(prompts[index]?.firstTs); const end = index + 1 < prompts.length ? tsMs(prompts[index + 1].firstTs) : Infinity; return samples.filter((sample) => { const ms = tsMs(sample.ts); return ms !== null && ms >= start && ms < end; }); }", + "function entryGroups(sample) { return [...arr(sample.turns).map((item) => ({ group: 'turn', item })), ...arr(sample.traceRows).map((item) => ({ group: 'traceRow', item })), ...arr(sample.messages).map((item) => ({ group: 'message', item }))]; }", + "function traceIdsFromSamples(items) { const ids = []; for (const sample of items) for (const entry of entryGroups(sample)) { const id = itemTraceId(entry.item); if (id) ids.push(id); } return unique(ids); }", + "function chooseTraceId(segment, prompt) { const promptTraceId = commandTraceId(prompt) || prompt?.traceId || null; const ids = traceIdsFromSamples(segment); if (promptTraceId && (ids.length === 0 || ids.includes(promptTraceId))) return promptTraceId; return ids.slice(-1)[0] || promptTraceId || null; }", + "function traceIdForPromptUserMessage(items, prompt) {", + " const hash = prompt?.textHash || commandTextHash(prompt);", + " if (!hash) return null;", + " for (const sample of items) {", + " for (const message of arr(sample.messages)) {", + " const role = String(message?.role || message?.dataRole || message?.messageRole || '').toLowerCase();", + " if (role && !/user/u.test(role)) continue;", + " if (message?.textHash === hash) {", + " const id = itemTraceId(message);", + " if (id) return id;", + " }", + " }", + " }", + " return null;", + "}", + "function traceEntries(items, traceId) { const entries = []; for (const sample of items) for (const entry of entryGroups(sample)) { const text = textOf(entry.item); const id = itemTraceId(entry.item); if (!traceId || id === traceId || text.includes(traceId)) entries.push({ ...entry, sample, text }); } return entries; }", + "function normalizeLifecycleStatus(value) { const raw = String(value || '').trim().toLowerCase(); if (/^(canceled|cancelled)$/u.test(raw)) return 'canceled'; if (/^(failed|failure|error)$/u.test(raw)) return 'failed'; if (/^(completed|complete|succeeded|success|terminal)$/u.test(raw)) return 'completed'; if (/^(running|admitting|queued|pending|in_progress|in-progress)$/u.test(raw)) return 'running'; return null; }", + "function statusFor(items, traceId) { const entries = traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) }))); const lastTurn = entries.filter((entry) => entry.group === 'turn').slice(-1)[0]?.item || null; const turnStatus = normalizeLifecycleStatus(lastTurn?.status); if (turnStatus) return turnStatus; const lastMessage = entries.filter((entry) => entry.group === 'message').slice(-1)[0]?.item || null; return normalizeLifecycleStatus(lastMessage?.status) || 'unknown'; }", + "function cleanFinalResponseText(value) { const raw = String(value || '').trim(); if (!raw) return ''; if (/^(completed|failed|canceled|cancelled|轮次完成|轮次失败|轮次取消|已记录)$/iu.test(raw.replace(/\\s+/gu, ' '))) return ''; if (/^(admitted|run|ok|error)\\s+/iu.test(raw)) return ''; return raw; }", + "function finalResponseTextFromEntry(entry) {", + " const explicit = cleanFinalResponseText(entry.item?.finalResponse?.text || entry.item?.finalResponse?.preview || '');", + " if (explicit && !/^Code Agent\\s*耗时/iu.test(explicit)) return explicit;", + " if (entry.group !== 'message') return '';", + " const role = String(entry.item?.role || entry.item?.dataRole || entry.item?.messageRole || '').toLowerCase();", + " if (role && !/assistant|agent|system/u.test(role)) return '';", + " const text = cleanFinalResponseText(entry.text);", + " return text && !/^Code Agent\\s*耗时/iu.test(text) ? text : '';", + "}", + "function finalResponseEmpty(items, traceId) { if (!/^(completed|failed|canceled)$/u.test(statusFor(items, traceId))) return true; const entries = (traceId ? traceEntries(items, traceId) : items.flatMap((sample) => entryGroups(sample).map((entry) => ({ ...entry, sample, text: textOf(entry.item) })))).slice().reverse(); for (const entry of entries) { if (finalResponseTextFromEntry(entry)) return false; } return true; }", + "function rowFor() {", + " const control = readJsonl('control.jsonl');", + " const samples = readJsonlTail('samples.jsonl');", + " const prompts = promptCommands(control);", + " const prompt = prompts[promptIndex - 1] || null;", + " if (!prompt) return { ok: true, round: promptIndex, status: 'command-pending', traceId: null, finalResponseEmpty: true, lastSeq: null, lastTs: null, promptMissing: true, valuesRedacted: true };", + " const done = readDone(prompt.commandId);", + " const failed = readFailed(prompt.commandId);", + " if (failed) return { ok: false, failure: 'observe-command-sendPrompt-failed', round: promptIndex, status: 'command-failed', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandFailure: short(failed.error?.message || failed.failure || failed.status || 'command failed'), valuesRedacted: true };", + " const promptTraceId = commandTraceId(prompt);", + " if (!done) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: promptTraceId || null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: !promptTraceId, valuesRedacted: true };", + " const segment = segmentFor(samples, prompts, promptIndex - 1);", + " const controlSegment = segment.filter((sample) => sample.pageRole === 'control');", + " const traceId = promptTraceId || traceIdForPromptUserMessage(segment, prompt) || chooseTraceId(controlSegment, prompt) || chooseTraceId(segment, prompt);", + " if (!traceId) return { ok: true, round: promptIndex, status: 'command-pending', commandId: prompt.commandId || null, traceId: null, finalResponseEmpty: true, commandPhase: prompt.phase || null, traceMissing: true, segmentSampleCount: segment.length, valuesRedacted: true };", + " const controlTraceSegment = traceId ? controlSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)) : [];", + " const statusSegment = controlTraceSegment.length > 0 ? controlSegment : segment;", + " const status = statusFor(statusSegment, traceId);", + " const sampleForTrace = traceId ? statusSegment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || segment.filter((sample) => traceIdsFromSamples([sample]).includes(traceId)).slice(-1)[0] || null : null;", + " const lastSample = sampleForTrace || segment.slice(-1)[0] || null;", + " const composerSample = segment.filter((sample) => sample.pageRole === 'control').slice(-1)[0] || lastSample;", + " const composer = composerSample?.composer || {};", + " const composerReadyForTurn = composer.inputPresent === true && composer.inputDisabled !== true && composer.submitPresent === true && composer.warningPresent !== true && composer.submitAction === 'turn';", + " return { ok: true, round: promptIndex, status, traceId, finalResponseEmpty: finalResponseEmpty(statusSegment, traceId), lastSeq: numOrNull(lastSample?.seq), lastTs: lastSample?.ts || null, composerReadyForTurn, composerAction: composer.submitAction || null, sampleScope: controlSegment.length > 0 ? 'control' : 'all', segmentSampleCount: segment.length, statusSampleCount: statusSegment.length, source: 'observe-artifact-wait-script', valuesRedacted: true };", + "}", + "function norm(value) { return String(value || '').trim().toLowerCase().replace(/_/gu, '-'); }", + "function successful(value) { return ['completed', 'succeeded', 'success'].includes(norm(value)); }", + "function terminal(value) { return ['completed', 'succeeded', 'success', 'failed', 'error', 'blocked', 'timeout', 'canceled', 'cancelled', 'terminal'].includes(norm(value)); }", + "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", + "(async () => {", + " const observations = [];", + " while (Date.now() - startedAt <= timeoutMs) {", + " const row = rowFor();", + " observations.push(row);", + " if (row.ok === false) { console.log(JSON.stringify({ ...row, ok: false, failure: row.failure || 'quick-verify-artifact-row-failed', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " if (successful(row.status) && row.finalResponseEmpty !== true) { console.log(JSON.stringify({ ...row, ok: true, observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " if (!successful(row.status) && terminal(row.status)) { console.log(JSON.stringify({ ...row, ok: false, failure: 'observe-turn-terminal-non-success', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true })); return; }", + " await sleep(Math.min(pollMs, Math.max(0, timeoutMs - (Date.now() - startedAt))));", + " }", + " const row = rowFor();", + " observations.push(row);", + " console.log(JSON.stringify({ ...row, ok: false, failure: 'quick-verify-wait-chunk-timeout', observations: observations.slice(-6), elapsedMs: Date.now() - startedAt, valuesRedacted: true }));", + "})().catch((error) => { console.log(JSON.stringify({ ok: false, failure: 'quick-verify-artifact-wait-script-error', error: error instanceof Error ? error.message : String(error), valuesRedacted: true })); });", + "NODE", + ].join("\n"); +} + +function isQuickVerifyTurnSuccessful(value: string | null): boolean { + const status = normalizeQuickVerifyStatus(value); + return status === "completed" || status === "succeeded" || status === "success"; +} + +function isQuickVerifyTurnTerminal(value: string | null): boolean { + const status = normalizeQuickVerifyStatus(value); + return status === "completed" + || status === "succeeded" + || status === "success" + || status === "failed" + || status === "error" + || status === "blocked" + || status === "timeout" + || status === "canceled" + || status === "cancelled" + || status === "terminal"; +} + +function normalizeQuickVerifyStatus(value: string | null): string { + return String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); +} + +function cliDataPayload(parsed: Record | null): Record { + const root = record(parsed); + const payload = isRecord(root.data) ? root.data : root; + return cliDumpPayload(payload) ?? payload; +} + +function cliDumpPayload(payload: Record): Record | null { + if (payload.outputTruncated !== true) return null; + const dumpPath = stringAtNullable(record(payload.dump), "path"); + if (dumpPath === null || !existsSync(dumpPath)) return null; + const dumped = parseJsonObject(readFileSync(dumpPath, "utf8")); + if (dumped === null) return null; + const dumpedRoot = record(dumped); + return isRecord(dumpedRoot.data) ? dumpedRoot.data : dumpedRoot; +} + +function findScenario(state: SentinelCicdState, scenarioId: string): Record | null { + const scenarios = readWebProbeSentinelConfigRefTarget(state.spec, state.configRefs.scenarios); + const items = Array.isArray(scenarios) ? scenarios : isRecord(scenarios) ? [scenarios] : []; + return items.map(record).find((item) => item.id === scenarioId) ?? null; +} + +function readPromptSetForScenario(state: SentinelCicdState, scenario: Record): { ok: true; prompts: string[]; summary: Record } | { ok: false; error: string; summary: Record } { + const promptSetRef = stringAt(scenario, "promptSetRef"); + const promptSet = recordTarget(readWebProbeSentinelConfigRefTarget(state.spec, promptSetRef), promptSetRef); + const sourceRef = stringAt(promptSet, "promptSourceRef"); + const key = stringAt(promptSet, "promptSourceKey"); + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true }; + const runtimeRaw = process.env[key]; + const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; + const raw = values[key] ?? runtimeRaw; + const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null; + if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary }; + if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary }; + const parsed = parsePromptJson(raw); + if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary }; + return { + ok: true, + prompts: parsed, + summary: { + ...summary, + sourceMode, + promptCount: parsed.length, + promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))), + promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`), + promptTextBytes: parsed.map((item) => Buffer.byteLength(item)), + valuesRedacted: true, + }, + }; +} + +function parsePromptJson(raw: string): string[] { + try { + const parsed = JSON.parse(raw) as unknown; + if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string" && item.length > 0); + const recordValue = record(parsed); + if (Array.isArray(recordValue.prompts)) return recordValue.prompts.filter((item): item is string => typeof item === "string" && item.length > 0); + if (typeof recordValue.prompt === "string" && recordValue.prompt.length > 0) return [recordValue.prompt]; + } catch { + if (raw.trim().length > 0) return [raw]; + } + return []; +} + +function readLocalObserveIndex(observerId: string): { stateDir: string } | null { + const path = rootPath(".state/web-observe/index.json"); + if (!existsSync(path)) return null; + const parsed = parseJsonObject(readFileSync(path, "utf8")); + const entry = record(parsed?.[observerId]); + const stateDir = typeof entry.stateDir === "string" ? entry.stateDir : null; + return stateDir === null ? null : { stateDir }; +} + +function observerIdFromText(textValue: string): string | null { + return /\bwebobs-[a-z0-9-]+\b/iu.exec(textValue)?.[0] ?? null; +} + +export function remainingSeconds(deadline: number, cap: number): number { + return Math.max(5, Math.min(cap, Math.ceil((deadline - Date.now()) / 1000))); +} + +export function metricNames(textValue: unknown): string[] { + if (typeof textValue !== "string") return []; + return textValue.split(/\r?\n/u).map((line) => /^([A-Za-z_:][A-Za-z0-9_:]*)/u.exec(line)?.[1]).filter((item): item is string => typeof item === "string"); +} + +export function validationBlocker(health: Record, metrics: Record, report: Record, publicExposure: Record, publicDashboard: Record, quickVerify: Record | null): Record { + const blockers = []; + if (!health.ok || record(health.bodyJson).ok !== true) blockers.push("health"); + if (!metrics.ok || !metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health")) blockers.push("metrics"); + if (!report.ok) blockers.push("recent-report"); + if (publicExposure.ok !== true) blockers.push("public-exposure"); + if (publicDashboard.ok !== true) blockers.push("public-dashboard"); + if (quickVerify !== null && quickVerify.ok !== true) blockers.push("quick-verify"); + return { code: "sentinel-validation-failed", blockers, valuesRedacted: true }; +} + +export function serviceUnavailableBlocker(state: SentinelCicdState): Record { + return { + code: "sentinel-service-unavailable", + policy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"), + reason: "sentinel service must be reachable through k3s internal Service DNS before quick verify can run; no public/fallback path is used.", + retry: `bun scripts/cli.ts web-probe sentinel validate --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, + valuesRedacted: true, + }; +} + +export function sentinelP5Next(state: SentinelCicdState): Record { + const node = state.spec.nodeId; + const lane = state.spec.lane; + const suffix = sentinelCliSuffix(state); + return { + validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`, + quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`, + maintenanceStart: `bun scripts/cli.ts web-probe sentinel maintenance start --node ${node} --lane ${lane}${suffix} --confirm --wait`, + maintenanceStop: `bun scripts/cli.ts web-probe sentinel maintenance stop --node ${node} --lane ${lane}${suffix} --confirm --wait`, + report: `bun scripts/cli.ts web-probe sentinel report --node ${node} --lane ${lane}${suffix} --view summary`, + }; +} + +function isSafeRelativeStateDir(value: string): boolean { + return value.startsWith(".state/web-observe/") && !value.includes("\0") && !value.includes(".."); +} + +function mergeFindingRecords(primary: readonly Record[], extra: readonly Record[]): Record[] { + const merged: Record[] = []; + const seen = new Set(); + for (const item of [...primary, ...extra]) { + const id = stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? stringAtNullable(item, "finding_id") ?? "finding"; + const severity = stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "unknown"; + const key = `${id}\0${severity}`; + if (seen.has(key)) continue; + seen.add(key); + merged.push(item); + } + return merged; +} + +function isQuickVerifyBlockingFinding(item: Record): boolean { + const severity = (stringAtNullable(item, "severity") ?? stringAtNullable(item, "level") ?? "").toLowerCase(); + if (!["critical", "red", "fatal", "error", "failed", "blocked"].includes(severity)) return false; + const id = (stringAtNullable(item, "id") ?? stringAtNullable(item, "kind") ?? stringAtNullable(item, "code") ?? "").toLowerCase(); + if (id === "observer-command-failed") return observerCommandFailureBlocks(item); + return [ + "quick-verify-no-business-turn", + "quick-verify-command-sequence-failed", + "quick-verify-observer-start-failed", + "quick-verify-account-secret-missing", + "prompt-chat-submit-failed", + "route-active-session-mismatch", + "final-response-flicker", + "round-completion-final-response-missing", + "turn-trace-id-missing", + "no-samples", + "jsonl-read-issues", + ].includes(id); +} + +function observerCommandFailureBlocks(item: Record): boolean { + const commands = Array.isArray(item.commands) ? item.commands.map(record) : []; + if (commands.length === 0) return true; + return commands.some((command) => { + const type = (stringAtNullable(command, "type") ?? "").toLowerCase(); + if (["stop", "cancel", "mark", "screenshot"].includes(type)) return false; + return true; + }); +} + +function quickVerifyControlFindings(failure: string | null, promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): Record[] { + if (quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame)) return []; + const rendered = [ + typeof turnSummary?.renderedText === "string" ? turnSummary.renderedText : "", + typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : "", + ].join("\n"); + const noPromptScenario = promptIndex <= 0; + if (noPromptScenario && failure === null) return []; + if (noPromptScenario && failure !== null) { + const observerStartFailure = failure === "observe-start-failed"; + return [{ + id: observerStartFailure ? "quick-verify-observer-start-failed" : "quick-verify-command-sequence-failed", + severity: "red", + count: 1, + summary: observerStartFailure + ? "quick verify observer failed to start before the no-prompt scenario could run." + : "quick verify no-prompt command sequence failed before the account/session workflow completed.", + failure, + promptIndex, + valuesRedacted: true, + }]; + } + const noTrace = /无\s*sendPrompt|no\s+sendPrompt|无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(rendered); + const emptyFinal = /Final Response[\s\S]*\(空内容\)/iu.test(rendered); + if (!noTrace && !emptyFinal && failure !== "observe-start-failed") return []; + return [{ + id: "quick-verify-no-business-turn", + severity: "red", + count: 1, + summary: "quick verify did not reach a durable business turn/session/trace rows/final response; public dashboard health cannot be treated as HWLAB recovery.", + failure: failure ?? null, + promptIndex, + valuesRedacted: true, + }]; +} + +function quickVerifyCompletedTurnSummaryRow(promptIndex: number, turnSummary: Record | null): Record | null { + const rows = Array.isArray(record(turnSummary?.collect).rows) ? record(turnSummary?.collect).rows.map(record) : []; + const scopedRows = promptIndex > 0 ? rows.filter((row) => numberAtNullable(row, "round") === promptIndex) : rows; + return scopedRows.find((row) => { + const finalResponse = record(row.finalResponse); + return isQuickVerifyTurnSuccessful(stringAtNullable(row, "status")) + && stringAtNullable(row, "traceId") !== null + && finalResponse.empty !== true; + }) ?? null; +} + +function quickVerifyTurnSummaryFallback(state: SentinelCicdState, observerId: string, promptIndex: number): Record { + const turnSummary = collectObserveView(state, observerId, "turn-summary", null, 25); + const row = quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary); + const rows = Array.isArray(record(turnSummary.collect).rows) ? record(turnSummary.collect).rows.map(record) : []; + if (row === null) { + return { + ok: false, + source: "turn-summary-fallback", + collectOk: turnSummary.ok === true, + rowCount: rows.length, + promptIndex, + result: turnSummary.result ?? null, + valuesRedacted: true, + }; + } + const finalResponse = record(row.finalResponse); + return { + ok: true, + source: "turn-summary-fallback", + collectOk: turnSummary.ok === true, + rowCount: rows.length, + promptIndex, + status: stringAtNullable(row, "status"), + traceId: stringAtNullable(row, "traceId"), + finalResponseEmpty: finalResponse.empty === true, + finalResponseBytes: numberAtNullable(finalResponse, "textBytes"), + result: turnSummary.result ?? null, + valuesRedacted: true, + }; +} + +function quickVerifyHasDurableBusinessTurn(promptIndex: number, turnSummary: Record | null, traceFrame: Record | null): boolean { + if (quickVerifyCompletedTurnSummaryRow(promptIndex, turnSummary) !== null) return true; + const renderedTrace = typeof traceFrame?.renderedText === "string" ? traceFrame.renderedText : ""; + if (!renderedTrace) return false; + if (/Final Response\s*\n\s*\(空内容\)/iu.test(renderedTrace)) return false; + return /Code Agent[^\n]*completed|轮次完成（总耗时/iu.test(renderedTrace) + && /Final Response\s*\n\s*\S/iu.test(renderedTrace) + && !/无\s*trace\s*rows|no\s+trace\s+rows|traceId=-|routeSession=-|activeSession=-/iu.test(renderedTrace); +} + +function quickVerifyBusinessStatus( + failure: string | null, + promptIndex: number, + turnSummary: Record | null, + traceFrame: Record | null, + elapsedMs: unknown, + budgetSeconds: number, +): Record { + const durableBusinessTurn = quickVerifyHasDurableBusinessTurn(promptIndex, turnSummary, traceFrame); + const elapsed = typeof elapsedMs === "number" && Number.isFinite(elapsedMs) ? elapsedMs : null; + const budgetMs = Math.max(0, budgetSeconds) * 1000; + const budgetExceeded = elapsed !== null && budgetMs > 0 && elapsed > budgetMs; + const observerTimeout = budgetExceeded || (failure !== null && isRecoverableQuickVerifyWaitFailure(failure)); + const status = durableBusinessTurn + ? "business-turn-completed" + : observerTimeout + ? "observer-timeout" + : "scenario-incomplete"; + return { + status, + durableBusinessTurn, + observerTimeout, + scenarioComplete: durableBusinessTurn, + failure: failure ?? null, + promptIndex, + elapsedMs: elapsed, + budgetSeconds, + sourceOfTruth: durableBusinessTurn ? "turn-summary-or-trace-frame" : observerTimeout ? "runner-wait-budget" : "control-findings", + valuesRedacted: true, + }; +} + +function isRecoverableQuickVerifyWaitFailure(failure: string): boolean { + return failure === "quick-verify-wait-chunk-timeout" + || failure === "quick-verify-timeout-over-budget" + || failure === "observe-turn-terminal-wait-failed"; +} + +function compactCommandWithTail(result: CommandResult): CompactCommandResult & { stdoutTail: string; stderrTail: string } { + return { + ...compactCommand(result), + stdoutPreview: result.stdout.trim().slice(0, 1200), + stderrPreview: result.stderr.trim().slice(0, 1200), + stdoutTail: result.stdout.trim().slice(-4000), + stderrTail: result.stderr.trim().slice(-4000), + }; +} + +function renderQuickVerifySummary(input: Record): string { + const artifact = record(input.artifactSummary); + const findings = Array.isArray(artifact.findings) ? artifact.findings.map(record).slice(0, 8) : []; + return [ + "Web Probe Sentinel Quick Verify", + "=======================================================", + `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, + `report=${artifact.reportJsonSha256 ?? "-"} artifacts=${artifact.artifactCount ?? "-"} findings=${artifact.findingCount ?? findings.length}`, + `publicOrigin=${input.publicOrigin ?? "-"}`, + "", + "Findings", + findings.length === 0 ? "-" : findings.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"), + ].join("\n"); +} + +function renderAuthSessionSwitchQuickVerifySummary(input: Record): string { + const artifact = record(input.artifactSummary); + const accountEnv = record(input.accountEnv); + const findingRows = Array.isArray(input.findings) + ? input.findings.map(record).slice(0, 8) + : Array.isArray(artifact.findings) + ? artifact.findings.map(record).slice(0, 8) + : []; + const steps = Array.isArray(input.steps) ? input.steps.map(record) : []; + const commandSteps = steps + .filter((step) => { + const phase = stringAtNullable(step, "phase") ?? ""; + return phase.startsWith("observe-command-") || phase.startsWith("observe-session-invariance-") || phase === "quick-verify-account-env"; + }) + .map((step) => { + const phase = stringAtNullable(step, "phase") ?? "-"; + const ok = step.ok === true ? "ok" : step.ok === false ? "failed" : "-"; + const result = record(step.result); + const status = stringAtNullable(result, "status") ?? stringAtNullable(result, "failure") ?? "-"; + return `${phase} ${ok} ${status}`; + }); + return [ + "Auth Session Switch Quick Verify", + "=======================================================", + `run=${input.runId ?? "-"} scenario=${input.scenarioId ?? "-"} observer=${input.observerId ?? "-"}`, + `status=${artifact.ok === true ? "ok" : "blocked"} report=${artifact.reportJsonSha256 ?? "-"} publicOrigin=${input.publicOrigin ?? "-"}`, + `accountEnv=${accountEnv.envCount ?? "-"} valuesRedacted=true`, + "", + "Command Sequence", + commandSteps.length === 0 ? "-" : commandSteps.join("\n"), + "", + "Findings", + findingRows.length === 0 ? "-" : findingRows.map((item) => `${item.severity ?? item.level ?? "-"} ${item.kind ?? item.id ?? item.code ?? "-"} count=${item.count ?? "-"}${formatQuickVerifyTimingSuffix(item)} ${item.summary ?? item.message ?? ""}`).join("\n"), + ].join("\n"); +} + +function formatQuickVerifyTimingSuffix(item: Record): string { + const status = stringAtNullable(item, "timingStatus"); + const source = stringAtNullable(item, "timingSourceOfTruth") ?? stringAtNullable(item, "expectedElapsedSource") ?? stringAtNullable(item, "evidenceKind"); + if (status === null && source === null) return ""; + return ` timing=${[status === null ? null : `status=${status}`, source === null ? null : `source=${source}`].filter((part) => part !== null).join(" ")}`; +} diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts new file mode 100644 index 00000000..1b25d7ce --- /dev/null +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -0,0 +1,1491 @@ +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. +// Responsibility: P5 web-probe sentinel service validation, maintenance, report and dashboard commands. +import type { CommandResult } from "./command"; +import { runCommand } from "./command"; +import { repoRoot } from "./config"; +import { startJob } from "./jobs"; +import type { RenderedCliResult } from "./output"; +import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact"; +import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd"; +import { + clipTail, + compactCommand, + compactSentinelServiceBodyJson, + mergeWarnings, + numberAt, + parseJsonObject, + pickFields, + record, + rendered, + renderAsyncJobResult, + safeJobSegment, + sentinelCliSuffix, + shellQuote, + short, + stringAt, + stringAtNullable, + table, + targetValidationElapsedWarnings, + text, + withWarnings, +} from "./hwlab-node-web-sentinel-cicd"; +import { metricNames, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe"; + +export function runSentinelMaintenance(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = `web-probe sentinel maintenance ${options.action}`; + const serviceHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); + if (options.action === "status") { + const maintenance = callSentinelService(state, "GET", "/api/maintenance", null, options.timeoutSeconds); + const result = { + ok: serviceHealth.ok && maintenance.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + serviceHealth, + maintenance, + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderMaintenanceResult(result)); + } + if (!options.confirm) { + const result = { + ok: serviceHealth.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "dry-run", + serviceHealth, + mutation: false, + planned: { + action: options.action, + releaseId: options.releaseId, + reason: options.reason, + quickVerify: options.action === "stop" && options.quickVerify, + }, + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderMaintenanceResult(result)); + } + if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); + if (!serviceHealth.ok) { + const result = { + ok: false, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "confirm-wait", + mutation: false, + serviceHealth, + blocker: serviceUnavailableBlocker(state), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(false, command, renderMaintenanceResult(result)); + } + const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true }; + const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds); + const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok + ? runSentinelQuickVerify(state, "maintenance-stop", options.timeoutSeconds) + : null; + const result = { + ok: mutation.ok && (quickVerify === null || quickVerify.ok === true), + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "confirm-wait", + mutation: true, + serviceHealth, + maintenance: mutation, + quickVerify, + blocker: mutation.ok ? null : serviceUnavailableBlocker(state), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderMaintenanceResult(result)); +} + +export function runSentinelValidate(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = "web-probe sentinel validate"; + const startedAt = Date.now(); + const initialHealth = callSentinelService(state, "GET", "/api/health", null, options.timeoutSeconds); + let quickVerify: Record | null = null; + if (options.quickVerify) { + if (!options.confirm) { + const result = { + ok: initialHealth.ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "dry-run", + serviceHealth: initialHealth, + planned: { quickVerify: true, waitRequired: true }, + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(result.ok, command, renderValidateResult(result)); + } + if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true); + if (!initialHealth.ok) { + const result = { + ok: false, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "confirm-wait", + serviceHealth: initialHealth, + blocker: serviceUnavailableBlocker(state), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(false, command, renderValidateResult(result)); + } + quickVerify = runSentinelQuickVerify(state, "manual-validate", options.timeoutSeconds); + } + const serviceProbeTimeoutSeconds = Math.min(options.timeoutSeconds, options.quickVerify ? 30 : 20); + const health = callSentinelService(state, "GET", "/api/health", null, serviceProbeTimeoutSeconds); + const metrics = callSentinelService(state, "GET", "/metrics", null, serviceProbeTimeoutSeconds); + const report = callSentinelService(state, "GET", "/api/report?view=summary", null, serviceProbeTimeoutSeconds); + const metricsOk = metrics.ok && metricNames(record(metrics).bodyTextPreview).includes("web_probe_sentinel_health"); + const publicHealth = health.ok ? null : probePublicSentinelService(state, "/api/health", serviceProbeTimeoutSeconds); + const publicMetrics = metricsOk ? null : probePublicSentinelService(state, "/metrics", serviceProbeTimeoutSeconds); + const publicReport = report.ok ? null : probePublicSentinelService(state, "/api/report?view=summary", serviceProbeTimeoutSeconds); + const effectiveHealth = health.ok ? health : record(publicHealth).ok === true ? record(publicHealth) : health; + const effectiveMetrics = metricsOk ? metrics : record(publicMetrics).ok === true ? record(publicMetrics) : metrics; + const effectiveReport = report.ok ? report : record(publicReport).ok === true ? record(publicReport) : report; + const publicExposure = probeSentinelPublicExposure(state, options.timeoutSeconds); + const publicDashboard = probeSentinelPublicDashboard(state, options.timeoutSeconds); + if (quickVerify !== null) { + quickVerify = withWarnings(quickVerify, targetValidationElapsedWarnings(Date.now() - startedAt, "sentinel validate quick verify confirm-wait", Math.min(options.timeoutSeconds, numberAt(state.cicd, "targetValidation.maxSeconds")))); + } + const publicFallbackWarnings = [ + ...(!health.ok && record(publicHealth).ok === true ? ["internal sentinel health probe failed through D601:k3s, but public /api/health passed; treating provider transport as a non-blocking validation warning."] : []), + ...(!metricsOk && record(publicMetrics).ok === true ? ["internal sentinel metrics probe failed through D601:k3s, but public /metrics exposed web_probe_sentinel_health; treating provider transport as a non-blocking validation warning."] : []), + ...(!report.ok && record(publicReport).ok === true ? ["internal sentinel report probe failed through D601:k3s, but public /api/report returned the indexed report; treating provider transport as a non-blocking validation warning."] : []), + ]; + const effectiveMetricsOk = effectiveMetrics.ok && metricNames(record(effectiveMetrics).bodyTextPreview).includes("web_probe_sentinel_health"); + const ok = effectiveHealth.ok + && record(effectiveHealth.bodyJson).ok === true + && effectiveMetricsOk + && effectiveReport.ok + && publicExposure.ok === true + && publicDashboard.ok === true + && (quickVerify === null || quickVerify.ok === true); + const result = { + ok, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: options.quickVerify ? "confirm-wait" : "status", + serviceHealth: effectiveHealth, + metrics: effectiveMetrics, + report: effectiveReport, + internalServiceHealth: health, + internalMetrics: metrics, + internalReport: report, + publicServiceHealth: publicHealth, + publicMetrics, + publicReport, + publicExposure, + publicDashboard, + quickVerify, + warnings: mergeWarnings(publicFallbackWarnings, quickVerify === null ? [] : Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []), + blocker: ok ? null : validationBlocker(effectiveHealth, effectiveMetrics, effectiveReport, publicExposure, publicDashboard, quickVerify), + next: sentinelP5Next(state), + valuesRedacted: true, + }; + return rendered(ok, command, renderValidateResult(result)); +} + +export function runSentinelReport(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = `web-probe sentinel report ${options.latest ? "--latest " : ""}--view ${options.view}`; + const query = new URLSearchParams({ view: options.view }); + if (options.runId !== null) query.set("run", options.runId); + if (options.traceId !== null) query.set("traceId", options.traceId); + if (options.sampleSeq !== null) query.set("sampleSeq", String(options.sampleSeq)); + const report = callSentinelService(state, "GET", `/api/report?${query.toString()}`, null, options.timeoutSeconds); + const body = record(report.bodyJson); + const renderedText = typeof body.renderedText === "string" ? body.renderedText : renderReportResult({ command, node: state.spec.nodeId, lane: state.spec.lane, report, valuesRedacted: true }); + const rawPayload = Object.keys(body).length > 0 ? body : report; + return rendered(report.ok && body.ok !== false, command, options.raw ? JSON.stringify(rawPayload, null, 2) : renderedText); +} + +export function runSentinelDashboard(state: SentinelCicdState, options: Extract): RenderedCliResult { + const command = `web-probe sentinel dashboard ${options.action}`; + const result = probeSentinelDashboardBrowser(state, options); + return rendered(result.ok === true, command, options.raw ? JSON.stringify(result, null, 2) : renderDashboardResult(result)); +} + +function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract): Record { + const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); + const [widthRaw, heightRaw] = options.viewport.split("x"); + const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : ""; + const script = [ + "set -eu", + `export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(`${publicBaseUrl}/`)}`, + `export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`, + `export UNIDESK_SENTINEL_DASHBOARD_CAPTURE=${shellQuote(options.action === "screenshot" ? "1" : "0")}`, + `export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`, + `export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`, + `export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, + `export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`, + `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`, + `export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`, + `export UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE=${shellQuote(`${state.spec.workspace}/node_modules/playwright/index.mjs`)}`, + "export PLAYWRIGHT_BROWSERS_PATH=0", + "if command -v chromium >/dev/null 2>&1; then", + " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium)", + "elif command -v chromium-browser >/dev/null 2>&1; then", + " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v chromium-browser)", + "elif command -v google-chrome >/dev/null 2>&1; then", + " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=$(command -v google-chrome)", + "else", + " export UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH=", + "fi", + "cat > \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\" <<'WEB_PROBE_SENTINEL_DASHBOARD_JS'", + sentinelDashboardBrowserModule(), + "WEB_PROBE_SENTINEL_DASHBOARD_JS", + "bun \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"", + ].join("\n"); + const route = `${state.spec.nodeId}:${state.spec.workspace}`; + const job = runWebProbeRemoteArtifactJob({ + route, + localDir: options.localDir, + waitTimeoutMs: options.waitTimeoutMs, + commandTimeoutMs: options.commandTimeoutSeconds * 1000, + inactivityTimeoutMs: 30000, + runIdPrefix: `web-probe-sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}`, + stdoutTailBytes: 32768, + }, script); + const result = job.result; + const transport = record(job.transport); + const remote = record(transport.remote); + const page = parseDashboardBrowserPayload(typeof remote.stdoutTail === "string" ? remote.stdoutTail : ""); + const artifacts = Array.isArray(transport.artifacts) ? transport.artifacts.map(record).map(compactDashboardArtifact) : []; + const screenshot = artifacts.find((artifact) => typeof artifact.localPath === "string" && String(artifact.localPath).endsWith(".png")) ?? null; + const browserOk = page?.ok === true; + const screenshotOk = options.action === "verify" || screenshot !== null && screenshot.verified === true; + const ok = result.exitCode === 0 && transport.ok === true && browserOk && screenshotOk; + return { + ok, + status: ok ? "pass" : "blocked", + command: `web-probe sentinel dashboard ${options.action}`, + node: state.spec.nodeId, + lane: state.spec.lane, + sentinelId: state.sentinelId, + publicUrl: `${publicBaseUrl}/`, + route, + viewport: options.viewport, + page, + screenshot, + artifacts, + artifactCount: artifacts.length, + remote: { + exitCode: remote.exitCode ?? null, + remoteDir: remote.remoteDir ?? null, + stdoutTail: ok ? "" : typeof remote.stdoutTail === "string" ? remote.stdoutTail.slice(-1200) : "", + stderrTail: ok ? "" : typeof remote.stderrTail === "string" ? remote.stderrTail.slice(-1200) : "", + }, + transport: { + ok: transport.ok ?? null, + runId: transport.runId ?? null, + artifactCount: transport.artifactCount ?? null, + expectedArtifactCount: transport.expectedArtifactCount ?? null, + downloadFailure: transport.downloadFailure ?? null, + }, + result: compactCommand(result), + degradedReason: ok ? null : dashboardDegradedReason(result, transport, page, screenshotOk), + valuesRedacted: true, + }; +} + +function sentinelDashboardBrowserModule(): string { + return String.raw`import { pathToFileURL } from "node:url"; + +const playwrightModulePath = process.env.UNIDESK_SENTINEL_DASHBOARD_PLAYWRIGHT_MODULE || ""; +const playwrightModuleSpecifier = playwrightModulePath ? pathToFileURL(playwrightModulePath).href : "playwright"; +const { chromium } = await import(playwrightModuleSpecifier); + +const url = process.env.UNIDESK_SENTINEL_DASHBOARD_URL; +const screenshotPath = process.env.UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT || ""; +const captureScreenshot = process.env.UNIDESK_SENTINEL_DASHBOARD_CAPTURE === "1"; +const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440); +const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900); +const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000); +const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"; +const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || ""; +const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || ""; +const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/"; + +if (!url) throw new Error("missing dashboard URL"); + +const consoleMessages = []; +const pageErrors = []; +const requestFailures = []; +const browser = await chromium.launch({ + headless: true, + args: ["--disable-gpu", "--no-sandbox"], + ...(executablePath ? { executablePath } : {}), +}); +const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 }); +const page = await context.newPage(); +page.on("console", (message) => { + if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) }); +}); +page.on("pageerror", (error) => { + if (pageErrors.length < 20) pageErrors.push({ message: String(error?.message || error).slice(0, 500) }); +}); +page.on("requestfailed", (request) => { + if (requestFailures.length < 20) requestFailures.push({ url: request.url().slice(0, 240), method: request.method(), failure: request.failure()?.errorText || null }); +}); + +let httpStatus = null; +let navigationError = null; +let navigationAttempts = 0; +const maxNavigationAttempts = 3; +const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts)); +for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) { + navigationAttempts = attempt; + navigationError = null; + try { + const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }); + httpStatus = response?.status() ?? null; + await page.waitForLoadState("networkidle", { timeout: Math.min(10000, perAttemptTimeout) }).catch(() => {}); + await page.waitForFunction(() => { + const root = document.querySelector("#monitor-web-root"); + if (!root) return false; + const ready = root.getAttribute("data-monitor-ready") === "true"; + const error = document.querySelector("#monitor-web-error"); + const runs = document.querySelectorAll(".run-list .run-row").length; + const trend = document.querySelector("[data-monitor-trend-curve]"); + return ready && (error || runs > 0 || trend); + }, null, { timeout: Math.min(15000, perAttemptTimeout) }).catch(() => {}); + await page.waitForTimeout(500); + const appReady = await page.evaluate(() => document.querySelector("#monitor-web-root")?.getAttribute("data-monitor-ready") === "true").catch(() => false); + if (appReady || attempt === maxNavigationAttempts) break; + } catch (error) { + navigationError = String(error?.message || error).slice(0, 500); + if (attempt === maxNavigationAttempts) break; + } + await page.waitForTimeout(750 * attempt); +} + +await page.evaluate(() => { + const detailPane = document.querySelector(".workspace-grid .pane-detail"); + if (detailPane instanceof HTMLElement) detailPane.scrollTop = Math.min(96, Math.max(0, detailPane.scrollHeight - detailPane.clientHeight)); +}).catch(() => {}); +await page.waitForTimeout(150); + +const trendHoverPoint = await page.evaluate(() => { + const target = document.querySelector(".trend-dot-hit .trend-dot-red") || document.querySelector(".trend-dot-hit .trend-dot-warning"); + if (!(target instanceof SVGElement)) return null; + const rect = target.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return null; + return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; +}).catch(() => null); +if (trendHoverPoint) { + await page.mouse.move(trendHoverPoint.x, trendHoverPoint.y); + await page.waitForTimeout(250); +} + +if (captureScreenshot && screenshotPath) { + await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => { + pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) }); + }); +} + +const dom = await page.evaluate(async ({ expectedRoutePrefix, expectedSentinelId }) => { + const visible = (element) => Boolean(element && !element.hidden); + const text = (selector) => String(document.querySelector(selector)?.textContent || "").replace(/\s+/g, " ").trim(); + const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; + const errorSampleCount = (counts) => numberValue(counts?.red) + numberValue(counts?.critical) + numberValue(counts?.error); + const warningSampleCount = (counts) => numberValue(counts?.warning) + numberValue(counts?.warn) + numberValue(counts?.amber); + const allSampleCount = (counts) => Object.values(counts || {}).reduce((sum, value) => sum + numberValue(value), 0); + const root = document.querySelector("#monitor-web-root"); + const shell = document.querySelector("[data-monitor-shell='true']"); + const error = document.querySelector("#monitor-web-error"); + const trend = document.querySelector("[data-monitor-trend-curve]"); + const trendTooltip = document.querySelector("[data-monitor-trend-tooltip='true']"); + const timeline = document.querySelector("[data-monitor-timeline='true']"); + const workspace = document.querySelector("[data-monitor-independent-scroll='true']"); + const checksPanel = document.querySelector("[data-monitor-checks='true']"); + const panes = Array.from(document.querySelectorAll(".workspace-grid .pane, [data-monitor-checks='true']")); + const detailPane = document.querySelector(".workspace-grid .pane-detail"); + const detailHeader = detailPane?.querySelector(".pane-header"); + const checksHeader = checksPanel?.querySelector(".pane-header"); + const internalTextPattern = /水合|投影|Trace|trace|Shell|API|DOM|Console|console|Runner|runner|JSONL|steer|facts|分页|HTTP|http|requestfailed|pageerror|Final Response|Code Agent|web-probe|observe|analyzer|终态/u; + const checkRows = Array.from(document.querySelectorAll("[data-check-row='true']")); + const cards = checkRows.slice(0, 8).map((row) => ({ + code: String(row.querySelector(".check-code")?.textContent || "").trim(), + title: String(row.querySelector(".check-title-cell strong")?.textContent || row.querySelector("strong")?.textContent || "").trim(), + body: String(row.textContent || "").replace(/\s+/g, " ").trim().slice(0, 180), + })); + const badCardTitles = cards.filter((card) => internalTextPattern.test(card.title)); + const badCardBodies = cards.filter((card) => internalTextPattern.test(card.body)); + const legendTexts = Array.from(document.querySelectorAll(".trend-legend .legend-item")).map((item) => String(item.textContent || "").replace(/\s+/g, " ").trim()); + const legendNumber = (label) => { + const row = legendTexts.find((item) => item.includes(label)) || ""; + const match = /(\d+)/u.exec(row); + return match ? Number(match[1]) : null; + }; + const chartCounts = { + error: legendNumber("最新点错误"), + warning: legendNumber("最新点告警"), + total: legendNumber("错误+告警合计"), + }; + chartCounts.ok = typeof chartCounts.error === "number" && typeof chartCounts.warning === "number" && typeof chartCounts.total === "number" + ? chartCounts.total === chartCounts.error + chartCounts.warning + : false; + const apiUrl = (path) => { + const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; + const prefix = basePath.replace(/\/+$/u, ""); + return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; + }; + const runsPayload = await fetch(apiUrl("/api/runs?limit=30&sort=updated"), { cache: "no-store" }).then((item) => item.json()).catch(() => null); + const overviewPayload = await fetch(apiUrl("/api/overview"), { cache: "no-store" }).then((item) => item.json()).catch(() => null); + const runs = Array.isArray(runsPayload?.runs) ? runsPayload.runs : Array.isArray(runsPayload?.items) ? runsPayload.items : []; + const latestRun = runs[0] || null; + const latestCounts = latestRun && latestRun.severityCounts && typeof latestRun.severityCounts === "object" && !Array.isArray(latestRun.severityCounts) + ? latestRun.severityCounts + : {}; + const latestRunCounts = { + runId: latestRun?.id || latestRun?.runId || null, + typeCount: numberValue(latestRun?.findingTypeCount ?? latestRun?.findingCount ?? latestRun?.finding_count), + error: 0, + warning: 0, + total: 0, + all: 0, + errorSamples: errorSampleCount(latestCounts), + warningSamples: warningSampleCount(latestCounts), + alertSamples: errorSampleCount(latestCounts) + warningSampleCount(latestCounts), + allSamples: allSampleCount(latestCounts), + }; + const latestDetailPayload = latestRunCounts.runId + ? await fetch(apiUrl("/api/runs/" + encodeURIComponent(latestRunCounts.runId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null) + : null; + const latestDetailRows = Array.isArray(latestDetailPayload?.findings) ? latestDetailPayload.findings : []; + const rowSeverity = (row) => { + const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase(); + if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red"; + if (["warning", "warn", "amber"].includes(raw)) return "warning"; + if (["info", "notice"].includes(raw)) return "info"; + return "healthy"; + }; + const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1; + const summarizeRows = (rows) => { + const errorRows = rows.filter((row) => rowSeverity(row) === "red"); + const warningRows = rows.filter((row) => rowSeverity(row) === "warning"); + const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0); + return { + typeCount: rows.length, + errorTypeCount: errorRows.length, + warningTypeCount: warningRows.length, + alertTypeCount: errorRows.length + warningRows.length, + errorSamples: sum(errorRows), + warningSamples: sum(warningRows), + alertSamples: sum(errorRows) + sum(warningRows), + }; + }; + const latestDetailSummary = summarizeRows(latestDetailRows); + latestRunCounts.typeCount = latestDetailSummary.typeCount; + latestRunCounts.error = latestDetailSummary.errorTypeCount; + latestRunCounts.warning = latestDetailSummary.warningTypeCount; + latestRunCounts.total = latestDetailSummary.alertTypeCount; + latestRunCounts.all = latestDetailSummary.typeCount; + const workspaceRect = workspace?.getBoundingClientRect(); + const checksRect = checksPanel?.getBoundingClientRect(); + const heightSummary = (rect) => { + const viewportHeight = window.innerHeight || 1; + const heightPx = rect ? Math.round(rect.height) : null; + const ratio = rect ? Math.round((rect.height / viewportHeight) * 1000) / 1000 : null; + return { + present: Boolean(rect), + heightPx, + ratio, + targetPx: Math.round(viewportHeight * 0.8), + bounded80: Boolean(rect && rect.height <= viewportHeight * 0.82 + 1), + near80: Boolean(rect && rect.height >= viewportHeight * 0.74 && rect.height <= viewportHeight * 0.82 + 1), + }; + }; + const workspacePaneHeights = Array.from(document.querySelectorAll(".workspace-grid > .pane")).map((pane) => heightSummary(pane.getBoundingClientRect())); + const stackedWorkspace = window.matchMedia("(max-width: 1120px)").matches; + const panelHeights = { + viewportHeight: window.innerHeight, + stackedWorkspace, + workspace: heightSummary(workspaceRect), + checks: heightSummary(checksRect), + workspacePanes: workspacePaneHeights, + workspacePaneBounded: workspacePaneHeights.length >= 2 && workspacePaneHeights.every((item) => item.bounded80 === true), + workspaceOk: false, + checksOk: false, + }; + panelHeights.workspaceOk = stackedWorkspace + ? panelHeights.workspacePaneBounded === true + : panelHeights.workspace.near80 === true && panelHeights.workspacePaneBounded === true; + panelHeights.checksOk = panelHeights.checks.near80 === true; + const checkScope = { + present: Boolean(checksPanel), + scope: checksPanel?.getAttribute("data-check-scope") || null, + runId: checksPanel?.getAttribute("data-check-run-id") || null, + typeCount: numberValue(checksPanel?.getAttribute("data-check-type-count")), + errorTypeCount: numberValue(checksPanel?.getAttribute("data-check-error-type-count")), + warningTypeCount: numberValue(checksPanel?.getAttribute("data-check-warning-type-count")), + alertTypeCount: numberValue(checksPanel?.getAttribute("data-check-alert-type-count")), + errorSamples: numberValue(checksPanel?.getAttribute("data-check-error-samples")), + warningSamples: numberValue(checksPanel?.getAttribute("data-check-warning-samples")), + alertSamples: numberValue(checksPanel?.getAttribute("data-check-alert-samples")), + visibleRowCount: document.querySelectorAll("[data-check-row='true']").length, + visibleAlertSamples: numberValue(checksPanel?.getAttribute("data-visible-check-alert-samples")), + matchesLatestRun: false, + matchesRunDetail: false, + belowWorkspace: Boolean(workspaceRect && checksRect && checksRect.top >= workspaceRect.bottom - 2), + fullWidth: Boolean(workspaceRect && checksRect && checksRect.width >= workspaceRect.width - 2), + }; + const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row")) + .find((row) => row.getAttribute("data-run-id") === runId) || null; + const selectedRunRow = runListRowFor(latestRunCounts.runId) || document.querySelector(".run-list .run-row.selected"); + const selectedRunTags = { + error: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), + warning: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), + errorVisible: Boolean(selectedRunRow?.querySelector("[data-run-error-tag='true']")), + warningVisible: Boolean(selectedRunRow?.querySelector("[data-run-warning-tag='true']")), + rowSelected: Boolean(selectedRunRow?.classList.contains("selected")), + matchesRunDetail: false, + }; + selectedRunTags.matchesRunDetail = selectedRunTags.error === latestDetailSummary.errorTypeCount + && selectedRunTags.warning === latestDetailSummary.warningTypeCount + && selectedRunTags.errorVisible === (latestDetailSummary.errorTypeCount > 0) + && selectedRunTags.warningVisible === (latestDetailSummary.warningTypeCount > 0) + && selectedRunTags.rowSelected === true; + checkScope.matchesLatestRun = checkScope.present === true + && checkScope.scope === "run" + && checkScope.runId === latestRunCounts.runId + && checkScope.errorTypeCount === latestRunCounts.error + && checkScope.warningTypeCount === latestRunCounts.warning + && checkScope.alertTypeCount === latestRunCounts.total; + checkScope.matchesRunDetail = checkScope.present === true + && checkScope.typeCount === latestDetailSummary.typeCount + && checkScope.errorTypeCount === latestDetailSummary.errorTypeCount + && checkScope.warningTypeCount === latestDetailSummary.warningTypeCount + && checkScope.alertTypeCount === latestDetailSummary.alertTypeCount + && checkScope.errorSamples === latestDetailSummary.errorSamples + && checkScope.warningSamples === latestDetailSummary.warningSamples + && checkScope.alertSamples === latestDetailSummary.alertSamples + && checkScope.visibleRowCount === latestDetailSummary.alertTypeCount + && checkScope.visibleAlertSamples === latestDetailSummary.alertSamples; + const overviewCounts = overviewPayload?.severityCounts && typeof overviewPayload.severityCounts === "object" && !Array.isArray(overviewPayload.severityCounts) + ? overviewPayload.severityCounts + : {}; + const overviewSamples = { + error: errorSampleCount(overviewCounts), + warning: warningSampleCount(overviewCounts), + total: errorSampleCount(overviewCounts) + warningSampleCount(overviewCounts), + allSamples: allSampleCount(overviewCounts), + }; + chartCounts.matchesLatestRun = chartCounts.ok === true + && chartCounts.error === latestRunCounts.error + && chartCounts.warning === latestRunCounts.warning + && chartCounts.total === latestRunCounts.total; + const trendPanel = document.querySelector(".trend-panel"); + const trendLegend = document.querySelector(".trend-panel .trend-legend"); + const trendPanelRect = trendPanel?.getBoundingClientRect(); + const trendLegendRect = trendLegend?.getBoundingClientRect(); + const trendPanelCompact = { + present: Boolean(trendPanelRect && trendLegendRect), + bottomSlackPx: trendPanelRect && trendLegendRect ? Math.round(trendPanelRect.bottom - trendLegendRect.bottom) : null, + ok: Boolean(trendPanelRect && trendLegendRect && trendPanelRect.bottom - trendLegendRect.bottom <= 28), + }; + const firstCheckRow = document.querySelector("[data-check-row='true']"); + let checkDialog = { opened: false, title: "", width: null, height: null, large: false }; + if (firstCheckRow instanceof HTMLElement) { + firstCheckRow.click(); + await new Promise((resolve) => window.setTimeout(resolve, 80)); + const dialog = document.querySelector("[data-check-dialog='true'] .check-dialog"); + const rect = dialog?.getBoundingClientRect(); + checkDialog = { + opened: Boolean(dialog), + title: String(dialog?.querySelector("#check-dialog-title")?.textContent || "").trim(), + width: rect ? Math.round(rect.width) : null, + height: rect ? Math.round(rect.height) : null, + large: Boolean(rect && rect.width >= Math.min(900, window.innerWidth * 0.7) && rect.height >= Math.min(460, window.innerHeight * 0.5)), + }; + const close = dialog?.querySelector("button[aria-label='关闭监测项详情']"); + if (close instanceof HTMLElement) close.click(); + } + const datasetSentinelId = root?.getAttribute("data-sentinel-id") || ""; + const finalPath = new URL(window.location.href).pathname.replace(/\/+$/u, "") || "/"; + const expectedPath = expectedRoutePrefix.replace(/\/+$/u, "") || "/"; + const routePrefixMatches = expectedPath === "/" ? finalPath === "/" : finalPath === expectedPath || finalPath.startsWith(expectedPath + "/"); + const sentinelBoundary = { + expectedSentinelId, + expectedRoutePrefix, + datasetSentinelId, + overviewSentinelId: overviewPayload?.sentinelId || null, + runsSentinelId: runsPayload?.sentinelId || null, + finalPath, + routePrefixMatches, + datasetMatches: expectedSentinelId ? datasetSentinelId === expectedSentinelId : true, + overviewMatches: expectedSentinelId ? overviewPayload?.sentinelId === expectedSentinelId : true, + runsPayloadMatches: expectedSentinelId ? runsPayload?.sentinelId === expectedSentinelId : true, + runRowsMatch: expectedSentinelId ? runs.every((run) => (run?.sentinelId || expectedSentinelId) === expectedSentinelId) : true, + }; + const statusText = text(".status-strip"); + const doc = document.documentElement; + const body = document.body; + const viewport = { width: window.innerWidth, height: window.innerHeight }; + const documentSize = { + width: Math.max(doc.scrollWidth, body?.scrollWidth || 0), + height: Math.max(doc.scrollHeight, body?.scrollHeight || 0), + }; + const overflow = []; + let overflowCount = 0; + for (const element of Array.from(document.querySelectorAll("body *"))) { + const rect = element.getBoundingClientRect(); + const overflowRight = rect.right - viewport.width; + const overflowLeft = -rect.left; + if (overflowRight > 1 || overflowLeft > 1) { + overflowCount += 1; + if (overflow.length < 5) { + overflow.push({ + tag: element.tagName.toLowerCase(), + className: String(element.className || "").slice(0, 40), + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + overflowRight: Math.max(0, Math.round(overflowRight)), + overflowLeft: Math.max(0, Math.round(overflowLeft)), + }); + } + } + } + return { + shell: Boolean(root && shell), + ready: root?.getAttribute("data-monitor-ready") === "true", + dataset: root ? { + node: root.getAttribute("data-node"), + lane: root.getAttribute("data-lane"), + sentinelId: datasetSentinelId, + basePath: root.getAttribute("data-base-path"), + contractVersion: root.getAttribute("data-contract-version"), + } : {}, + sentinelBoundary, + title: document.title, + finalUrl: window.location.href, + statusText: text(".topbar .pill"), + subtitle: text(".subtitle"), + summaryText: text(".status-strip"), + runRows: document.querySelectorAll(".run-list .run-row").length, + checkRows: document.querySelectorAll("[data-check-row='true']").length, + badCardTitleCount: badCardTitles.length, + badCardBodyCount: badCardBodies.length, + trendCurve: Boolean(trend), + trendDotCount: document.querySelectorAll(".trend-dot-hit").length, + trendTooltip: tooltipSummary(trendTooltip), + trendPanelText: text("#trend-heading"), + chartCounts, + latestRunCounts, + latestDetailSummary, + checkScope, + selectedRunTags, + trendPanelCompact, + checkDialog, + overviewSamples, + panelHeights, + scopeLabels: { + latestPointLegend: legendTexts.some((item) => item.includes("最新点")), + historicalSamples: legendTexts.some((item) => item.includes("历史样本累计")) || statusText.includes("历史错误样本"), + }, + timelineItems: document.querySelectorAll(".timeline-list .timeline-item").length, + timelineVisible: Boolean(timeline), + errorVisible: visible(error), + errorText: visible(error) ? text("#monitor-web-error").slice(0, 500) : "", + scrollModel: { + workspace: Boolean(workspace), + paneCount: panes.length, + independentScroll: panes.length >= 3 && panes.every((pane) => { + const style = window.getComputedStyle(pane); + return style.overflowY === "auto" || style.overflowY === "scroll"; + }), + stickyHeader: stickyHeaderSummary(detailPane, detailHeader), + stickyChecksHeader: stickyHeaderSummary(checksPanel, checksHeader), + }, + layout: { + viewport, + documentSize, + horizontalOverflow: documentSize.width > viewport.width + 1, + overflowCount, + overflow: overflow.slice(0, 2), + }, + }; + + function tooltipSummary(element) { + const body = String(element?.textContent || "").replace(/\s+/g, " ").trim(); + return { + visible: Boolean(element && body.length > 0), + text: body.slice(0, 240), + hasValues: /错误\s+\d+/u.test(body) && /告警\s+\d+/u.test(body) && /合计\s+\d+/u.test(body), + hasTime: /UTC/u.test(body) || /\d{4}-\d{2}-\d{2}/u.test(body), + }; + } + + function stickyHeaderSummary(pane, header) { + if (!(pane instanceof HTMLElement) || !(header instanceof HTMLElement)) { + return { present: false, coversScroll: false, backgroundOpaque: false, detailScrollTop: null }; + } + const rect = header.getBoundingClientRect(); + const style = window.getComputedStyle(header); + const sampleX = Math.round(rect.left + Math.min(32, Math.max(2, rect.width / 2))); + const sampleY = Math.round(rect.top + Math.min(12, Math.max(2, rect.height / 2))); + const topElement = document.elementFromPoint(sampleX, sampleY); + return { + present: true, + detailScrollTop: pane.scrollTop, + headerTop: Math.round(rect.top), + headerBottom: Math.round(rect.bottom), + zIndex: style.zIndex, + backgroundColor: style.backgroundColor, + coversScroll: Boolean(topElement && header.contains(topElement)), + backgroundOpaque: backgroundIsOpaque(style.backgroundColor), + topElementClass: String(topElement?.className || "").slice(0, 80), + }; + } + + function backgroundIsOpaque(value) { + const rgba = /rgba?\(([^)]+)\)/u.exec(value); + if (rgba === null) return value.length > 0 && value !== "transparent"; + const parts = rgba[1].split(",").map((part) => part.trim()); + if (parts.length < 4) return true; + return Number(parts[3]) >= 0.99; + } +}, { expectedRoutePrefix, expectedSentinelId }); + +const runFilterProbe = await page.evaluate(async ({ expectedRoutePrefix }) => { + const numberValue = (value) => Number.isFinite(Number(value)) ? Number(value) : 0; + const root = document.querySelector("#monitor-web-root"); + const apiUrl = (path) => { + const basePath = root?.getAttribute("data-base-path") || expectedRoutePrefix || ""; + const prefix = basePath.replace(/\/+$/u, ""); + return (prefix + (path.startsWith("/") ? path : "/" + path)) || path; + }; + const rowSeverity = (row) => { + const raw = String(row?.maxSeverity || row?.checkLevel || row?.severity || row?.level || "").toLowerCase(); + if (["red", "critical", "error", "blocked", "failed"].includes(raw)) return "red"; + if (["warning", "warn", "amber"].includes(raw)) return "warning"; + if (["info", "notice"].includes(raw)) return "info"; + return "healthy"; + }; + const sampleCount = (row) => Number.isFinite(Number(row?.count)) ? Number(row.count) : 1; + const summarizeRows = (rows) => { + const errorRows = rows.filter((row) => rowSeverity(row) === "red"); + const warningRows = rows.filter((row) => rowSeverity(row) === "warning"); + const sum = (items) => items.reduce((total, row) => total + sampleCount(row), 0); + return { + typeCount: rows.length, + errorTypeCount: errorRows.length, + warningTypeCount: warningRows.length, + alertTypeCount: errorRows.length + warningRows.length, + errorSamples: sum(errorRows), + warningSamples: sum(warningRows), + alertSamples: sum(errorRows) + sum(warningRows), + }; + }; + const runListRowFor = (runId) => Array.from(document.querySelectorAll(".run-list .run-row")) + .find((row) => row.getAttribute("data-run-id") === runId) || null; + const panelCounts = () => { + const panel = document.querySelector("[data-monitor-checks='true']"); + const panelRunId = panel?.getAttribute("data-check-run-id") || null; + const selectedRunRow = runListRowFor(panelRunId) || document.querySelector(".run-list .run-row.selected"); + return { + present: Boolean(panel), + runId: panelRunId, + typeCount: numberValue(panel?.getAttribute("data-check-type-count")), + errorTypeCount: numberValue(panel?.getAttribute("data-check-error-type-count")), + warningTypeCount: numberValue(panel?.getAttribute("data-check-warning-type-count")), + alertTypeCount: numberValue(panel?.getAttribute("data-check-alert-type-count")), + errorSamples: numberValue(panel?.getAttribute("data-check-error-samples")), + warningSamples: numberValue(panel?.getAttribute("data-check-warning-samples")), + alertSamples: numberValue(panel?.getAttribute("data-check-alert-samples")), + visibleRowCount: document.querySelectorAll("[data-check-row='true']").length, + visibleAlertSamples: numberValue(panel?.getAttribute("data-visible-check-alert-samples")), + selectedRunErrorTag: numberValue(String(selectedRunRow?.querySelector("[data-run-error-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), + selectedRunWarningTag: numberValue(String(selectedRunRow?.querySelector("[data-run-warning-tag='true']")?.textContent || "").match(/(\d+)/u)?.[1]), + selectedRunRowSelected: Boolean(selectedRunRow?.classList.contains("selected")), + }; + }; + const waitForRun = async (runId) => { + for (let index = 0; index < 30; index += 1) { + const current = panelCounts(); + if (current.runId === runId && current.present === true) return true; + await new Promise((resolve) => window.setTimeout(resolve, 100)); + } + return false; + }; + const select = document.querySelector("select[aria-label='选择运行记录']"); + if (!(select instanceof HTMLSelectElement)) { + return { ok: false, requestedRunId: null, reason: "run-select-missing" }; + } + const options = Array.from(select.options).filter((option) => option.value); + const requestedOption = options[Math.min(2, Math.max(0, options.length - 1))] || options[0] || null; + const requestedRunId = requestedOption?.value || null; + const fallbackOption = requestedOption || null; + const targetRunId = fallbackOption?.value || requestedRunId; + if (!targetRunId) return { ok: false, requestedRunId, reason: "run-option-missing" }; + select.value = targetRunId; + select.dispatchEvent(new Event("change", { bubbles: true })); + const panelReady = await waitForRun(targetRunId); + const detailPayload = await fetch(apiUrl("/api/runs/" + encodeURIComponent(targetRunId)), { cache: "no-store" }).then((item) => item.json()).catch(() => null); + const detailRows = Array.isArray(detailPayload?.findings) ? detailPayload.findings : []; + const expected = summarizeRows(detailRows); + const observed = panelCounts(); + const matchesRunDetail = observed.runId === targetRunId + && observed.typeCount === expected.typeCount + && observed.errorTypeCount === expected.errorTypeCount + && observed.warningTypeCount === expected.warningTypeCount + && observed.alertTypeCount === expected.alertTypeCount + && observed.errorSamples === expected.errorSamples + && observed.warningSamples === expected.warningSamples + && observed.alertSamples === expected.alertSamples + && observed.visibleRowCount === expected.alertTypeCount + && observed.visibleAlertSamples === expected.alertSamples + && observed.selectedRunErrorTag === expected.errorTypeCount + && observed.selectedRunWarningTag === expected.warningTypeCount + && observed.selectedRunRowSelected === true; + return { + ok: panelReady === true && matchesRunDetail === true, + requestedRunId, + requestedOptionPresent: Boolean(requestedOption), + targetRunId, + panelReady, + observed, + expected, + matchesRunDetail, + }; +}, { expectedRoutePrefix }); +dom.runFilterProbe = runFilterProbe; + +const consoleErrors = consoleMessages.filter((item) => item.type === "error"); +const ok = !navigationError + && httpStatus !== null + && httpStatus >= 200 + && httpStatus < 300 + && dom.shell === true + && dom.ready === true + && dom.sentinelBoundary?.datasetMatches === true + && dom.sentinelBoundary?.overviewMatches === true + && dom.sentinelBoundary?.runsPayloadMatches === true + && dom.sentinelBoundary?.runRowsMatch === true + && dom.sentinelBoundary?.routePrefixMatches === true + && dom.errorVisible !== true + && dom.trendCurve === true + && dom.chartCounts?.ok === true + && dom.chartCounts?.matchesLatestRun === true + && dom.checkScope?.matchesLatestRun === true + && dom.checkScope?.matchesRunDetail === true + && dom.selectedRunTags?.matchesRunDetail === true + && dom.runFilterProbe?.ok === true + && dom.runFilterProbe?.requestedOptionPresent === true + && dom.checkScope?.belowWorkspace === true + && dom.checkScope?.fullWidth === true + && dom.scopeLabels?.latestPointLegend === true + && dom.scopeLabels?.historicalSamples === true + && (dom.trendDotCount === 0 || (dom.trendTooltip?.visible === true && dom.trendTooltip?.hasValues === true && dom.trendTooltip?.hasTime === true)) + && dom.trendPanelCompact?.ok === true + && dom.checkRows > 0 + && dom.checkDialog?.opened === true + && dom.checkDialog?.large === true + && dom.badCardTitleCount === 0 + && dom.badCardBodyCount === 0 + && dom.timelineVisible === true + && dom.scrollModel?.independentScroll === true + && dom.scrollModel?.stickyHeader?.present === true + && dom.scrollModel?.stickyHeader?.backgroundOpaque === true + && dom.scrollModel?.stickyChecksHeader?.present === true + && dom.scrollModel?.stickyChecksHeader?.backgroundOpaque === true + && dom.panelHeights?.workspaceOk === true + && dom.panelHeights?.checksOk === true + && dom.layout?.horizontalOverflow !== true + && pageErrors.length === 0; + +console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ + ok, + url, + httpStatus, + navigationError, + navigationAttempts, + executablePath: executablePath || null, + viewport: { width, height }, + screenshotPath: captureScreenshot ? screenshotPath : null, + dom, + consoleCount: consoleMessages.length, + consoleErrorCount: consoleErrors.length, + pageErrorCount: pageErrors.length, + requestFailureCount: requestFailures.length, + consoleMessages: consoleMessages.slice(0, 8), + pageErrors: pageErrors.slice(0, 8), + requestFailures: requestFailures.slice(0, 8), + valuesRedacted: true, +})); + +await context.close().catch(() => {}); +await browser.close().catch(() => {}); +`; +} + +function parseDashboardBrowserPayload(textValue: string): Record | null { + const marker = "__WEB_PROBE_SENTINEL_DASHBOARD_JSON__"; + const index = textValue.lastIndexOf(marker); + if (index < 0) return null; + try { + return record(JSON.parse(textValue.slice(index + marker.length).trim())); + } catch { + return null; + } +} + +function dashboardScreenshotName(options: Extract, state: SentinelCicdState): string { + const raw = options.name ?? `sentinel-dashboard-${state.spec.nodeId.toLowerCase()}-${state.spec.lane}-${state.sentinelId}.png`; + const safe = raw.replace(/[^A-Za-z0-9._-]+/gu, "-").slice(0, 120); + return safe.endsWith(".png") ? safe : `${safe}.png`; +} + +function compactDashboardArtifact(artifact: Record): Record { + const transfer = record(artifact.transfer); + return { + remotePath: typeof artifact.remotePath === "string" ? artifact.remotePath : null, + localPath: typeof artifact.localPath === "string" ? artifact.localPath : null, + bytes: Number.isFinite(Number(artifact.bytes)) ? Number(artifact.bytes) : null, + sha256: typeof artifact.sha256 === "string" ? artifact.sha256 : null, + verified: artifact.verified === true, + transfer: Object.keys(transfer).length === 0 ? null : { + strategy: transfer.strategy ?? null, + transport: transfer.transport ?? null, + chunks: transfer.chunks ?? null, + elapsedMs: transfer.elapsedMs ?? null, + throughputBytesPerSecond: transfer.throughputBytesPerSecond ?? null, + }, + }; +} + +function dashboardDegradedReason(result: CommandResult, transport: Record, page: Record | null, screenshotOk: boolean): string { + if (result.timedOut) return "sentinel-dashboard-command-timeout"; + if (transport.ok !== true) return "sentinel-dashboard-transport-failed"; + if (page === null) return "sentinel-dashboard-browser-output-missing"; + if (page.ok !== true) return "sentinel-dashboard-render-failed"; + if (!screenshotOk) return "sentinel-dashboard-screenshot-missing"; + return "sentinel-dashboard-unknown"; +} + +function renderAsyncP5Job(state: SentinelCicdState, subcommand: readonly string[], timeoutSeconds: number, releaseId: string | null, reason: string | null, quickVerify: boolean): RenderedCliResult { + const args = ["web-probe", "sentinel", ...subcommand, "--node", state.spec.nodeId, "--lane", state.spec.lane, "--sentinel", state.sentinelId, "--confirm", "--wait", "--timeout-seconds", String(timeoutSeconds)]; + if (releaseId !== null) args.push("--release-id", releaseId); + if (reason !== null) args.push("--reason", reason); + if (quickVerify) args.push("--quick-verify"); + const job = startJob(`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_${subcommand.join("_")}`, ["bun", "scripts/cli.ts", ...args], `Run HWLAB ${state.spec.lane} web-probe sentinel ${state.sentinelId} ${subcommand.join(" ")} for node ${state.spec.nodeId}`); + const command = `web-probe sentinel ${subcommand.join(" ")}`; + return rendered(true, command, renderAsyncJobResult({ + ok: true, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode: "async-job", + mutation: true, + job, + next: { + status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + wait: ["bun", "scripts/cli.ts", ...args].join(" "), + }, + valuesRedacted: true, + })); +} + +function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number): Record { + const namespace = stringAt(state.runtime, "namespace"); + const serviceName = stringAt(state.runtime, "serviceName"); + const servicePort = numberAt(state.runtime, "servicePort"); + const deploymentName = stringAt(state.runtime, "deploymentName"); + const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`; + if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") { + return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url); + } + const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; + const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); + const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); + const postScript = [ + "const path = Buffer.from(process.env.SENTINEL_PATH_B64 || '', 'base64').toString('utf8');", + "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", + `const url = 'http://127.0.0.1:${servicePort}' + path;`, + "fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body }).then(async (response) => {", + " const text = await response.text();", + " process.stdout.write(text);", + " if (!response.ok) process.exit(22);", + "}).catch((error) => {", + " console.error(error && error.stack ? error.stack : String(error));", + " process.exit(23);", + "});", + ].join(" "); + const script = method === "GET" + ? `kubectl get --raw ${shellQuote(proxyPath)}` + : [ + "set -eu", + `kubectl exec -n ${shellQuote(namespace)} deploy/${shellQuote(deploymentName)} -- env SENTINEL_PATH_B64=${shellQuote(pathB64)} SENTINEL_BODY_B64=${shellQuote(bodyB64)} node -e ${shellQuote(postScript)}`, + ].join("\n"); + const maxAttempts = method === "GET" ? 3 : 1; + const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); + const attempts: Record[] = []; + let result: CommandResult | null = null; + let parsed: Record | null = null; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: attemptTimeoutSeconds * 1000 }); + parsed = parseJsonObject(result.stdout); + attempts.push({ attempt, ...compactCommand(result), parsedOk: parsed !== null, valuesRedacted: true }); + if (result.exitCode === 0) break; + } + const compactBodyJson = compactSentinelServiceBodyJson(parsed); + return { + ok: result?.exitCode === 0, + method, + path: pathWithQuery, + internalUrl: `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`, + httpStatus: result?.exitCode === 0 ? 200 : null, + bodyJson: record(compactBodyJson), + bodyTextPreview: parsed === null ? clipTail(result?.stdout ?? "", 4000) : "", + bodyBytes: Buffer.byteLength(result?.stdout ?? ""), + error: result?.exitCode === 0 ? null : clipTail(`${result?.stderr ?? ""}${result?.stdout ?? ""}`, 1000), + proxyPath, + result: result === null ? null : compactCommand(result), + attempts, + valuesRedacted: true, + }; +} + +function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number, url: string): Record { + const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); + const fetchScript = [ + "const method = process.env.SENTINEL_METHOD || 'GET';", + "const url = process.env.SENTINEL_URL || '';", + "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", + "const attempts = Math.max(1, Number(process.env.SENTINEL_ATTEMPTS || '1') || 1);", + "const delayMs = Math.max(0, Number(process.env.SENTINEL_RETRY_DELAY_MS || '0') || 0);", + "const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;", + "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));", + "(async () => {", + " let lastError = null;", + " for (let attempt = 1; attempt <= attempts; attempt += 1) {", + " try {", + " const response = await fetch(url, { method, headers, body: method === 'POST' ? body : undefined });", + " const text = await response.text();", + " process.stdout.write(text);", + " if (!response.ok) process.exit(22);", + " process.exit(0);", + " } catch (error) {", + " lastError = error;", + " console.error(JSON.stringify({ attempt, attempts, code: error?.cause?.code ?? null, address: error?.cause?.address ?? null, valuesRedacted: true }));", + " if (attempt < attempts && delayMs > 0) await sleep(delayMs);", + " }", + " }", + " console.error(lastError && lastError.stack ? lastError.stack : String(lastError));", + " process.exit(23);", + "})().catch((error) => {", + " console.error(error && error.stack ? error.stack : String(error));", + " process.exit(24);", + "});", + ].join(" "); + const attempts = method === "GET" ? 6 : 3; + const retryDelayMs = 1000; + const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 20 : 70)); + const result = runCommand(["node", "-e", fetchScript], repoRoot, { + timeoutMs: attemptTimeoutSeconds * 1000, + env: { + ...process.env, + SENTINEL_METHOD: method, + SENTINEL_URL: url, + SENTINEL_BODY_B64: bodyB64, + SENTINEL_ATTEMPTS: String(attempts), + SENTINEL_RETRY_DELAY_MS: String(retryDelayMs), + }, + }); + const parsed = parseJsonObject(result.stdout); + const compactBodyJson = compactSentinelServiceBodyJson(parsed); + return { + ok: result.exitCode === 0, + method, + path: pathWithQuery, + internalUrl: url, + httpStatus: result.exitCode === 0 ? 200 : null, + bodyJson: record(compactBodyJson), + bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "", + bodyBytes: Buffer.byteLength(result.stdout), + error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000), + proxyPath: null, + result: compactCommand(result), + attempts: [{ attempt: "1..n", maxAttempts: attempts, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }], + transport: "direct-service", + valuesRedacted: true, + }; +} + +function compactSentinelServiceBodyJson(value: Record | null): unknown { + if (value === null || typeof value.renderedText !== "string") return value; + return { + ...pickFields(value, ["ok", "view", "error", "availableViews", "valuesRedacted"]), + run: pickFields(record(value.run), ["id", "scenario_id", "status", "node", "lane", "observer_id", "state_dir", "report_json_sha256", "finding_count", "artifact_count", "maintenance", "created_at", "updated_at"]), + summary: pickFields(record(value.summary), ["reason", "status", "valuesRedacted"]), + findings: Array.isArray(value.findings) ? value.findings.slice(0, 12) : [], + renderedText: value.renderedText, + valuesRedacted: true, + }; +} + +function pickFields(value: Record, keys: readonly string[]): Record { + const picked: Record = {}; + for (const key of keys) { + if (Object.prototype.hasOwnProperty.call(value, key)) picked[key] = value[key]; + } + return picked; +} + +function clipTail(value: string, maxChars: number): string { + return value.length <= maxChars ? value : value.slice(-maxChars); +} + +function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record { + const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); + const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`; + const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000)); + const js = [ + "const url=process.env.REQ_URL||'';", + "const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);", + "let out;", + "try{", + " const controller=new AbortController();", + " const timer=setTimeout(()=>controller.abort(), timeoutMs);", + " const started=Date.now();", + " const res=await fetch(url,{signal:controller.signal});", + " const text=await res.text();", + " clearTimeout(timer);", + " let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}", + " out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};", + "}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}", + "console.log(JSON.stringify(out));", + ].join(""); + const result = runCommand(["bun", "-e", js], repoRoot, { + timeoutMs: timeoutMs + 2000, + env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) }, + }); + const parsed = parseJsonObject(result.stdout); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + method: "GET", + path: pathWithQuery, + publicUrl: url, + httpStatus: parsed?.httpStatus ?? null, + bodyJson: record(parsed?.bodyJson), + bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "", + bodyBytes: parsed?.bodyBytes ?? null, + error: parsed?.error ?? null, + result: compactCommand(result), + valuesRedacted: true, + }; +} + +function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { + const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl"); + const hostname = stringAt(state.publicExposure, "hostname"); + const expectedA = stringAt(state.publicExposure, "expectedA"); + const probeUrl = `${publicBaseUrl.replace(/\/$/u, "")}/api/health`; + const script = [ + "set +e", + `host=${shellQuote(hostname)}`, + `expected=${shellQuote(expectedA)}`, + `url=${shellQuote(probeUrl)}`, + "dns=$(getent ahostsv4 \"$host\" 2>/dev/null | awk '{print $1}' | sort -u | paste -sd, -)", + "headers=$(mktemp)", + "body=$(mktemp)", + "writeout=$(curl -sS -D \"$headers\" -o \"$body\" --connect-timeout 8 --max-time 20 --write-out '%{http_code} %{ssl_verify_result} %{remote_ip}' \"$url\" 2>/tmp/web-probe-sentinel-public.err)", + "curl_rc=$?", + "body_head=$(head -c 4000 \"$body\" | base64 | tr -d '\\n')", + "node - \"$dns\" \"$expected\" \"$writeout\" \"$curl_rc\" \"$url\" \"$body_head\" \"$headers\" <<'NODE'", + "const fs=require('node:fs');", + "const [dns,expected,writeout,rcRaw,url,bodyB64,headersPath]=process.argv.slice(2);", + "const [statusRaw,sslRaw,remoteIp]=String(writeout||'').trim().split(/\\s+/);", + "const status=Number(statusRaw||0);", + "const ssl=Number(sslRaw||-1);", + "const addrs=dns?dns.split(',').filter(Boolean):[];", + "const headers=(()=>{try{return fs.readFileSync(headersPath,'utf8')}catch{return ''}})();", + "const body=Buffer.from(bodyB64||'', 'base64').toString('utf8');", + "let bodyJson=null; try{bodyJson=JSON.parse(body)}catch{}", + "const authCovered=status===401||status===403||status>=200&&status<300;", + "const edgeOk=Number(rcRaw)===0&&ssl===0&&status>0&&status<500;", + "const upstreamOk=status>=200&&status<300&&(bodyJson?.ok===true||body.includes('valuesRedacted'));", + "const dnsMatches=addrs.includes(expected);", + "console.log(JSON.stringify({ok:dnsMatches&&edgeOk&&authCovered&&upstreamOk,publicUrl:url,dns:{addresses:addrs,expectedA:expected,matches:dnsMatches},tls:{verified:ssl===0,sslVerifyResult:ssl,remoteIp:remoteIp||null},https:{curlExitCode:Number(rcRaw),httpStatus:status,edgeOk},auth:{requestAuthorizationHeader:false,covered:authCovered,status},upstream:{ok:upstreamOk,bodyPreview:body.slice(0,200)},headers:{wwwAuthenticate:/^www-authenticate:/im.test(headers)},valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function probeSentinelPublicDashboard(state: SentinelCicdState, timeoutSeconds: number): Record { + const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); + const rootUrl = `${publicBaseUrl}/`; + const cssUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.css`; + const jsUrl = `${publicBaseUrl}/monitor-web/assets/monitor-web.js`; + const vueUrl = `${publicBaseUrl}/monitor-web/assets/vendor/vue.esm-browser.prod.js`; + const script = [ + "set +e", + `root_url=${shellQuote(rootUrl)}`, + `css_url=${shellQuote(cssUrl)}`, + `js_url=${shellQuote(jsUrl)}`, + `vue_url=${shellQuote(vueUrl)}`, + "root_body=$(mktemp)", + "css_body=$(mktemp)", + "js_body=$(mktemp)", + "vue_body=$(mktemp)", + "root_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$root_body\" --write-out '%{http_code}' \"$root_url\" 2>/tmp/web-probe-sentinel-dashboard-root.err); root_rc=$?", + "css_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$css_body\" --write-out '%{http_code}' \"$css_url\" 2>/tmp/web-probe-sentinel-dashboard-css.err); css_rc=$?", + "js_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$js_body\" --write-out '%{http_code}' \"$js_url\" 2>/tmp/web-probe-sentinel-dashboard-js.err); js_rc=$?", + "vue_code=$(curl -sS -L --connect-timeout 8 --max-time 20 -o \"$vue_body\" --write-out '%{http_code}' \"$vue_url\" 2>/tmp/web-probe-sentinel-dashboard-vue.err); vue_rc=$?", + "node - \"$root_url\" \"$css_url\" \"$js_url\" \"$vue_url\" \"$root_code\" \"$root_rc\" \"$css_code\" \"$css_rc\" \"$js_code\" \"$js_rc\" \"$vue_code\" \"$vue_rc\" \"$root_body\" \"$css_body\" \"$js_body\" \"$vue_body\" <<'NODE'", + "const fs=require('node:fs');", + "const [rootUrl,cssUrl,jsUrl,vueUrl,rootCode,rootRc,cssCode,cssRc,jsCode,jsRc,vueCode,vueRc,rootPath,cssPath,jsPath,vuePath]=process.argv.slice(2);", + "function read(path){try{return fs.readFileSync(path,'utf8')}catch{return ''}}", + "const root=read(rootPath); const css=read(cssPath); const js=read(jsPath); const vue=read(vuePath);", + "const rootOk=Number(rootRc)===0&&Number(rootCode)>=200&&Number(rootCode)<300&&root.includes('id=\"monitor-web-root\"')&&root.includes('/monitor-web/assets/monitor-web.js')&&root.includes('monitor-web-bootstrap');", + "const cssOk=Number(cssRc)===0&&Number(cssCode)>=200&&Number(cssCode)<300&&css.includes('monitor-shell')&&css.includes('workspace-grid')&&css.includes('trend-stage')&&css.length>1000;", + "const jsOk=Number(jsRc)===0&&Number(jsCode)>=200&&Number(jsCode)<300&&js.includes('createApp')&&js.includes('/api/overview')&&js.includes('data-monitor-trend-curve')&&js.length>1000;", + "const vueOk=Number(vueRc)===0&&Number(vueCode)>=200&&Number(vueCode)<300&&vue.includes('createApp')&&vue.length>80000;", + "console.log(JSON.stringify({ok:rootOk&&cssOk&&jsOk&&vueOk,root:{url:rootUrl,httpStatus:Number(rootCode),bytes:Buffer.byteLength(root),shell:root.includes('id=\"monitor-web-root\"'),contract:root.includes('draft-2026-06-27-p11-monitor-web-observability-dashboard')},css:{url:cssUrl,httpStatus:Number(cssCode),bytes:Buffer.byteLength(css),workspaceGrid:css.includes('workspace-grid'),trendStage:css.includes('trend-stage')},js:{url:jsUrl,httpStatus:Number(jsCode),bytes:Buffer.byteLength(js),vueApp:js.includes('createApp'),apiClient:js.includes('/api/overview'),trend:js.includes('data-monitor-trend-curve')},vue:{url:vueUrl,httpStatus:Number(vueCode),bytes:Buffer.byteLength(vue),runtime:vue.includes('createApp')},valuesRedacted:true}));", + "NODE", + ].join("\n"); + const result = runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 30) * 1000 }); + const parsed = parseJsonObject(result.stdout); + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; +} + +function renderMaintenanceResult(result: Record): string { + const serviceHealth = record(result.serviceHealth); + const maintenance = record(result.maintenance); + const quickVerify = record(result.quickVerify); + const quickVerifyBusiness = record(quickVerify.businessStatus); + const planned = record(result.planned); + const blocker = record(result.blocker); + const next = record(result.next); + const maintenanceBody = record(maintenance.bodyJson); + const state = record(maintenanceBody.maintenance); + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status", result.mutation ?? false]]), + "", + table(["SERVICE", "HTTP", "INTERNAL_URL"], [[serviceHealth.ok, serviceHealth.httpStatus, serviceHealth.internalUrl]]), + "", + Object.keys(state).length > 0 + ? table(["ACTIVE", "RELEASE", "STARTED", "STOPPED", "VERIFY_RUN"], [[state.active, state.releaseId, state.startedAt, state.stoppedAt, state.quickVerifyPlannedRunId]]) + : Object.keys(planned).length > 0 + ? table(["ACTION", "RELEASE", "REASON", "QUICK_VERIFY"], [[planned.action, planned.releaseId, planned.reason, planned.quickVerify]]) + : "MAINTENANCE\n-", + "", + Object.keys(quickVerify).length === 0 ? "QUICK_VERIFY\n-" : table(["OK", "BUSINESS", "RUN", "SCENARIO", "OBSERVER", "REPORT", "FINDINGS"], [[quickVerify.ok, quickVerifyBusiness.status ?? "-", quickVerify.runId, quickVerify.scenarioId, quickVerify.observerId, quickVerify.reportJsonSha256, quickVerify.findingCount]]), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), + "", + "NEXT", + ` validate: ${next.validate ?? "-"}`, + ` report: ${next.report ?? "-"}`, + ` maintenance-stop: ${next.maintenanceStop ?? "-"}`, + "", + "DISCLOSURE", + " maintenance uses the k3s internal sentinel Service DNS; no public fallback or second runner is used.", + ].join("\n"); +} + +function renderValidateResult(result: Record): string { + const health = record(result.serviceHealth); + const metrics = record(result.metrics); + const report = record(result.report); + const publicExposure = record(result.publicExposure); + const publicDashboard = record(result.publicDashboard); + const quickVerify = record(result.quickVerify); + const blocker = record(result.blocker); + const next = record(result.next); + const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []); + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]), + "", + table(["CHECK", "OK", "DETAIL"], [ + ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`], + ["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`], + ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], + ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], + ["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`], + ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`], + ]), + "", + warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "BLOCKERS"], [[blocker.code, Array.isArray(blocker.blockers) ? blocker.blockers.join(",") : blocker.reason]]), + "", + "NEXT", + ` quick-verify: ${next.quickVerify ?? "-"}`, + ` report: ${next.report ?? "-"}`, + ` maintenance-start: ${next.maintenanceStart ?? "-"}`, + "", + "DISCLOSURE", + " validate checks /api/health, /metrics, indexed analyze report and publicExposure without printing tokens.", + ].join("\n"); +} + +function renderDashboardResult(result: Record): string { + const page = record(result.page); + const dom = record(page.dom); + const dataset = record(dom.dataset); + const layout = record(dom.layout); + const chartCounts = record(dom.chartCounts); + const latestRunCounts = record(dom.latestRunCounts); + const latestDetailSummary = record(dom.latestDetailSummary); + const checkScope = record(dom.checkScope); + const selectedRunTags = record(dom.selectedRunTags); + const trendPanelCompact = record(dom.trendPanelCompact); + const checkDialog = record(dom.checkDialog); + const runFilterProbe = record(dom.runFilterProbe); + const runFilterObserved = record(runFilterProbe.observed); + const runFilterExpected = record(runFilterProbe.expected); + const overviewSamples = record(dom.overviewSamples); + const panelHeights = record(dom.panelHeights); + const workspaceHeight = record(panelHeights.workspace); + const checksHeight = record(panelHeights.checks); + const screenshot = record(result.screenshot); + const remote = record(result.remote); + const transport = record(result.transport); + const degradedReason = result.degradedReason ?? null; + return [ + String(result.command), + "", + table(["NODE", "LANE", "SENTINEL", "STATUS", "URL"], [[result.node, result.lane, result.sentinelId, result.ok === true ? "pass" : "blocked", result.publicUrl]]), + "", + table(["HTTP", "SHELL", "RUN_ROWS", "CHECK_ROWS", "TABS", "ERRORS", "CONSOLE_ERR", "REQ_FAIL"], [[ + page.httpStatus ?? "-", + dom.shell, + dom.runRows, + dom.checkRows, + dom.detailTabs, + page.pageErrorCount, + page.consoleErrorCount, + page.requestFailureCount, + ]]), + "", + table(["TITLE", "STATUS_TEXT", "CONTRACT", "BASE_PATH"], [[dom.title, dom.statusText, dataset.contractVersion, dataset.basePath ?? "-"]]), + "", + table(["TREND_ERR_TYPES", "TREND_ALERT_TYPES", "TREND_TOTAL_TYPES", "TREND_EXACT", "MATCH_LATEST", "BAD_TITLE", "BAD_BODY"], [[ + chartCounts.error ?? "-", + chartCounts.warning ?? "-", + chartCounts.total ?? "-", + chartCounts.ok ?? "-", + chartCounts.matchesLatestRun ?? "-", + dom.badCardTitleCount ?? "-", + dom.badCardBodyCount ?? "-", + ]]), + "", + table(["LATEST_RUN", "TYPE_COUNT", "ERR_TYPES", "ALERT_TYPES", "TOTAL_TYPES", "SAMPLE_TOTAL", "HIST_ERR", "HIST_ALERT"], [[ + latestRunCounts.runId ?? "-", + latestRunCounts.typeCount ?? "-", + latestRunCounts.error ?? "-", + latestRunCounts.warning ?? "-", + latestRunCounts.total ?? "-", + latestRunCounts.alertSamples ?? "-", + overviewSamples.error ?? "-", + overviewSamples.warning ?? "-", + ]]), + "", + table(["CHECK_SCOPE", "CHECK_RUN", "CHECK_TYPES", "CHECK_ERR_TYPES", "CHECK_ALERT_TYPES", "SAMPLE_ERR", "SAMPLE_ALERT", "CHECK_MATCH_LATEST", "CHECK_MATCH_DETAIL"], [[ + checkScope.scope ?? "-", + checkScope.runId ?? "-", + `${checkScope.typeCount ?? "-"}/${latestDetailSummary.typeCount ?? "-"}`, + `${checkScope.errorTypeCount ?? "-"}/${latestDetailSummary.errorTypeCount ?? "-"}`, + `${checkScope.alertTypeCount ?? "-"}/${latestDetailSummary.alertTypeCount ?? "-"}`, + checkScope.errorSamples ?? "-", + checkScope.alertSamples ?? "-", + checkScope.matchesLatestRun ?? "-", + checkScope.matchesRunDetail ?? "-", + ]]), + "", + table(["CHECK_VISIBLE_ROWS", "CHECK_VISIBLE_ALERT", "RUN_TAG_ERR", "RUN_TAG_ALERT", "RUN_TAG_MATCH", "BELOW_WORKSPACE", "FULL_WIDTH"], [[ + checkScope.visibleRowCount ?? "-", + checkScope.visibleAlertSamples ?? "-", + selectedRunTags.error ?? "-", + selectedRunTags.warning ?? "-", + selectedRunTags.matchesRunDetail ?? "-", + checkScope.belowWorkspace ?? "-", + checkScope.fullWidth ?? "-", + ]]), + "", + table(["TREND_PANEL_SLACK", "TREND_PANEL_COMPACT", "DETAIL_DIALOG", "DIALOG_LARGE"], [[ + trendPanelCompact.bottomSlackPx ?? "-", + trendPanelCompact.ok ?? "-", + checkDialog.opened ?? "-", + checkDialog.large ?? "-", + ]]), + "", + table(["WORKSPACE_H", "WORKSPACE_RATIO", "WORKSPACE_80", "CHECKS_H", "CHECKS_RATIO", "CHECKS_80", "PANES_80"], [[ + `${workspaceHeight.heightPx ?? "-"}/${workspaceHeight.targetPx ?? "-"}`, + workspaceHeight.ratio ?? "-", + panelHeights.workspaceOk ?? "-", + `${checksHeight.heightPx ?? "-"}/${checksHeight.targetPx ?? "-"}`, + checksHeight.ratio ?? "-", + panelHeights.checksOk ?? "-", + panelHeights.workspacePaneBounded ?? "-", + ]]), + "", + table(["FILTER_RUN", "FILTER_OPTION", "FILTER_TYPES", "FILTER_ERR_TYPES", "FILTER_ALERT_TYPES", "FILTER_SAMPLE_ERR", "FILTER_SAMPLE_ALERT", "FILTER_MATCH_DETAIL"], [[ + runFilterProbe.targetRunId ?? "-", + runFilterProbe.requestedOptionPresent ?? "-", + `${runFilterObserved.typeCount ?? "-"}/${runFilterExpected.typeCount ?? "-"}`, + `${runFilterObserved.errorTypeCount ?? "-"}/${runFilterExpected.errorTypeCount ?? "-"}`, + `${runFilterObserved.alertTypeCount ?? "-"}/${runFilterExpected.alertTypeCount ?? "-"}`, + runFilterObserved.errorSamples ?? "-", + runFilterObserved.alertSamples ?? "-", + runFilterProbe.matchesRunDetail ?? "-", + ]]), + "", + table(["VIEWPORT", "DOC", "H_OVERFLOW", "OVERFLOW_COUNT"], [[ + result.viewport, + `${record(layout.documentSize).width ?? "-"}x${record(layout.documentSize).height ?? "-"}`, + layout.horizontalOverflow, + layout.overflowCount, + ]]), + "", + Object.keys(screenshot).length === 0 + ? "SCREENSHOT\n-" + : table(["LOCAL_PATH", "BYTES", "SHA256", "VERIFIED"], [[screenshot.localPath, screenshot.bytes, short(screenshot.sha256), screenshot.verified]]), + "", + degradedReason === null ? "BLOCKER\n-" : table(["CODE", "REMOTE_EXIT", "TRANSPORT"], [[degradedReason, remote.exitCode, transport.ok]]), + "", + "NEXT", + ` screenshot: bun scripts/cli.ts web-probe sentinel dashboard screenshot --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, + ` validate: bun scripts/cli.ts web-probe sentinel validate --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId}`, + "", + "DISCLOSURE", + " dashboard verify uses the YAML publicExposure URL and remote browser execution; it does not start a sentinel run or inspect provider payloads.", + ].join("\n"); +} + +function renderReportResult(result: Record): string { + const report = record(result.report); + const body = record(report.bodyJson); + const run = record(body.run); + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "VIEW", "RUN"], [[result.node, result.lane, report.ok ? "ok" : "blocked", body.view ?? "-", run.id ?? "-"]]), + "", + table(["HTTP", "ERROR", "REPORT"], [[report.httpStatus, body.error ?? report.error ?? "-", short(run.report_json_sha256)]]), + "", + "DISCLOSURE", + " report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.", + ].join("\n"); +} diff --git a/scripts/src/hwlab-node-web-sentinel-resolver.ts b/scripts/src/hwlab-node-web-sentinel-resolver.ts index bf062af0..c2f63fe1 100644 --- a/scripts/src/hwlab-node-web-sentinel-resolver.ts +++ b/scripts/src/hwlab-node-web-sentinel-resolver.ts @@ -2,6 +2,7 @@ // Responsibility: Resolve YAML-first web-probe sentinel registry entries into one selected sentinel config graph. import { existsSync, readFileSync } from "node:fs"; import { rootPath } from "./config"; +import { readWebProbeSentinelConfigRefTarget, type WebProbeSentinelTemplateContext } from "./hwlab-node-web-sentinel-config-ref"; import { HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS, type HwlabRuntimeLaneSpec, type HwlabRuntimeWebProbeSentinelConfigRefKey, type HwlabRuntimeWebProbeSentinelRegistryItemSpec } from "./hwlab-node-lanes"; export interface ResolvedWebProbeSentinel { @@ -110,7 +111,7 @@ function resolveRegistrySentinel(spec: HwlabRuntimeLaneSpec, registry: readonly const ids = registry.map((item) => item.id).join(", "); throw new Error(`unknown web-probe sentinel ${sentinelId ?? "-"} for ${spec.nodeId}/${spec.lane}; available: ${ids}`); } - const target = readConfigRefRecord(selected.configRef); + const target = readConfigRefRecord(selected.configRef, spec); const targetId = optionalStringAt(target, "id") ?? selected.id; if (targetId !== selected.id) { throw new Error(`${selected.configRef}.id=${targetId} does not match registry id ${selected.id}`); @@ -151,13 +152,14 @@ function normalizeSentinelConfigRefs(target: Record, ref: strin return Object.fromEntries(HWLAB_WEB_PROBE_SENTINEL_CONFIG_REF_KEYS.map((key) => [key, normalized[key]])) as Record; } -function readConfigRefRecord(ref: string): Record { - const target = readConfigRefTarget(ref); +function readConfigRefRecord(ref: string, context: WebProbeSentinelTemplateContext): Record { + const target = readConfigRefTarget(ref, context); if (!isRecord(target)) throw new Error(`${ref} must point to a YAML object`); return target; } -export function readConfigRefTarget(ref: string): unknown { +export function readConfigRefTarget(ref: string, context?: WebProbeSentinelTemplateContext): unknown { + if (context !== undefined) return readWebProbeSentinelConfigRefTarget(context, ref); const [file, path, extra] = ref.split("#"); if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) { throw new Error(`${ref} must use path/to/file.yaml#object.path syntax`); diff --git a/scripts/src/hwlab-node-web-sentinel-service.ts b/scripts/src/hwlab-node-web-sentinel-service.ts index 90793903..4e09dec7 100644 --- a/scripts/src/hwlab-node-web-sentinel-service.ts +++ b/scripts/src/hwlab-node-web-sentinel-service.ts @@ -7,10 +7,9 @@ // Responsibility: Persistent HTTP wrapper service for web-probe observe scheduling, index, health, metrics, maintenance, and dashboard. import { Buffer } from "node:buffer"; import { createHash, randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { Database } from "bun:sqlite"; -import { rootPath } from "./config"; import { renderWebProbeSentinelDashboardHtml, webProbeSentinelDashboardAssetResponse } from "./hwlab-node-web-sentinel-dashboard-assets"; import { webProbeSentinelConfigPlan, type WebProbeSentinelConfigPlan } from "./hwlab-node-web-sentinel-config"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; @@ -91,12 +90,12 @@ export interface WebProbeSentinelService { export function loadWebProbeSentinelServiceConfig(spec: HwlabRuntimeLaneSpec, options: Omit = {}): WebProbeSentinelServiceConfig { const sentinel = resolveWebProbeSentinel(spec, options.sentinelId ?? null); const plan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); - const runtime = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.runtime)); - const scenarios = scenarioArrayTarget(readSentinelConfigRefTarget(sentinel.configRefs.scenarios)); - const reportViews = resolveReportViewsWithCheckCatalog(recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.reportViews))); - const rawPublicExposure = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.publicExposure)); + const runtime = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.runtime, spec)); + const scenarios = scenarioArrayTarget(readSentinelConfigRefTarget(sentinel.configRefs.scenarios, spec)); + const reportViews = resolveReportViewsWithCheckCatalog(recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.reportViews, spec)), spec); + const rawPublicExposure = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.publicExposure, spec)); const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure); - const cicd = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.cicd)); + const cicd = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.cicd, spec)); const stateRoot = options.stateRootOverride ?? stringAt(runtime, "stateRoot"); const yamlSqlitePath = stringAt(runtime, "sqlite.path"); return { @@ -269,12 +268,12 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp return service; } -function resolveReportViewsWithCheckCatalog(reportViews: Record): Record { +function resolveReportViewsWithCheckCatalog(reportViews: Record, spec: HwlabRuntimeLaneSpec): Record { const ref = stringOrNull(reportViews.checkCatalogRef); if (ref === null) return reportViews; return { ...reportViews, - checkCatalog: recordTarget(readSentinelConfigRefTarget(ref)), + checkCatalog: recordTarget(readSentinelConfigRefTarget(ref, spec)), }; } @@ -694,13 +693,6 @@ function plannedRunBacklog(config: WebProbeSentinelServiceConfig, db: Database): }; } -function readConfigRefTarget(ref: string): unknown { - const [file, path] = ref.split("#"); - if (file === undefined || path === undefined) throw new Error(`invalid configRef: ${ref}`); - const text = readFileSync(rootPath(file), "utf8"); - return valueAtPath(Bun.YAML.parse(text) as unknown, path); -} - function writeMetadata(db: Database, key: string, value: unknown): void { db.query("INSERT INTO metadata (key, value_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at") .run(key, JSON.stringify(value), nowIso()); diff --git a/scripts/src/hwlab-node/control-actions.ts b/scripts/src/hwlab-node/control-actions.ts index c1fa2ab2..fcc446ec 100644 --- a/scripts/src/hwlab-node/control-actions.ts +++ b/scripts/src/hwlab-node/control-actions.ts @@ -468,6 +468,15 @@ export function nodeRuntimeSync(scoped: ReturnType \"$terminate_patch_file\"", + "printf '%s' \"$stale_operation_status_patch_b64\" | base64 -d > \"$stale_operation_status_patch_file\"", "printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"", "before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)", "operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)", "terminate_code=0", "terminate_output=", "terminated_operation=false", + "post_terminate_phase=", + "stale_status_patch_code=0", + "stale_status_patch_output=", + "patched_stale_operation_status=false", "if [ \"$operation_phase\" = Running ]; then", " if [ \"$dry_run\" = true ]; then", " terminated_operation=would-terminate", + " patched_stale_operation_status=would-patch", " else", " terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)", " terminate_code=$?", " if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi", " sleep 2", + " post_terminate_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)", + " if [ \"$terminate_code\" -eq 0 ] && [ \"$post_terminate_phase\" = Running ]; then", + " stale_status_patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$stale_operation_status_patch_file\" -o name 2>&1)", + " stale_status_patch_code=$?", + " if [ \"$stale_status_patch_code\" -eq 0 ]; then patched_stale_operation_status=true; else patched_stale_operation_status=failed; fi", + " sleep 1", + " fi", " fi", "fi", "failed_hook_count=0", @@ -570,6 +594,10 @@ export function nodeRuntimeSync(scoped: ReturnType { +export function nodeRuntimeGitMirrorJobManifest( + mirror: NodeRuntimeGitMirrorTargetSpec, + action: "sync" | "flush", + jobName: string, + options: { discardStaleGitops?: boolean } = {}, +): Record { const volumes: Record[] = [ { name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, { name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } }, @@ -88,6 +93,7 @@ export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTarg }, }, spec: { + ...(mirror.egressProxy.mode === "host-route" ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}), restartPolicy: "Never", volumes, containers: [{ @@ -95,7 +101,11 @@ export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTarg image: mirror.toolsImage, imagePullPolicy: mirror.toolsImagePullPolicy, command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"], - env: [...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror)], + env: [ + ...nodeRuntimeGitMirrorProxyEnv(mirror), + ...nodeRuntimeGitMirrorGithubTransportEnv(mirror), + ...(options.discardStaleGitops === true ? [{ name: "UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS", value: "true" }] : []), + ], volumeMounts, }], }, @@ -107,7 +117,7 @@ export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTarg export function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record[] { const proxy = mirror.egressProxy; if (proxy.mode === "direct") return []; - const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`; + const proxyUrl = proxy.mode === "host-route" ? proxy.proxyUrl : `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`; const noProxy = proxy.noProxy.join(","); return [ { name: "HTTP_PROXY", value: proxyUrl }, @@ -1101,7 +1111,7 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;", - "if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;", + "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", @@ -1186,7 +1196,7 @@ export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, s "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;", - "if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;", + "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", @@ -1283,6 +1293,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { " runtimeStore: overlay.runtimeStore,", " codeAgentRuntime: overlay.codeAgentRuntime,", " observability: overlay.observability,", + " deployYamlGitMirror: overlay.deployYamlGitMirror,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " dockerProxyHttp: overlay.dockerProxyHttp,", " dockerProxyHttps: overlay.dockerProxyHttps,", @@ -1332,6 +1343,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;", + "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;", "fs.writeFileSync(file, YAML.stringify(doc));", "console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));", "NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;", @@ -2200,6 +2212,117 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { " if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);", " fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", "}", + "function patchGitMirrorHostRouteYaml() {", + " const mirror = overlay.gitMirror || {};", + " const proxy = mirror.egressProxy || {};", + " if (proxy.mode !== 'host-route') return;", + " if (!YAML) throw new Error('yaml module is required to patch git-mirror host-route proxy');", + " const requireString = (pathName, value) => {", + " if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror host-route proxy');", + " return value;", + " };", + " const proxyUrl = requireString('gitMirror.egressProxy.proxyUrl', proxy.proxyUrl);", + " const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);", + " let proxyEndpoint;", + " try { proxyEndpoint = new URL(proxyUrl); } catch (error) { throw new Error(`overlay.gitMirror.egressProxy.proxyUrl is invalid: ${error.message}`); }", + " if (proxyEndpoint.protocol !== 'http:') throw new Error('overlay.gitMirror.egressProxy.proxyUrl must use http:// for host-route');", + " const proxyHost = proxyEndpoint.hostname;", + " const proxyPort = Number(proxyEndpoint.port || '80');", + " if (!proxyHost || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65535) throw new Error('overlay.gitMirror.egressProxy.proxyUrl must include a valid host and port');", + " const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.join(',') : '';", + " const envEntries = [", + " ['HTTP_PROXY', proxyUrl],", + " ['HTTPS_PROXY', proxyUrl],", + " ['ALL_PROXY', proxyUrl],", + " ['http_proxy', proxyUrl],", + " ['https_proxy', proxyUrl],", + " ['all_proxy', proxyUrl],", + " ['NO_PROXY', noProxy],", + " ['no_proxy', noProxy],", + " ];", + " function readDocs(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", + " function flattenDocs(docs) {", + " const manifests = [];", + " for (const doc of docs) {", + " if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);", + " else manifests.push(doc);", + " }", + " return manifests;", + " }", + " function setContainerEnv(container, name, value) {", + " if (!container || typeof container !== 'object') return false;", + " container.env = Array.isArray(container.env) ? container.env : [];", + " let item = container.env.find((env) => env && env.name === name);", + " if (!item) { item = { name }; container.env.push(item); }", + " const changed = item.value !== value || item.valueFrom !== undefined;", + " item.value = value;", + " delete item.valueFrom;", + " return changed;", + " }", + " function patchProxyScript(script, key) {", + " let next = String(script || '');", + " if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);", + " next = next.replace(/node \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) \\S+ \\d+/g, `node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort}`);", + " const missing = [];", + " for (const [name, value] of envEntries) {", + " let replaced = false;", + " const exportPattern = new RegExp(`(^|\\\\n)([ \\\\t]*export ${escapeRegExp(name)}=)[^\\\\n]*`, 'g');", + " next = next.replace(exportPattern, (_match, prefix, left) => { replaced = true; return `${prefix}${left}${shellSingle(value)}`; });", + " if (!replaced) missing.push([name, value]);", + " }", + " if (missing.length > 0) {", + " const block = missing.map(([name, value]) => `export ${name}=${shellSingle(value)}`).join('\\\\n');", + " if (next.includes('\\\\nset -eu\\\\n')) next = next.replace('\\\\nset -eu\\\\n', `\\\\nset -eu\\\\n${block}\\\\n`);", + " else next = `${block}\\\\n${next}`;", + " }", + " next = next.replace(/mode=node-global/g, 'mode=host-route');", + " return next;", + " }", + " const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');", + " if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);", + " const docs = readDocs(gitMirrorFile);", + " const manifests = flattenDocs(docs);", + " let configMapChanged = false;", + " let workloadChanged = false;", + " const workloadNames = new Set([mirror.serviceReadName, mirror.serviceWriteName].filter((value) => typeof value === 'string' && value.length > 0));", + " for (const doc of manifests) {", + " if (!doc || typeof doc !== 'object') continue;", + " if (doc.kind === 'ConfigMap' && doc.metadata && doc.metadata.name === configMapName) {", + " doc.data = doc.data || {};", + " doc.data['sync.sh'] = patchProxyScript(doc.data['sync.sh'], 'sync.sh');", + " doc.data['flush.sh'] = patchProxyScript(doc.data['flush.sh'], 'flush.sh');", + " configMapChanged = true;", + " continue;", + " }", + " if (!['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(doc.kind)) continue;", + " const name = doc.metadata && doc.metadata.name ? String(doc.metadata.name) : '';", + " const labels = doc.metadata && doc.metadata.labels && typeof doc.metadata.labels === 'object' ? doc.metadata.labels : {};", + " if (!workloadNames.has(name) && labels['app.kubernetes.io/name'] !== 'git-mirror' && !name.includes('git-mirror')) continue;", + " doc.spec = doc.spec || {};", + " doc.spec.template = doc.spec.template || {};", + " doc.spec.template.metadata = doc.spec.template.metadata || {};", + " doc.spec.template.metadata.annotations = doc.spec.template.metadata.annotations || {};", + " doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-egress-proxy'] = 'host-route';", + " doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-host-proxy-config-ref'] = String(proxy.hostProxyConfigRef || '');", + " doc.spec.template.spec = doc.spec.template.spec || {};", + " if (proxy.podHostNetwork) {", + " doc.spec.template.spec.hostNetwork = true;", + " doc.spec.template.spec.dnsPolicy = 'ClusterFirstWithHostNet';", + " } else {", + " delete doc.spec.template.spec.hostNetwork;", + " delete doc.spec.template.spec.dnsPolicy;", + " }", + " for (const group of ['containers', 'initContainers']) {", + " for (const container of Array.isArray(doc.spec.template.spec[group]) ? doc.spec.template.spec[group] : []) {", + " for (const [envName, envValue] of envEntries) setContainerEnv(container, envName, envValue);", + " }", + " }", + " workloadChanged = true;", + " }", + " if (!configMapChanged) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);", + " if (!workloadChanged) throw new Error(`generated git-mirror workload for host-route proxy was not found in ${gitMirrorFile}`);", + " fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", + "}", "const structured = patchStructuredPipeline();", "function replaceParamDefault(name, value) {", " const namePattern = escapeRegExp(name);", @@ -2253,6 +2376,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "if (text.includes('/yaml/-/yaml-') || text.includes('bun add --no-save --ignore-scripts') || text.includes('npm install --package-lock=false --no-save')) { throw new Error(`generated pipeline still downloads yaml during prepare-source in ${pipelinePath}`); }", "if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }", "fs.writeFileSync(pipelinePath, text);", + "patchGitMirrorHostRouteYaml();", "patchGitMirrorTransportYaml();", "function patchArgoYaml(filePath) {", " if (!YAML || !fs.existsSync(filePath)) return;", diff --git a/scripts/src/hwlab-node/runtime-common.ts b/scripts/src/hwlab-node/runtime-common.ts index 70f9510b..0fc2ec67 100644 --- a/scripts/src/hwlab-node/runtime-common.ts +++ b/scripts/src/hwlab-node/runtime-common.ts @@ -292,28 +292,48 @@ export function runNodeHostScriptAsync(spec: HwlabRuntimeLaneSpec, script: strin return commandResultFromAsync(spec, payload, statusPath, false); } } - const kill = runNodeHostScript(spec, [ + const pending = runNodeHostScript(spec, [ "set +e", `status_path=${shellQuote(statusPath)}`, - "pid=$(python3 - \"$status_path\" <<'PY' 2>/dev/null", - "import json, sys", - "try: print(json.load(open(sys.argv[1])).get('pid') or '')", - "except Exception: print('')", + `stdout_path=${shellQuote(stdoutPath)}`, + `stderr_path=${shellQuote(stderrPath)}`, + `timeout_seconds=${shellQuote(String(timeoutSeconds))}`, + "python3 - \"$status_path\" \"$stdout_path\" \"$stderr_path\" \"$timeout_seconds\" <<'PY'", + "import datetime, json, pathlib, subprocess, sys", + "status_path, stdout_path, stderr_path, timeout_seconds = sys.argv[1:5]", + "try:", + " payload = json.load(open(status_path))", + "except Exception:", + " payload = {'state': 'missing', 'ok': False, 'exitCode': 127}", + "def tail(path):", + " try:", + " return pathlib.Path(path).read_text(errors='replace')[-12000:]", + " except FileNotFoundError:", + " return ''", + "pid = payload.get('pid')", + "pid_alive = None", + "if isinstance(pid, int) and pid > 0:", + " pid_alive = subprocess.run(['kill', '-0', str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0", + "payload['stdout'] = tail(stdout_path)", + "stderr = tail(stderr_path)", + "pending_note = f\"remote async command still pending after {timeout_seconds}s; statusPath={status_path}; pidAlive={pid_alive}\"", + "payload['stderr'] = '\\n'.join([item for item in [pending_note, stderr] if item])", + "payload['timedOutAt'] = datetime.datetime.utcnow().isoformat() + 'Z'", + "payload['pidAlive'] = pid_alive", + "json.dump(payload, sys.stdout, indent=2)", "PY", - ")", - "if [ -n \"$pid\" ]; then kill \"$pid\" 2>/dev/null || true; fi", - "cat \"$status_path\" 2>/dev/null || true", ].join("\n"), 55); + payload = parseJsonObject(pending.stdout.trim()); return { command: [transPath(), spec.nodeRoute, "sh", "--", ""], cwd: repoRoot, exitCode: 124, - stdout: typeof payload.stdout === "string" ? payload.stdout : "", + stdout: typeof payload.stdout === "string" ? payload.stdout : pending.stdout, stderr: [ - `remote async command timed out after ${timeoutSeconds}s; statusPath=${statusPath}`, + `remote async command pending after ${timeoutSeconds}s; statusPath=${statusPath}`, typeof payload.stderr === "string" ? payload.stderr : "", lastStatus?.stderr.trim() ?? "", - kill.stderr.trim(), + pending.stderr.trim(), ].filter(Boolean).join("\n"), signal: null, timedOut: true, diff --git a/scripts/src/hwlab-node/secret-scripts.ts b/scripts/src/hwlab-node/secret-scripts.ts index 8e50efc8..38d82909 100644 --- a/scripts/src/hwlab-node/secret-scripts.ts +++ b/scripts/src/hwlab-node/secret-scripts.ts @@ -1051,10 +1051,15 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run `namespace=${shellQuote(spec.namespace)}`, `name=${shellQuote(spec.bootstrapAdminSecret)}`, `password_hash_key=${shellQuote(spec.bootstrapAdminPasswordHashKey)}`, - `username=${shellQuote(spec.bootstrapAdminUsername)}`, + `username=${shellQuote(material?.username ?? spec.bootstrapAdminUsername)}`, `display_name=${shellQuote(spec.bootstrapAdminDisplayName)}`, + `username_source_ref=${shellQuote(material?.usernameSourceRef ?? "")}`, + `username_source_key=${shellQuote(material?.usernameSourceKey ?? "")}`, + `username_source_line=${shellQuote(material?.usernameSourceLine === null || material?.usernameSourceLine === undefined ? "" : String(material.usernameSourceLine))}`, + `username_source_fingerprint=${shellQuote(material?.usernameFingerprint ?? "")}`, `source_ref=${shellQuote(spec.bootstrapAdminPasswordSourceRef ?? "")}`, `source_key=${shellQuote(spec.bootstrapAdminPasswordSourceKey ?? "")}`, + `source_line=${shellQuote(material?.sourceLine === null || material?.sourceLine === undefined ? "" : String(material.sourceLine))}`, `source_path=${shellQuote(material?.sourcePath === null || material?.sourcePath === undefined ? "" : displayRepoPath(material.sourcePath))}`, `source_present=${shellQuote(material?.sourcePresent === true ? "yes" : "no")}`, `source_fingerprint=${shellQuote(material?.sourceFingerprint ?? "")}`, @@ -1075,8 +1080,13 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run "before_hash_b64=$(secret_b64_key \"$password_hash_key\")", "before_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-ref)", "before_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-key)", + "before_source_line=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-line)", "before_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-fingerprint)", "before_username=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username)", + "before_username_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-ref)", + "before_username_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-key)", + "before_username_source_line=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-line)", + "before_username_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-fingerprint)", "before_hash_present=$([ -n \"$before_hash_b64\" ] && printf yes || printf no)", "before_hash_bytes=$(decoded_length \"$before_hash_b64\")", "action=observed", @@ -1087,7 +1097,8 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run "if [ \"$action_request\" = ensure ]; then", " needs_sync=false", " [ \"$before_exists\" = yes ] && [ \"$before_hash_bytes\" -gt 0 ] || needs_sync=true", - " [ \"$before_source_ref\" = \"$source_ref\" ] && [ \"$before_source_key\" = \"$source_key\" ] && [ \"$before_source_fingerprint\" = \"$source_fingerprint\" ] && [ \"$before_username\" = \"$username\" ] || needs_sync=true", + " [ \"$before_source_ref\" = \"$source_ref\" ] && [ \"$before_source_key\" = \"$source_key\" ] && [ \"$before_source_line\" = \"$source_line\" ] && [ \"$before_source_fingerprint\" = \"$source_fingerprint\" ] && [ \"$before_username\" = \"$username\" ] || needs_sync=true", + " [ \"$before_username_source_ref\" = \"$username_source_ref\" ] && [ \"$before_username_source_key\" = \"$username_source_key\" ] && [ \"$before_username_source_line\" = \"$username_source_line\" ] && [ \"$before_username_source_fingerprint\" = \"$username_source_fingerprint\" ] || needs_sync=true", " [ \"$force_sync\" = true ] && needs_sync=true", " if [ \"$material_ok\" != true ]; then", " action=${source_error:-secret-source-invalid}", @@ -1109,8 +1120,13 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run " annotations:", " hwlab.pikastech.local/bootstrap-admin-username: \"$username\"", " hwlab.pikastech.local/bootstrap-admin-display-name: \"$display_name\"", + " hwlab.pikastech.local/bootstrap-admin-username-source-ref: \"$username_source_ref\"", + " hwlab.pikastech.local/bootstrap-admin-username-source-key: \"$username_source_key\"", + " hwlab.pikastech.local/bootstrap-admin-username-source-line: \"$username_source_line\"", + " hwlab.pikastech.local/bootstrap-admin-username-source-fingerprint: \"$username_source_fingerprint\"", " hwlab.pikastech.local/bootstrap-admin-source-ref: \"$source_ref\"", " hwlab.pikastech.local/bootstrap-admin-source-key: \"$source_key\"", + " hwlab.pikastech.local/bootstrap-admin-source-line: \"$source_line\"", " hwlab.pikastech.local/bootstrap-admin-source-fingerprint: \"$source_fingerprint\"", " hwlab.pikastech.local/bootstrap-admin-password-transform: \"$transform\"", " labels:", @@ -1140,8 +1156,13 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run "after_hash_b64=$(secret_b64_key \"$password_hash_key\")", "after_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-ref)", "after_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-key)", + "after_source_line=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-line)", "after_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-source-fingerprint)", "after_username=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username)", + "after_username_source_ref=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-ref)", + "after_username_source_key=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-key)", + "after_username_source_line=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-line)", + "after_username_source_fingerprint=$(secret_annotation hwlab.pikastech.local/bootstrap-admin-username-source-fingerprint)", "after_hash_present=$([ -n \"$after_hash_b64\" ] && printf yes || printf no)", "after_hash_bytes=$(decoded_length \"$after_hash_b64\")", "printf 'namespace\\t%s\\n' \"$namespace\"", @@ -1150,8 +1171,13 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run "printf 'preset\\t%s\\n' \"$preset\"", "printf 'username\\t%s\\n' \"$username\"", "printf 'displayName\\t%s\\n' \"$display_name\"", + "printf 'usernameSourceRef\\t%s\\n' \"$username_source_ref\"", + "printf 'usernameSourceKey\\t%s\\n' \"$username_source_key\"", + "printf 'usernameSourceLine\\t%s\\n' \"$username_source_line\"", + "printf 'usernameSourceFingerprint\\t%s\\n' \"$username_source_fingerprint\"", "printf 'sourceRef\\t%s\\n' \"$source_ref\"", "printf 'sourceKey\\t%s\\n' \"$source_key\"", + "printf 'sourceLine\\t%s\\n' \"$source_line\"", "printf 'sourcePath\\t%s\\n' \"$source_path\"", "printf 'sourceExists\\t%s\\n' \"$source_present\"", "printf 'sourceFingerprint\\t%s\\n' \"$source_fingerprint\"", @@ -1165,15 +1191,25 @@ export function bootstrapAdminSecretScript(options: NodeSecretOptions, spec: Run "printf 'beforePasswordHashBytes\\t%s\\n' \"$before_hash_bytes\"", "printf 'beforeSourceRef\\t%s\\n' \"$before_source_ref\"", "printf 'beforeSourceKey\\t%s\\n' \"$before_source_key\"", + "printf 'beforeSourceLine\\t%s\\n' \"$before_source_line\"", "printf 'beforeSourceFingerprint\\t%s\\n' \"$before_source_fingerprint\"", "printf 'beforeUsername\\t%s\\n' \"$before_username\"", + "printf 'beforeUsernameSourceRef\\t%s\\n' \"$before_username_source_ref\"", + "printf 'beforeUsernameSourceKey\\t%s\\n' \"$before_username_source_key\"", + "printf 'beforeUsernameSourceLine\\t%s\\n' \"$before_username_source_line\"", + "printf 'beforeUsernameSourceFingerprint\\t%s\\n' \"$before_username_source_fingerprint\"", "printf 'afterExists\\t%s\\n' \"$after_exists\"", "printf 'afterPasswordHashPresent\\t%s\\n' \"$after_hash_present\"", "printf 'afterPasswordHashBytes\\t%s\\n' \"$after_hash_bytes\"", "printf 'afterSourceRef\\t%s\\n' \"$after_source_ref\"", "printf 'afterSourceKey\\t%s\\n' \"$after_source_key\"", + "printf 'afterSourceLine\\t%s\\n' \"$after_source_line\"", "printf 'afterSourceFingerprint\\t%s\\n' \"$after_source_fingerprint\"", "printf 'afterUsername\\t%s\\n' \"$after_username\"", + "printf 'afterUsernameSourceRef\\t%s\\n' \"$after_username_source_ref\"", + "printf 'afterUsernameSourceKey\\t%s\\n' \"$after_username_source_key\"", + "printf 'afterUsernameSourceLine\\t%s\\n' \"$after_username_source_line\"", + "printf 'afterUsernameSourceFingerprint\\t%s\\n' \"$after_username_source_fingerprint\"", "printf 'cloudApiDeployment\\t%s\\n' \"$cloud_api_deployment\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", @@ -1759,8 +1795,13 @@ export function secretStatusFromText(text: string, commandOk: boolean, exitCode: fields.sourceFingerprint.length > 0 && fields.afterSourceRef === fields.sourceRef && fields.afterSourceKey === fields.sourceKey && + (fields.afterSourceLine || "") === (fields.sourceLine || "") && fields.afterSourceFingerprint === fields.sourceFingerprint && - fields.afterUsername === fields.username + fields.afterUsername === fields.username && + (fields.afterUsernameSourceRef || "") === (fields.usernameSourceRef || "") && + (fields.afterUsernameSourceKey || "") === (fields.usernameSourceKey || "") && + (fields.afterUsernameSourceLine || "") === (fields.usernameSourceLine || "") && + (fields.afterUsernameSourceFingerprint || "") === (fields.usernameSourceFingerprint || "") ); const healthy = targetHashReady && yamlSourceReady; return { @@ -1777,6 +1818,7 @@ export function secretStatusFromText(text: string, commandOk: boolean, exitCode: ? { sourceRef: fields.sourceRef, sourceKey: fields.sourceKey || null, + sourceLine: numericField(fields.sourceLine), sourcePath: fields.sourcePath || null, exists: fields.sourceExists === "yes", fingerprint: fields.sourceFingerprint || null, @@ -1799,8 +1841,13 @@ export function secretStatusFromText(text: string, commandOk: boolean, exitCode: ...(yamlSourceMode ? { sourceRef: fields.beforeSourceRef || null, sourceKey: fields.beforeSourceKey || null, + sourceLine: numericField(fields.beforeSourceLine), sourceFingerprint: fields.beforeSourceFingerprint || null, username: fields.beforeUsername || null, + usernameSourceRef: fields.beforeUsernameSourceRef || null, + usernameSourceKey: fields.beforeUsernameSourceKey || null, + usernameSourceLine: numericField(fields.beforeUsernameSourceLine), + usernameSourceFingerprint: fields.beforeUsernameSourceFingerprint || null, } : {}), }, after: { @@ -1809,8 +1856,13 @@ export function secretStatusFromText(text: string, commandOk: boolean, exitCode: ...(yamlSourceMode ? { sourceRef: fields.afterSourceRef || null, sourceKey: fields.afterSourceKey || null, + sourceLine: numericField(fields.afterSourceLine), sourceFingerprint: fields.afterSourceFingerprint || null, username: fields.afterUsername || null, + usernameSourceRef: fields.afterUsernameSourceRef || null, + usernameSourceKey: fields.afterUsernameSourceKey || null, + usernameSourceLine: numericField(fields.afterUsernameSourceLine), + usernameSourceFingerprint: fields.afterUsernameSourceFingerprint || null, } : {}), }, cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, diff --git a/scripts/src/hwlab-node/status.ts b/scripts/src/hwlab-node/status.ts index 4806a7d3..1fec2a31 100644 --- a/scripts/src/hwlab-node/status.ts +++ b/scripts/src/hwlab-node/status.ts @@ -117,7 +117,14 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType/tmp/hwlab-node-gitmirror-status.err || true)", - "if ! printf '%s' \"$summary_json\" | node -e 'let s=\"\"; process.stdin.on(\"data\", c => s += c); process.stdin.on(\"end\", () => { try { const o = JSON.parse(s || \"{}\"); process.exit(o && o.localSource ? 0 : 1); } catch { process.exit(1); } });'; then", + "if ! SUMMARY_JSON=\"$summary_json\" python3 - <<'PY'; then", + "import json, os, sys", + "try:", + " value = json.loads(os.environ.get('SUMMARY_JSON') or '{}')", + "except Exception:", + " value = {}", + "sys.exit(0 if isinstance(value, dict) and value.get('localSource') else 1)", + "PY", " summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 gitops_branch=\\$3 node <<'NODE'", "const { execFileSync } = require('node:child_process');", "const { readFileSync, existsSync } = require('node:fs');", @@ -161,29 +168,37 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType/dev/null 2>&1; then cache_host_path_exists=true; fi", - "SUMMARY_JSON=\"$summary_json\" GITHUB_TRANSPORT_JSON=\"$github_transport_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" node <<'NODE'", - "const summary = (() => { try { return JSON.parse(process.env.SUMMARY_JSON || '{}'); } catch { return {}; } })();", - "const githubTransport = (() => { try { return JSON.parse(process.env.GITHUB_TRANSPORT_JSON || '{}'); } catch { return {}; } })();", - "const env = process.env;", - "const ok = env.read_deployment_ready === 'true' && env.write_deployment_ready === 'true' && env.read_service_exists === 'true' && env.write_service_exists === 'true' && env.read_endpoints_ready === 'true' && env.write_endpoints_ready === 'true' && (env.cache_pvc_exists === 'true' || env.cache_host_path_exists === 'true') && githubTransport.ready !== false && summary.localSource;", - "console.log(JSON.stringify({", - " ok: Boolean(ok),", - " resources: {", - " readDeploymentReady: env.read_deployment_ready === 'true',", - " writeDeploymentReady: env.write_deployment_ready === 'true',", - " readServiceExists: env.read_service_exists === 'true',", - " writeServiceExists: env.write_service_exists === 'true',", - " readEndpointsReady: env.read_endpoints_ready === 'true',", - " writeEndpointsReady: env.write_endpoints_ready === 'true',", - " cachePvcExists: env.cache_pvc_exists === 'true',", - " cacheHostPathConfigured: Boolean(env.cache_host_path),", - " cacheHostPathExists: env.cache_host_path_exists === 'true'", - " },", - " githubTransport,", - " summary,", - " valuesPrinted: false", - "}));", - "NODE", + "SUMMARY_JSON=\"$summary_json\" GITHUB_TRANSPORT_JSON=\"$github_transport_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" python3 - <<'PY'", + "import json, os", + "def load_env_json(name):", + " try:", + " value = json.loads(os.environ.get(name) or '{}')", + " return value if isinstance(value, dict) else {}", + " except Exception:", + " return {}", + "def truth(name):", + " return os.environ.get(name) == 'true'", + "summary = load_env_json('SUMMARY_JSON')", + "github_transport = load_env_json('GITHUB_TRANSPORT_JSON')", + "ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource'))", + "print(json.dumps({", + " 'ok': bool(ok),", + " 'resources': {", + " 'readDeploymentReady': truth('read_deployment_ready'),", + " 'writeDeploymentReady': truth('write_deployment_ready'),", + " 'readServiceExists': truth('read_service_exists'),", + " 'writeServiceExists': truth('write_service_exists'),", + " 'readEndpointsReady': truth('read_endpoints_ready'),", + " 'writeEndpointsReady': truth('write_endpoints_ready'),", + " 'cachePvcExists': truth('cache_pvc_exists'),", + " 'cacheHostPathConfigured': bool(os.environ.get('cache_host_path')),", + " 'cacheHostPathExists': truth('cache_host_path_exists'),", + " },", + " 'githubTransport': github_transport,", + " 'summary': summary,", + " 'valuesPrinted': False,", + "}))", + "PY", ].join("\n"); const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); const parsed = record(parseJsonObject(statusText(result))); @@ -240,7 +255,9 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType 0 ? compactNodeRuntimeGitMirrorStatus(after) : {}; const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit; @@ -633,7 +658,6 @@ export function nodeRuntimeGitMirrorNeedsFlush(status: Record): const githubGitops = typeof summary.githubGitops === "string" ? summary.githubGitops : null; return summary.pendingFlush === true || summary.flushNeeded === true - || summary.githubInSync === false || (localGitops !== null && githubGitops !== null && localGitops !== githubGitops); } diff --git a/scripts/src/hwlab-node/web-observe-scripts.ts b/scripts/src/hwlab-node/web-observe-scripts.ts index 5392e25c..622567cc 100644 --- a/scripts/src/hwlab-node/web-observe-scripts.ts +++ b/scripts/src/hwlab-node/web-observe-scripts.ts @@ -470,7 +470,7 @@ export function runNodeWebProbeScript( credential: Record, ): Record { const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode); - const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "", webProbeProxy); + const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const commandTimedOut = result.timedOut || result.exitCode === 124; const stdoutReport = parseJsonObject(result.stdout); @@ -606,7 +606,7 @@ function webProbeScriptPreferredCommands(options: NodeWebProbeScriptOptions): Re }; } -export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string, webProbeProxy: NodeWebProbeHostProxyEnv): string { +export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv): string { const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64"); const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64"); return [ @@ -625,7 +625,7 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions [ ...webProbeProxy.envAssignments, `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, - `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, + `HWLAB_WEB_USER=${shellQuote(username)}`, `HWLAB_WEB_PASS=${shellQuote(password)}`, "UNIDESK_WEB_PROBE_RUN_DIR=\"$run_dir\"", "UNIDESK_WEB_PROBE_USER_SCRIPT=\"$user_script\"", diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index c7faf4af..fab63d84 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -582,7 +582,7 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record { return { - username: secretSpec.bootstrapAdminUsername, + username: material.username === null ? secretSpec.bootstrapAdminUsername : "", + usernameSourceRef: material.usernameSourceRef, + usernameSourceKey: material.usernameSourceKey, + usernameSourceLine: material.usernameSourceLine, + usernameFingerprint: material.usernameFingerprint, sourceRef: material.sourceRef, sourceKey: material.sourceKey, sourcePath: material.sourcePath === null ? null : displayRepoPath(material.sourcePath), @@ -1310,7 +1314,7 @@ export function runNodeWebProbeObserveStart( ...webProbeProxy.envAssignments, ...webProbeAccountEnvAssignments(), `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, - `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, + `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`, `UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`, `UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`, diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index fab8ffe7..289ae647 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -49,12 +49,28 @@ export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record= 0 @@ -1034,6 +1052,16 @@ export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, " if [ \"$mode\" = build-moonbridge ]; then", " action=build", " source_present_before=build-source", + " effective_build_container_http_proxy=\"$build_container_http_proxy\"", + " effective_build_container_https_proxy=\"$build_container_https_proxy\"", + " effective_build_container_all_proxy=\"$build_container_all_proxy\"", + " effective_docker_build_add_host_args=\"$docker_build_add_host_args\"", + " if [ \"$docker_network_mode\" = host ]; then", + " effective_build_container_http_proxy=\"$build_http_proxy\"", + " effective_build_container_https_proxy=\"$build_https_proxy\"", + " effective_build_container_all_proxy=\"$build_all_proxy\"", + " effective_docker_build_add_host_args=\"\"", + " fi", " tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)", " dockerfile=\"$tmpdir/Dockerfile\"", " cat > \"$dockerfile\" <<'DOCKERFILE'", @@ -1069,7 +1097,7 @@ export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, "USER 65532:65532", "ENTRYPOINT [\"/app/moonbridge\"]", "DOCKERFILE", - " if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$build_container_https_proxy\" --build-arg ALL_PROXY=\"$build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$build_container_http_proxy\" --build-arg https_proxy=\"$build_container_https_proxy\" --build-arg all_proxy=\"$build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then", + " if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then", " docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }", " else", " cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true", @@ -1122,9 +1150,12 @@ export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean, ? runNodeHostScriptAsync(spec, script, Math.min(timeoutSeconds, 600), "runtime-image-build") : runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300)); const imageRows = parseNodeRuntimeImageRows(statusText(result)); + const dependencyCount = nodeRuntimeImageDependencies(spec).length; + const imageRowsComplete = imageRows.length === dependencyCount + && imageRows.every((image) => image.presentAfter === true || image.registryTagPresent === true); const base = imageRows.find((image) => image.id === "base") ?? {}; return { - ok: isCommandSuccess(result), + ok: isCommandSuccess(result) || imageRowsComplete, dryRun, target: base.target ?? spec.baseImage, source: base.source ?? spec.baseImageSource, @@ -1196,6 +1227,7 @@ export function readSecretSourceValue(secretRoot: string, sourceRef: string, key } export function secretSourcePaths(sourceRef: string): string[] { + if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; const marker = "/.worktree/"; const index = repoRoot.indexOf(marker); @@ -1203,6 +1235,14 @@ export function secretSourcePaths(sourceRef: string): string[] { return [...new Set(paths)]; } +function ownerFileSourcePaths(sourceRef: string): string[] { + if (sourceRef.includes("..") || sourceRef.includes("\0")) return []; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot]; + return [...new Set(roots.map((root) => join(root, sourceRef)))]; +} + export function displayRepoPath(path: string): string { const normalizedRoot = repoRoot.replace(/\/+$/u, ""); if (path === normalizedRoot) return "."; @@ -1225,27 +1265,28 @@ export function hwlabPasswordHash(password: string): string { export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial { const sourceRef = spec.bootstrapAdminPasswordSourceRef; const sourceKey = spec.bootstrapAdminPasswordSourceKey; + const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null; if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { - return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" }; - } - const paths = secretSourcePaths(sourceRef); - const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); - if (!existsSync(sourcePath)) { - return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" }; - } - const values = parseEnvFile(readFileSync(sourcePath, "utf8")); - const password = values[sourceKey]; - if (password === undefined || password.length === 0) { - return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" }; + return bootstrapAdminSecretMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing"); } + const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine); + if (!password.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error); + const username = readBootstrapAdminUsername(spec); + if (!username.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error); return { ok: true, + username: username.value, + usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, + usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, + usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, + usernameFingerprint: shortSecretFingerprint(username.value), sourceRef, sourceKey, - sourcePath, + sourceLine, + sourcePath: password.sourcePath, sourcePresent: true, - sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`, - passwordHash: hwlabPasswordHash(password), + sourceFingerprint: shortSecretFingerprint(password.value), + passwordHash: hwlabPasswordHash(password.value), error: null, }; } @@ -1253,32 +1294,96 @@ export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): Boots export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial { const sourceRef = spec.bootstrapAdminPasswordSourceRef; const sourceKey = spec.bootstrapAdminPasswordSourceKey; + const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null; if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) { - return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" }; - } - const paths = secretSourcePaths(sourceRef); - const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); - const runtimePassword = process.env[sourceKey]; - if (!existsSync(sourcePath) && (runtimePassword === undefined || runtimePassword.length === 0)) { - return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" }; - } - const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; - const password = values[sourceKey] ?? runtimePassword; - if (password === undefined || password.length === 0) { - return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" }; + return bootstrapAdminPasswordMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing"); } + const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine); + if (!password.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error); + const username = readBootstrapAdminUsername(spec); + if (!username.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error); return { ok: true, + username: username.value, + usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, + usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, + usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, + usernameFingerprint: shortSecretFingerprint(username.value), sourceRef, sourceKey, - sourcePath, + sourceLine, + sourcePath: password.sourcePath, sourcePresent: true, - sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`, - password, + sourceFingerprint: shortSecretFingerprint(password.value), + password: password.value, error: null, }; } +function readBootstrapAdminUsername(spec: RuntimeSecretSpec): { ok: true; value: string } | { ok: false; error: string } { + const ref = spec.bootstrapAdminUsernameSourceRef; + if (ref === undefined) return { ok: true, value: spec.bootstrapAdminUsername }; + const key = spec.bootstrapAdminUsernameSourceKey ?? "username"; + const material = readSecretSourceScalar(ref, key, spec.bootstrapAdminUsernameSourceLine ?? null); + if (!material.ok) return { ok: false, error: `username-${material.error}` }; + return { ok: true, value: material.value }; +} + +function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: true } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } { + const paths = secretSourcePaths(sourceRef); + const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); + if (!existsSync(sourcePath)) return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" }; + const text = readFileSync(sourcePath, "utf8"); + if (sourceLine !== null) { + const line = text.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? ""; + if (line.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-line-missing" }; + return { ok: true, value: line, sourcePath, sourcePresent: true }; + } + const values = parseEnvFile(text); + const runtimeValue = process.env[sourceKey]; + const value = values[sourceKey] ?? runtimeValue; + if (value === undefined || value.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-key-missing" }; + return { ok: true, value, sourcePath, sourcePresent: true }; +} + +function bootstrapAdminSecretMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminSecretMaterial { + return { + ok: false, + username: null, + usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, + usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, + usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, + usernameFingerprint: null, + sourceRef, + sourceKey, + sourceLine, + sourcePath, + sourcePresent: sourcePresent === true, + sourceFingerprint: null, + passwordHash: null, + error, + }; +} + +function bootstrapAdminPasswordMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminPasswordMaterial { + return { + ok: false, + username: null, + usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null, + usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null, + usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null, + usernameFingerprint: null, + sourceRef, + sourceKey, + sourceLine, + sourcePath, + sourcePresent: sourcePresent === true, + sourceFingerprint: null, + password: null, + error, + }; +} + export function parseEnvFile(text: string): Record { const values: Record = {}; for (const rawLine of text.split(/\r?\n/u)) { @@ -1547,11 +1652,13 @@ export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRunt const gitMirror = record(target.gitMirror); const gitMirrorEgressProxy = record(gitMirror.egressProxy); const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode"); - if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`); + if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "host-route" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global, host-route, or direct for node=${spec.nodeId} lane=${spec.lane}`); const nodeEgressProxy = gitMirrorEgressMode === "direct" ? { mode: "direct" as const, required: false as const } : nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`); - if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); + if ((gitMirrorEgressMode === "node-global" || gitMirrorEgressMode === "host-route") && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); + if (gitMirrorEgressMode === "node-global" && nodeEgressProxy.mode !== "k8s-service-cluster-ip") throw new Error(`gitMirror.egressProxy.mode=node-global requires nodes.${spec.nodeId}.egressProxy.mode=k8s-service-cluster-ip`); + if (gitMirrorEgressMode === "host-route" && nodeEgressProxy.mode !== "host-route") throw new Error(`gitMirror.egressProxy.mode=host-route requires nodes.${spec.nodeId}.egressProxy.mode=host-route`); const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport"); const source = record(target.source); const gitops = record(target.gitops); @@ -1626,7 +1733,19 @@ function gitMirrorSecretSourceEncoding(raw: unknown, path: string): "plain" | "b export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record, path: string): NodeRuntimeGitMirrorEgressProxySpec { const mode = stringValue(raw.mode, `${path}.mode`); - if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`); + if (mode === "host-route") { + const noProxyRaw = raw.noProxy; + if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`); + return { + mode, + clientName: stringValue(raw.clientName, `${path}.clientName`), + hostProxyConfigRef: stringValue(raw.hostProxyConfigRef, `${path}.hostProxyConfigRef`), + proxyEnvPath: stringValue(raw.proxyEnvPath, `${path}.proxyEnvPath`), + proxyUrl: stringValue(raw.proxyUrl, `${path}.proxyUrl`), + noProxy: noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)), + }; + } + if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`); const port = Number(raw.port); if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`); const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null; diff --git a/scripts/src/platform-infra-egress-proxy.ts b/scripts/src/platform-infra-egress-proxy.ts index 83357aaa..61f6912b 100644 --- a/scripts/src/platform-infra-egress-proxy.ts +++ b/scripts/src/platform-infra-egress-proxy.ts @@ -34,6 +34,10 @@ interface TrafficOptions { type EgressProxyOptions = BenchmarkOptions | TrafficOptions; export async function runPlatformInfraEgressProxyCommand(_config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { + if (args[0] === "host") { + const { runPlatformInfraHostProxyCommand } = await import("./platform-infra-host-proxy"); + return await runPlatformInfraHostProxyCommand(_config, args.slice(1)); + } if (args[0] === "k3s-build-benchmark") return runK3sBuildBenchmarkCommand(args.slice(1)); const options = parseEgressProxyOptions(args); if (options.action === "traffic") { diff --git a/scripts/src/platform-infra-host-proxy.ts b/scripts/src/platform-infra-host-proxy.ts new file mode 100644 index 00000000..800c1855 --- /dev/null +++ b/scripts/src/platform-infra-host-proxy.ts @@ -0,0 +1,1154 @@ +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import type { UniDeskConfig } from "./config"; +import { rootPath } from "./config"; +import { runCommand, type CommandResult } from "./command"; +import { resolveEgressProxySourceRef, type MasterShadowsocksSourceSpec } from "./egress-proxy-sources"; +import type { RenderedCliResult } from "./output"; +import { shQuote } from "./platform-infra-public-service"; +import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; + +const configPath = "config/platform-infra/host-proxy.yaml"; + +interface HostProxyConfig { + defaults: { targetId: string }; + server: HostProxyServer; + sources: Record; + targets: Record; +} + +interface HostProxyServer { + id: string; + enabled: boolean; + benchmarkRef: string; + implementationRef: string; + sourceConfigRef: string; + sourceRef: string; + sourceKey: string; + sourceFingerprint: string; + masterShadowsocks: MasterShadowsocksSourceSpec; + composeFile: string; + serviceName: string; + containerName: string; + image: string; + configPath: string; + listenHost: string; + listenPort: number; + health: { mode: "tcp"; host: string; port: number }; + fingerprint: string; +} + +interface HostProxySource { + id: string; + sourceType: "benchmark-validated-master-shadowsocks"; + serverRef: string; + benchmarkRef: string; + implementationRef: string; + sourceConfigRef: string; + sourceRef: string; + sourceKey: string; + sourceFingerprint: string; + masterShadowsocks: MasterShadowsocksSourceSpec; + client: HostProxyClient; + proxyUrl: string; + externalProbeUrl: string; + fingerprint: string; +} + +interface HostProxyClient { + mode: "trans-static-binary"; + upstreamUrl: string; + version: string; + archiveCachePath: string; + archiveSha256: string; + archiveInstallPath: string; + binaryMember: string; + binaryCachePath: string; + binarySha256: string; + installPath: string; + configPath: string; + unitPath: string; + serviceName: string; + listenHost: string; + listenPort: number; + podAccess: HostProxyPodAccess | null; + clashApiListen: string; + healthUrl: string; +} + +interface HostProxyPodAccess { + enabled: boolean; + listenHost: string; + listenPort: number; + proxyUrl: string; +} + +interface HostProxyTarget { + id: string; + route: string; + enabled: boolean; + sourceRef: string; + source: HostProxySource; + env: { + httpProxy: string; + httpsProxy: string; + allProxy: string; + noProxy: string[]; + }; + files: { + envFile: string; + profile: string; + apt: string; + dockerSystemdDropIn: string; + k3sSystemdDropIn: string; + }; + apply: { + reloadSystemd: boolean; + restartDocker: boolean; + restartK3s: boolean; + }; +} + +interface HostProxySecretMaterial { + serverConfigJson: string; + clientConfigJson: string; + fingerprint: string; + sourcePath: string; + valuesPrinted: false; +} + +type HostProxyAction = "plan" | "apply" | "status"; + +interface HostProxyOptions { + action: HostProxyAction; + targetId: string; + confirm: boolean; + dryRun: boolean; +} + +export async function runPlatformInfraHostProxyCommand(_config: UniDeskConfig, args: string[]): Promise> { + const options = parseOptions(args); + const config = readHostProxyConfig(); + const target = resolveTarget(config, options.targetId); + if (options.action === "plan") return renderPlan(plan(config.server, target)); + if (options.action === "status") return renderStatus(status(config.server, target)); + if (!options.confirm || options.dryRun) { + return renderPlan({ + ...plan(config.server, target), + mode: "dry-run", + mutation: false, + next: { + apply: `bun scripts/cli.ts platform-infra egress-proxy host apply --target ${target.id} --confirm`, + status: `bun scripts/cli.ts platform-infra egress-proxy host status --target ${target.id}`, + }, + }); + } + return renderApply(apply(config.server, target)); +} + +function parseOptions(args: string[]): HostProxyOptions { + const actionRaw = args[0] ?? "plan"; + if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status") { + throw new Error("platform-infra egress-proxy host usage: host plan|apply|status --target JD01 [--dry-run|--confirm]"); + } + const targetId = option(args, "--target"); + const confirm = args.includes("--confirm"); + const dryRun = args.includes("--dry-run") || !confirm; + if (confirm && args.includes("--dry-run")) throw new Error("host apply accepts only one of --confirm or --dry-run"); + return { action: actionRaw, targetId: targetId ?? readHostProxyConfig().defaults.targetId, confirm, dryRun }; +} + +function readHostProxyConfig(): HostProxyConfig { + const root = record(Bun.YAML.parse(readFileSync(rootPath(configPath), "utf8")), configPath); + if (root.kind !== "platform-infra-host-proxy") throw new Error(`${configPath}.kind must be platform-infra-host-proxy`); + const defaultsRaw = record(root.defaults, `${configPath}.defaults`); + const server = parseServer(record(root.server, `${configPath}.server`), `${configPath}.server`); + const sources = Object.fromEntries(Object.entries(record(root.sources, `${configPath}.sources`)).map(([id, value]) => { + const source = parseSource(id, record(value, `${configPath}.sources.${id}`)); + if (source.serverRef !== `server.${server.id}`) throw new Error(`${configPath}.sources.${id}.serverRef must be server.${server.id}`); + return [id, source]; + })); + const targets: Record = {}; + for (const [id, value] of Object.entries(record(root.targets, `${configPath}.targets`))) { + targets[id] = parseTarget(id, record(value, `${configPath}.targets.${id}`), sources); + } + return { + defaults: { targetId: stringField(defaultsRaw, "targetId", `${configPath}.defaults`) }, + server, + sources, + targets, + }; +} + +function parseServer(raw: Record, label: string): HostProxyServer { + const sourceConfigRef = stringField(raw, "sourceConfigRef", label); + const resolved = resolveEgressProxySourceRef(sourceConfigRef, `${label}.sourceConfigRef`); + if (resolved.sourceType !== "master-shadowsocks" || resolved.masterShadowsocks === null) { + throw new Error(`${label}.sourceConfigRef must reference a master-shadowsocks source`); + } + const healthRaw = record(raw.health, `${label}.health`); + const server = { + id: stringField(raw, "id", label), + enabled: booleanField(raw, "enabled", label), + benchmarkRef: stringField(raw, "benchmarkRef", label), + implementationRef: stringField(raw, "implementationRef", label), + sourceConfigRef, + sourceRef: resolved.sourceRef, + sourceKey: resolved.sourceKey, + sourceFingerprint: resolved.fingerprint, + masterShadowsocks: resolved.masterShadowsocks, + composeFile: repoYamlPathField(raw, "composeFile", label), + serviceName: stringField(raw, "serviceName", label), + containerName: stringField(raw, "containerName", label), + image: imageField(raw, "image", label), + configPath: absolutePathField(raw, "configPath", label), + listenHost: hostField(raw, "listenHost", label), + listenPort: portField(raw, "listenPort", label), + health: { + mode: enumField(healthRaw, "mode", `${label}.health`, ["tcp"] as const), + host: hostField(healthRaw, "host", `${label}.health`), + port: portField(healthRaw, "port", `${label}.health`), + }, + fingerprint: "", + }; + return { + ...server, + fingerprint: fingerprint({ + benchmarkRef: server.benchmarkRef, + implementationRef: server.implementationRef, + sourceConfigRef, + sourceFingerprint: server.sourceFingerprint, + composeFile: server.composeFile, + serviceName: server.serviceName, + containerName: server.containerName, + configPath: server.configPath, + listenHost: server.listenHost, + listenPort: server.listenPort, + health: server.health, + }), + }; +} + +function parseSource(id: string, raw: Record): HostProxySource { + const sourceType = enumField(raw, "sourceType", `${configPath}.sources.${id}`, ["benchmark-validated-master-shadowsocks"] as const); + const sourceConfigRef = stringField(raw, "sourceConfigRef", `${configPath}.sources.${id}`); + const resolved = resolveEgressProxySourceRef(sourceConfigRef, `${configPath}.sources.${id}.sourceConfigRef`); + if (resolved.sourceType !== "master-shadowsocks" || resolved.masterShadowsocks === null) { + throw new Error(`${configPath}.sources.${id}.sourceConfigRef must reference a master-shadowsocks source`); + } + const client = clientSpec(record(raw.client, `${configPath}.sources.${id}.client`), `${configPath}.sources.${id}.client`); + const proxyUrl = urlField(raw, "proxyUrl", `${configPath}.sources.${id}`); + const expectedProxyUrl = `http://${client.listenHost}:${client.listenPort}`; + if (proxyUrl !== expectedProxyUrl) throw new Error(`${configPath}.sources.${id}.proxyUrl must be ${expectedProxyUrl}`); + const source = { + id, + sourceType, + serverRef: stringField(raw, "serverRef", `${configPath}.sources.${id}`), + benchmarkRef: stringField(raw, "benchmarkRef", `${configPath}.sources.${id}`), + implementationRef: stringField(raw, "implementationRef", `${configPath}.sources.${id}`), + sourceConfigRef, + sourceRef: resolved.sourceRef, + sourceKey: resolved.sourceKey, + sourceFingerprint: resolved.fingerprint, + masterShadowsocks: resolved.masterShadowsocks, + client, + proxyUrl, + externalProbeUrl: urlField(raw, "externalProbeUrl", `${configPath}.sources.${id}`), + fingerprint: "", + }; + return { + ...source, + fingerprint: fingerprint({ + sourceType, + benchmarkRef: source.benchmarkRef, + implementationRef: source.implementationRef, + sourceConfigRef, + sourceFingerprint: source.sourceFingerprint, + client, + proxyUrl, + externalProbeUrl: source.externalProbeUrl, + }), + }; +} + +function clientSpec(raw: Record, label: string): HostProxyClient { + const mode = enumField(raw, "mode", label, ["trans-static-binary"] as const); + const podAccess = raw.podAccess === undefined ? null : podAccessSpec(record(raw.podAccess, `${label}.podAccess`), `${label}.podAccess`); + const client = { + mode, + upstreamUrl: httpsUrlField(raw, "upstreamUrl", label), + version: stringField(raw, "version", label), + archiveCachePath: repoStatePathField(raw, "archiveCachePath", label), + archiveSha256: sha256Field(raw, "archiveSha256", label), + archiveInstallPath: absolutePathField(raw, "archiveInstallPath", label), + binaryMember: relativePathField(raw, "binaryMember", label), + binaryCachePath: repoStatePathField(raw, "binaryCachePath", label), + binarySha256: sha256Field(raw, "binarySha256", label), + installPath: absolutePathField(raw, "installPath", label), + configPath: absolutePathField(raw, "configPath", label), + unitPath: absolutePathField(raw, "unitPath", label), + serviceName: stringField(raw, "serviceName", label), + listenHost: hostField(raw, "listenHost", label), + listenPort: portField(raw, "listenPort", label), + podAccess, + clashApiListen: listenAddressField(raw, "clashApiListen", label), + healthUrl: urlField(raw, "healthUrl", label), + }; + if (client.listenHost !== "127.0.0.1") throw new Error(`${label}.listenHost must stay 127.0.0.1 for host-local bootstrap proxy`); + if (client.unitPath !== `/etc/systemd/system/${client.serviceName}.service`) throw new Error(`${label}.unitPath must match serviceName`); + return client; +} + +function podAccessSpec(raw: Record, label: string): HostProxyPodAccess { + const enabled = booleanField(raw, "enabled", label); + const listenHost = hostField(raw, "listenHost", label); + const listenPort = portField(raw, "listenPort", label); + const proxyUrl = urlField(raw, "proxyUrl", label); + const expectedProxyUrl = `http://${listenHost}:${listenPort}`; + if (proxyUrl !== expectedProxyUrl) throw new Error(`${label}.proxyUrl must be ${expectedProxyUrl}`); + if (listenHost === "0.0.0.0" || listenHost === "127.0.0.1") { + throw new Error(`${label}.listenHost must be a pod-reachable private host address, not ${listenHost}`); + } + if (!privateIpv4(listenHost)) throw new Error(`${label}.listenHost must be a private IPv4 address`); + return { enabled, listenHost, listenPort, proxyUrl }; +} + +function parseTarget(id: string, raw: Record, sources: Record): HostProxyTarget { + const sourceRef = stringField(raw, "sourceRef", `${configPath}.targets.${id}`); + if (!sourceRef.startsWith("sources.")) throw new Error(`${configPath}.targets.${id}.sourceRef must use sources.`); + const sourceId = sourceRef.slice("sources.".length); + const source = sources[sourceId]; + if (source === undefined) throw new Error(`${configPath}.targets.${id}.sourceRef points to missing source ${sourceId}`); + const envRaw = record(raw.env, `${configPath}.targets.${id}.env`); + const filesRaw = record(raw.files, `${configPath}.targets.${id}.files`); + const applyRaw = record(raw.apply, `${configPath}.targets.${id}.apply`); + const noProxy = stringArrayField(envRaw, "noProxy", `${configPath}.targets.${id}.env`); + if (!noProxy.includes("hyueapi.com") || !noProxy.includes(".hyueapi.com")) { + throw new Error(`${configPath}.targets.${id}.env.noProxy must preserve hyueapi.com and .hyueapi.com`); + } + const target: HostProxyTarget = { + id, + route: routeField(raw, "route", `${configPath}.targets.${id}`), + enabled: booleanField(raw, "enabled", `${configPath}.targets.${id}`), + sourceRef, + source, + env: { + httpProxy: urlField(envRaw, "httpProxy", `${configPath}.targets.${id}.env`), + httpsProxy: urlField(envRaw, "httpsProxy", `${configPath}.targets.${id}.env`), + allProxy: urlField(envRaw, "allProxy", `${configPath}.targets.${id}.env`), + noProxy, + }, + files: { + envFile: absolutePathField(filesRaw, "envFile", `${configPath}.targets.${id}.files`), + profile: absolutePathField(filesRaw, "profile", `${configPath}.targets.${id}.files`), + apt: absolutePathField(filesRaw, "apt", `${configPath}.targets.${id}.files`), + dockerSystemdDropIn: absolutePathField(filesRaw, "dockerSystemdDropIn", `${configPath}.targets.${id}.files`), + k3sSystemdDropIn: absolutePathField(filesRaw, "k3sSystemdDropIn", `${configPath}.targets.${id}.files`), + }, + apply: { + reloadSystemd: booleanField(applyRaw, "reloadSystemd", `${configPath}.targets.${id}.apply`), + restartDocker: booleanField(applyRaw, "restartDocker", `${configPath}.targets.${id}.apply`), + restartK3s: booleanField(applyRaw, "restartK3s", `${configPath}.targets.${id}.apply`), + }, + }; + if (target.env.httpProxy !== source.proxyUrl || target.env.httpsProxy !== source.proxyUrl || target.env.allProxy !== source.proxyUrl) { + throw new Error(`${configPath}.targets.${id}.env proxy URLs must match ${sourceRef}.proxyUrl`); + } + return target; +} + +function resolveTarget(config: HostProxyConfig, targetId: string): HostProxyTarget { + const target = config.targets[targetId]; + if (target === undefined) throw new Error(`${configPath}.targets.${targetId} is not defined`); + return target; +} + +function plan(server: HostProxyServer, target: HostProxyTarget): Record { + const client = target.source.client; + return { + ok: true, + command: "platform-infra egress-proxy host plan", + mode: "plan", + mutation: false, + server: serverSummary(server), + target: targetSummary(target), + artifact: artifactSummary(client), + files: target.files, + apply: target.apply, + env: { proxyUrl: target.env.httpProxy, noProxyCount: target.env.noProxy.length, noProxyIncludesHyueapi: true }, + valuesPrinted: false, + next: { + dryRun: `bun scripts/cli.ts platform-infra egress-proxy host apply --target ${target.id} --dry-run`, + apply: `bun scripts/cli.ts platform-infra egress-proxy host apply --target ${target.id} --confirm`, + status: `bun scripts/cli.ts platform-infra egress-proxy host status --target ${target.id}`, + }, + }; +} + +function apply(server: HostProxyServer, target: HostProxyTarget): Record { + const material = proxySecretMaterial(server, target.source); + const serverApply = applyMasterProxyServer(server, material); + const artifact = prepareClientArtifact(target.source.client); + const remotePrepare = runTrans(target.route, remotePrepareScript(target.source.client), 30); + const remoteArchive = remotePrepare.exitCode === 0 + ? runTrans(target.route, remoteArchiveStatusScript(target.source.client), 30) + : remotePrepare; + const remoteArchiveParsed = record(parseJson(remoteArchive.stdout) ?? {}); + const upload = remotePrepare.exitCode === 0 && remoteArchive.exitCode === 0 && remoteArchiveParsed.archiveSha256Ok === true + ? skippedCommandResult(["bun", "scripts/ssh-cli.ts", "ssh", target.route, "upload", rootPath(target.source.client.archiveCachePath), target.source.client.archiveInstallPath], "remote archive already matches YAML sha256") + : remotePrepare.exitCode === 0 + ? runTransUpload(target.route, rootPath(target.source.client.archiveCachePath), target.source.client.archiveInstallPath, 1_800_000) + : remotePrepare; + const remoteApply = upload.exitCode === 0 + ? runTrans(target.route, remoteApplyScript(target, material.clientConfigJson), 60) + : upload; + const parsed = parseJson(remoteApply.stdout); + return { + ok: serverApply.ok === true && artifact.ok === true && remotePrepare.exitCode === 0 && upload.exitCode === 0 && remoteApply.exitCode === 0 && record(parsed).ok !== false, + command: "platform-infra egress-proxy host apply", + mode: "apply", + mutation: serverApply.ok === true && remoteApply.exitCode === 0, + server: serverSummary(server), + serverApply, + target: targetSummary(target), + artifact, + distribution: { + mode: target.source.client.mode, + artifact: "archive", + prepare: compactLocalResult(remotePrepare), + remoteArchive: compactLocalResult(remoteArchive), + upload: compactLocalResult(upload), + }, + remote: parsed ?? { stdoutPreview: remoteApply.stdout.slice(0, 2000) }, + stderrTail: remoteApply.stderr.slice(-2000), + valuesPrinted: false, + next: { status: `bun scripts/cli.ts platform-infra egress-proxy host status --target ${target.id}` }, + }; +} + +function status(server: HostProxyServer, target: HostProxyTarget): Record { + const serverStatus = masterProxyServerStatus(server); + const result = runTrans(target.route, remoteStatusScript(target), 45); + const parsed = parseJson(result.stdout); + return { + ok: serverStatus.ok === true && result.exitCode === 0 && record(parsed).ok !== false, + command: "platform-infra egress-proxy host status", + mode: "status", + mutation: false, + server: serverSummary(server), + serverStatus, + target: targetSummary(target), + remote: parsed ?? { stdoutPreview: result.stdout.slice(0, 2000) }, + stderrTail: result.stderr.slice(-2000), + valuesPrinted: false, + }; +} + +function proxySecretMaterial(server: HostProxyServer, source: HostProxySource): HostProxySecretMaterial { + const envSource = readEnvSourceFile({ + root: rootPath(".state", "secrets"), + sourceRef: source.sourceRef, + missingMessage: (sourcePath) => `host proxy requires ${sourcePath} with ${source.sourceKey}`, + }); + const password = requiredEnvValue(envSource.values, source.sourceKey, source.sourceRef); + const serverConfigJson = renderMasterShadowsocksServerConfig(server, password); + const clientConfigJson = renderSingBoxClientConfig(source, password); + return { + serverConfigJson, + clientConfigJson, + sourcePath: envSource.sourcePathRedacted, + fingerprint: fingerprintSecretValues({ password, serverConfigJson, clientConfigJson }, ["password", "serverConfigJson", "clientConfigJson"]), + valuesPrinted: false, + }; +} + +function renderMasterShadowsocksServerConfig(server: HostProxyServer, password: string): string { + return `${JSON.stringify({ + server: server.listenHost, + server_port: server.listenPort, + password, + method: server.masterShadowsocks.method, + timeout: 300, + mode: "tcp_only", + }, null, 2)}\n`; +} + +function renderSingBoxClientConfig(source: HostProxySource, password: string): string { + const client = source.client; + const inbounds = [ + { type: "mixed", tag: "mixed-host-local", listen: client.listenHost, listen_port: client.listenPort }, + ...(client.podAccess?.enabled + ? [{ type: "mixed", tag: "mixed-pod-access", listen: client.podAccess.listenHost, listen_port: client.podAccess.listenPort }] + : []), + ]; + return `${JSON.stringify({ + log: { level: "info", timestamp: true }, + experimental: { clash_api: { external_controller: client.clashApiListen } }, + inbounds, + outbounds: [ + { + type: "shadowsocks", + tag: "master-vpn", + server: source.masterShadowsocks.serverHost, + server_port: source.masterShadowsocks.serverPort, + method: source.masterShadowsocks.method, + password, + }, + { type: "direct", tag: "direct" }, + { type: "block", tag: "block" }, + ], + route: { + rules: [ + { ip_is_private: true, outbound: "direct" }, + { domain_suffix: ["cluster.local", "svc"], outbound: "direct" }, + ], + final: "master-vpn", + }, + }, null, 2)}\n`; +} + +function applyMasterProxyServer(server: HostProxyServer, material: HostProxySecretMaterial): Record { + mkdirSync(dirname(server.configPath), { recursive: true }); + writeFileSync(server.configPath, material.serverConfigJson, "utf8"); + chmodSync(server.configPath, 0o600); + const compose = runCommand(["docker", "compose", "-f", rootPath(server.composeFile), "up", "-d", server.serviceName], rootPath(), { timeoutMs: 60_000 }); + const health = masterProxyTcpHealth(server); + const inspect = runCommand(["docker", "inspect", server.containerName, "--format", "{{.State.Status}} {{.State.Running}} {{.Config.Image}}"], rootPath(), { timeoutMs: 15_000 }); + const existingHealthy = compose.exitCode !== 0 + && /container name .* already in use|Conflict\. The container name/i.test(compose.stderr) + && inspect.exitCode === 0 + && /\brunning true\b/.test(inspect.stdout.trim()) + && health.ok === true; + return { + ok: server.enabled && (compose.exitCode === 0 || existingHealthy) && health.ok, + enabled: server.enabled, + benchmarkRef: server.benchmarkRef, + implementationRef: server.implementationRef, + sourceConfigRef: server.sourceConfigRef, + sourceFingerprint: server.sourceFingerprint, + materialFingerprint: material.fingerprint, + compose: compactLocalResult(compose), + existingHealthy, + inspect: compactLocalResult(inspect), + health, + valuesPrinted: false, + }; +} + +function masterProxyServerStatus(server: HostProxyServer): Record { + const inspect = runCommand(["docker", "inspect", server.containerName, "--format", "{{.State.Status}} {{.State.Running}} {{.Config.Image}}"], rootPath(), { timeoutMs: 15_000 }); + const health = masterProxyTcpHealth(server); + return { + ok: server.enabled && inspect.exitCode === 0 && health.ok, + enabled: server.enabled, + benchmarkRef: server.benchmarkRef, + implementationRef: server.implementationRef, + sourceConfigRef: server.sourceConfigRef, + sourceFingerprint: server.sourceFingerprint, + container: inspect.exitCode === 0 ? inspect.stdout.trim() : "missing", + inspect: compactLocalResult(inspect), + health, + valuesPrinted: false, + }; +} + +function prepareClientArtifact(client: HostProxyClient): Record { + const binaryPath = rootPath(client.binaryCachePath); + const archivePath = rootPath(client.archiveCachePath); + let archiveAction = "existing"; + let binaryAction = "existing"; + if (!existsSync(binaryPath) || sha256File(binaryPath) !== client.binarySha256) { + if (!existsSync(archivePath) || sha256File(archivePath) !== client.archiveSha256) { + mkdirSync(dirname(archivePath), { recursive: true }); + const download = runCommand(["curl", "-fL", "--connect-timeout", "15", "--max-time", "240", "-o", archivePath, client.upstreamUrl], rootPath(), { timeoutMs: 300_000 }); + if (download.exitCode !== 0) return { ok: false, action: "download", result: compactLocalResult(download), valuesPrinted: false }; + archiveAction = "downloaded"; + } + if (sha256File(archivePath) !== client.archiveSha256) { + return { ok: false, action: archiveAction, reason: "archive-sha256-mismatch", path: client.archiveCachePath, valuesPrinted: false }; + } + mkdirSync(dirname(binaryPath), { recursive: true }); + const extract = runCommand(["bash", "-lc", `tar -xOf ${shQuote(archivePath)} ${shQuote(client.binaryMember)} > ${shQuote(binaryPath)} && chmod 0755 ${shQuote(binaryPath)}`], rootPath(), { timeoutMs: 120_000 }); + if (extract.exitCode !== 0) return { ok: false, action: "extract", result: compactLocalResult(extract), valuesPrinted: false }; + binaryAction = "extracted"; + } + const binarySha = sha256File(binaryPath); + return { + ok: binarySha === client.binarySha256, + mode: client.mode, + version: client.version, + archiveCachePath: client.archiveCachePath, + binaryCachePath: client.binaryCachePath, + archiveAction, + binaryAction, + binarySha256: binarySha, + installPath: client.installPath, + valuesPrinted: false, + }; +} + +function remotePrepareScript(client: HostProxyClient): string { + return ` +set -eu +mkdir -p ${shQuote(dirname(client.installPath))} ${shQuote(dirname(client.configPath))} ${shQuote(dirname(client.unitPath))} ${shQuote(dirname(client.archiveInstallPath))} +`; +} + +function remoteArchiveStatusScript(client: HostProxyClient): string { + return ` +set -eu +archive_sha="$(sha256sum ${shQuote(client.archiveInstallPath)} 2>/dev/null | awk '{print $1}' || true)" +python3 - "$archive_sha" <<'PY' +import json +import sys + +archive_sha = sys.argv[1] +payload = { + "ok": True, + "archivePresent": bool(archive_sha), + "archiveSha256Ok": archive_sha == "${client.archiveSha256}", + "archiveSha256Prefix": archive_sha[:12] if archive_sha else "", + "valuesPrinted": False, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +`; +} + +function remoteApplyScript(target: HostProxyTarget, clientConfigJson: string): string { + const client = target.source.client; + const envFile = renderEnvFile(target); + const profile = renderProfile(target); + const apt = renderApt(target); + const dockerDropIn = renderSystemdDropIn(target.files.envFile); + const k3sDropIn = renderSystemdDropIn(target.files.envFile); + const unit = renderClientUnit(client); + const daemonReload = target.apply.reloadSystemd ? "systemctl daemon-reload >/dev/null 2>&1 || true" : ": # reloadSystemd disabled by YAML"; + const restartDocker = target.apply.restartDocker + ? "nohup sh -c 'sleep 1; systemctl restart docker >/tmp/unidesk-host-proxy-docker-restart.out 2>/tmp/unidesk-host-proxy-docker-restart.err || true' >/dev/null 2>&1 &" + : ": # restartDocker disabled by YAML"; + const restartK3s = target.apply.restartK3s + ? "nohup sh -c 'sleep 1; systemctl restart k3s >/tmp/unidesk-host-proxy-k3s-restart.out 2>/tmp/unidesk-host-proxy-k3s-restart.err || true' >/dev/null 2>&1 &" + : ": # restartK3s disabled by YAML"; + return ` +set -eu +write_b64() { + path="$1" + mode="$2" + data="$3" + mkdir -p "$(dirname "$path")" + tmp="$path.tmp.$$" + printf '%s' "$data" | base64 -d >"$tmp" + chmod "$mode" "$tmp" + mv "$tmp" "$path" +} +write_b64 ${shQuote(client.configPath)} 0600 ${shQuote(b64(clientConfigJson))} +write_b64 ${shQuote(client.unitPath)} 0644 ${shQuote(b64(unit))} +write_b64 ${shQuote(target.files.envFile)} 0644 ${shQuote(b64(envFile))} +write_b64 ${shQuote(target.files.profile)} 0644 ${shQuote(b64(profile))} +write_b64 ${shQuote(target.files.apt)} 0644 ${shQuote(b64(apt))} +write_b64 ${shQuote(target.files.dockerSystemdDropIn)} 0644 ${shQuote(b64(dockerDropIn))} +write_b64 ${shQuote(target.files.k3sSystemdDropIn)} 0644 ${shQuote(b64(k3sDropIn))} +archive_sha="$(sha256sum ${shQuote(client.archiveInstallPath)} 2>/dev/null | awk '{print $1}' || true)" +if [ "$archive_sha" != ${shQuote(client.archiveSha256)} ]; then + printf 'archive sha256 mismatch\\n' >&2 + exit 23 +fi +tmp_bin="${client.installPath}.tmp.$$" +tar -xOf ${shQuote(client.archiveInstallPath)} ${shQuote(client.binaryMember)} >"$tmp_bin" +chmod 0755 "$tmp_bin" +binary_sha="$(sha256sum "$tmp_bin" | awk '{print $1}')" +if [ "$binary_sha" != ${shQuote(client.binarySha256)} ]; then + rm -f "$tmp_bin" + printf 'binary sha256 mismatch\\n' >&2 + exit 24 +fi +mv -f "$tmp_bin" ${shQuote(client.installPath)} +chmod 0755 ${shQuote(client.installPath)} +${daemonReload} +systemctl enable --now ${shQuote(client.serviceName)} >/tmp/unidesk-host-proxy-systemctl.out 2>/tmp/unidesk-host-proxy-systemctl.err || true +systemctl restart ${shQuote(client.serviceName)} >/tmp/unidesk-host-proxy-restart.out 2>/tmp/unidesk-host-proxy-restart.err || true +sleep 2 +${remoteStatusProbeShell(target)} +${restartDocker} +${restartK3s} + `; +} + +function remoteStatusScript(target: HostProxyTarget): string { + return ` +set -eu +${remoteStatusProbeShell(target)} +`; +} + +function remoteStatusProbeShell(target: HostProxyTarget): string { + const client = target.source.client; + const podAccess = client.podAccess; + const podAccessShell = podAccess?.enabled + ? ` +pod_probe_rc=0 +pod_probe_status="$(HTTP_PROXY=${shQuote(podAccess.proxyUrl)} HTTPS_PROXY=${shQuote(podAccess.proxyUrl)} ALL_PROXY=${shQuote(podAccess.proxyUrl)} NO_PROXY=${shQuote(target.env.noProxy.join(","))} curl -fsS -o /dev/null -w '%{http_code}' --max-time 20 ${shQuote(target.source.externalProbeUrl)} 2>/tmp/unidesk-host-proxy-pod-probe.err)" || pod_probe_rc=$? +` + : ` +pod_probe_rc=0 +pod_probe_status="skipped" +`; + return ` +exists() { [ -f "$1" ] && printf present || printf missing; } +contains() { grep -F "$2" "$1" >/dev/null 2>&1 && printf true || printf false; } +service_active="$(systemctl is-active ${shQuote(client.serviceName)} 2>/dev/null || true)" +binary_sha="$(sha256sum ${shQuote(client.installPath)} 2>/dev/null | awk '{print $1}' || true)" +archive_sha="$(sha256sum ${shQuote(client.archiveInstallPath)} 2>/dev/null | awk '{print $1}' || true)" +health_rc=0 +curl -fsS --max-time 5 ${shQuote(client.healthUrl)} >/tmp/unidesk-host-proxy-health.out 2>/tmp/unidesk-host-proxy-health.err || health_rc=$? +probe_rc=0 +probe_status="$(HTTP_PROXY=${shQuote(target.env.httpProxy)} HTTPS_PROXY=${shQuote(target.env.httpsProxy)} ALL_PROXY=${shQuote(target.env.allProxy)} NO_PROXY=${shQuote(target.env.noProxy.join(","))} curl -fsS -o /dev/null -w '%{http_code}' --max-time 20 ${shQuote(target.source.externalProbeUrl)} 2>/tmp/unidesk-host-proxy-probe.err)" || probe_rc=$? +${podAccessShell} +python3 - "$health_rc" "$probe_rc" "$probe_status" "$(exists ${shQuote(target.files.envFile)})" "$(exists ${shQuote(target.files.profile)})" "$(exists ${shQuote(target.files.apt)})" "$(exists ${shQuote(target.files.dockerSystemdDropIn)})" "$(exists ${shQuote(target.files.k3sSystemdDropIn)})" "$(contains ${shQuote(target.files.envFile)} hyueapi.com)" "$(contains ${shQuote(target.files.envFile)} .hyueapi.com)" "$service_active" "$binary_sha" "$archive_sha" ${shQuote(podAccess?.enabled ? "true" : "false")} "$pod_probe_rc" "$pod_probe_status" <<'PY' +import json +import sys + +health_rc = int(sys.argv[1]) +probe_rc = int(sys.argv[2]) +service_active = sys.argv[11] +binary_sha = sys.argv[12] +archive_sha = sys.argv[13] +pod_access_enabled = sys.argv[14] == "true" +pod_probe_rc = int(sys.argv[15]) +payload = { + "ok": health_rc == 0 and probe_rc == 0 and (not pod_access_enabled or pod_probe_rc == 0) and service_active == "active" and binary_sha == "${client.binarySha256}" and archive_sha == "${client.archiveSha256}" and all(value == "present" for value in sys.argv[4:9]) and sys.argv[9] == "true" and sys.argv[10] == "true", + "target": "${target.id}", + "route": "${target.route}", + "sourceRef": "${target.sourceRef}", + "sourceFingerprint": "${target.source.fingerprint}", + "client": { + "mode": "${client.mode}", + "service": "${client.serviceName}", + "serviceActive": service_active, + "binarySha256Ok": binary_sha == "${client.binarySha256}", + "archiveSha256Ok": archive_sha == "${client.archiveSha256}", + "listen": "${client.listenHost}:${client.listenPort}", + "podAccess": { + "enabled": pod_access_enabled, + "listen": "${podAccess?.enabled ? `${podAccess.listenHost}:${podAccess.listenPort}` : ""}", + "proxyUrl": "${podAccess?.enabled ? podAccess.proxyUrl : ""}", + }, + }, + "files": { + "envFile": sys.argv[4], + "profile": sys.argv[5], + "apt": sys.argv[6], + "dockerSystemdDropIn": sys.argv[7], + "k3sSystemdDropIn": sys.argv[8], + }, + "noProxy": {"hyueapi": sys.argv[9] == "true", "wildcardHyueapi": sys.argv[10] == "true"}, + "health": {"exitCode": health_rc, "url": "${client.healthUrl}"}, + "externalProbe": {"exitCode": probe_rc, "status": sys.argv[3], "url": "${target.source.externalProbeUrl}"}, + "podAccessProbe": {"enabled": pod_access_enabled, "exitCode": pod_probe_rc, "status": sys.argv[16], "url": "${target.source.externalProbeUrl}"}, + "valuesPrinted": False, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +`; +} + +function renderClientUnit(client: HostProxyClient): string { + return [ + "[Unit]", + "Description=UniDesk host egress proxy client", + "After=network-online.target", + "Wants=network-online.target", + "", + "[Service]", + "Type=simple", + `ExecStart=${client.installPath} run -c ${client.configPath}`, + "Restart=always", + "RestartSec=5", + "LimitNOFILE=1048576", + "NoNewPrivileges=true", + "", + "[Install]", + "WantedBy=multi-user.target", + "", + ].join("\n"); +} + +function renderEnvFile(target: HostProxyTarget): string { + const noProxy = target.env.noProxy.join(","); + return [ + "# Generated by: bun scripts/cli.ts platform-infra egress-proxy host apply", + "# Source: config/platform-infra/host-proxy.yaml", + `HTTP_PROXY=${target.env.httpProxy}`, + `HTTPS_PROXY=${target.env.httpsProxy}`, + `ALL_PROXY=${target.env.allProxy}`, + `NO_PROXY=${noProxy}`, + `http_proxy=${target.env.httpProxy}`, + `https_proxy=${target.env.httpsProxy}`, + `all_proxy=${target.env.allProxy}`, + `no_proxy=${noProxy}`, + "", + ].join("\n"); +} + +function renderProfile(target: HostProxyTarget): string { + return renderEnvFile(target) + .split("\n") + .filter((line) => line.length > 0 && !line.startsWith("#")) + .map((line) => `export ${line}`) + .join("\n") + "\n"; +} + +function renderApt(target: HostProxyTarget): string { + return [ + `Acquire::http::Proxy "${target.env.httpProxy}";`, + `Acquire::https::Proxy "${target.env.httpsProxy}";`, + "", + ].join("\n"); +} + +function renderSystemdDropIn(envFile: string): string { + return ["[Service]", `EnvironmentFile=-${envFile}`, ""].join("\n"); +} + +function serverSummary(server: HostProxyServer): Record { + return { + id: server.id, + enabled: server.enabled, + benchmarkRef: server.benchmarkRef, + implementationRef: server.implementationRef, + sourceConfigRef: server.sourceConfigRef, + sourceFingerprint: server.sourceFingerprint, + composeFile: server.composeFile, + serviceName: server.serviceName, + containerName: server.containerName, + image: server.image, + listen: `${server.listenHost}:${server.listenPort}`, + health: `${server.health.host}:${server.health.port}`, + fingerprint: server.fingerprint, + valuesPrinted: false, + }; +} + +function targetSummary(target: HostProxyTarget): Record { + const client = target.source.client; + return { + id: target.id, + route: target.route, + enabled: target.enabled, + sourceRef: target.sourceRef, + sourceType: target.source.sourceType, + sourceFingerprint: target.source.fingerprint, + proxyUrl: target.source.proxyUrl, + benchmarkRef: target.source.benchmarkRef, + implementationRef: target.source.implementationRef, + client: { + mode: client.mode, + version: client.version, + installPath: client.installPath, + configPath: client.configPath, + unitPath: client.unitPath, + serviceName: client.serviceName, + listen: `${client.listenHost}:${client.listenPort}`, + }, + }; +} + +function artifactSummary(client: HostProxyClient): Record { + return { + mode: client.mode, + version: client.version, + upstreamUrl: client.upstreamUrl, + archiveCachePath: client.archiveCachePath, + archiveSha256: client.archiveSha256, + archiveInstallPath: client.archiveInstallPath, + binaryCachePath: client.binaryCachePath, + binarySha256: client.binarySha256, + installPath: client.installPath, + }; +} + +function masterProxyTcpHealth(server: HostProxyServer): Record { + const result = runCommand(["bash", "-lc", `timeout 5 bash -lc ${shQuote(` { + return { + exitCode: result.exitCode, + timedOut: result.timedOut, + stdoutTail: result.stdout.slice(-1000), + stderrTail: result.stderr.slice(-1000), + }; +} + +function renderPlan(result: Record): RenderedCliResult { + const server = record(result.server); + const target = record(result.target); + const env = record(result.env); + const artifact = record(result.artifact); + const next = record(result.next); + const lines = [ + "PLATFORM-INFRA HOST PROXY PLAN", + ...table(["SERVER", "LISTEN", "BENCHMARK", "SOURCE"], [[text(server.id), text(server.listen), text(server.benchmarkRef), text(server.sourceConfigRef)]]), + "", + ...table(["TARGET", "ROUTE", "SOURCE", "PROXY"], [[text(target.id), text(target.route), text(target.sourceRef), text(target.proxyUrl)]]), + "", + ...table(["CLIENT_MODE", "VERSION", "INSTALL", "SHA256"], [[text(artifact.mode), text(artifact.version), text(artifact.installPath), shortSha(text(artifact.binarySha256))]]), + "", + ...table(["PROXY", "NO_PROXY_COUNT", "HYUEAPI", "VALUES"], [[text(env.proxyUrl), text(env.noProxyCount), "preserved", "printed=false"]]), + "", + "NEXT", + ` dry-run: ${text(next.dryRun, "")}`, + ` apply: ${text(next.apply, "")}`, + ` status: ${text(next.status, "")}`, + "", + "Disclosure: Secret material and generated proxy config are not printed.", + ]; + return { ok: result.ok !== false, command: text(result.command, "platform-infra egress-proxy host plan"), renderedText: lines.join("\n"), contentType: "text/plain" }; +} + +function renderApply(result: Record): RenderedCliResult { + const server = record(result.server); + const serverApply = record(result.serverApply); + const serverHealth = record(serverApply.health); + const target = record(result.target); + const artifact = record(result.artifact); + const distribution = record(result.distribution); + const remote = record(result.remote); + const client = record(remote.client); + const podProbe = record(remote.podAccessProbe); + const probe = record(remote.externalProbe); + const lines = [ + "PLATFORM-INFRA HOST PROXY APPLY", + ...table(["SERVER", "OK", "LISTEN", "BENCHMARK"], [[text(server.id), text(serverApply.ok), text(server.listen), text(server.benchmarkRef)]]), + "", + ...table(["SERVER_HEALTH", "CLIENT_BINARY", "DISTRIBUTION"], [[text(serverHealth.ok), text(artifact.binaryAction), text(distribution.mode)]]), + "", + ...table(["TARGET", "ROUTE", "OK", "CLIENT"], [[text(target.id), text(target.route), result.ok === false ? "false" : "true", text(client.service)]]), + "", + ...table(["SERVICE_ACTIVE", "BINARY_SHA", "HOST_PROBE", "POD_PROBE", "VALUES"], [[text(client.serviceActive), text(client.binarySha256Ok), text(probe.status), text(podProbe.status, "skipped"), "printed=false"]]), + "", + `NEXT\n ${text(record(result.next).status, "")}`, + ]; + return { ok: result.ok !== false, command: "platform-infra egress-proxy host apply", renderedText: lines.join("\n"), contentType: "text/plain" }; +} + +function renderStatus(result: Record): RenderedCliResult { + const server = record(result.server); + const serverStatus = record(result.serverStatus); + const target = record(result.target); + const remote = record(result.remote); + const client = record(remote.client); + const files = record(remote.files); + const noProxy = record(remote.noProxy); + const health = record(remote.health); + const probe = record(remote.externalProbe); + const podProbe = record(remote.podAccessProbe); + const lines = [ + "PLATFORM-INFRA HOST PROXY STATUS", + ...table(["SERVER", "OK", "LISTEN", "CONTAINER"], [[text(server.id), text(serverStatus.ok), text(server.listen), text(serverStatus.container)]]), + "", + ...table(["TARGET", "ROUTE", "OK", "CLIENT"], [[text(target.id), text(target.route), result.ok === false ? "false" : "true", text(client.service)]]), + "", + ...table(["SERVICE_ACTIVE", "BINARY_SHA", "HEALTH_EXIT", "HOST_PROBE", "POD_PROBE"], [[text(client.serviceActive), text(client.binarySha256Ok), text(health.exitCode), text(probe.status), text(podProbe.status, "skipped")]]), + "", + ...table(["ENV", "PROFILE", "APT", "DOCKER_DROPIN", "K3S_DROPIN"], [[text(files.envFile), text(files.profile), text(files.apt), text(files.dockerSystemdDropIn), text(files.k3sSystemdDropIn)]]), + "", + ...table(["NO_PROXY_HYUEAPI", "NO_PROXY_WILDCARD", "VALUES"], [[text(noProxy.hyueapi), text(noProxy.wildcardHyueapi), "printed=false"]]), + "", + "Disclosure: file content, Secret values and generated proxy config are not printed.", + ]; + return { ok: result.ok !== false, command: "platform-infra egress-proxy host status", renderedText: lines.join("\n"), contentType: "text/plain" }; +} + +function option(args: string[], name: string): string | null { + const index = args.indexOf(name); + if (index === -1) return null; + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); + return value; +} + +function parseJson(value: string): unknown { + const trimmed = value.trim(); + if (trimmed.length === 0) return null; + try { return JSON.parse(trimmed) as unknown; } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { return JSON.parse(trimmed.slice(start, end + 1)) as unknown; } catch {} + } + } + return null; +} + +function record(value: unknown, _label = "value"): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return {}; + return value as Record; +} + +function stringField(obj: Record, key: string, label: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${label}.${key} must be a non-empty string`); + return value.trim(); +} + +function routeField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (!/^[A-Za-z0-9:_./-]+$/u.test(value) || value.includes("..")) throw new Error(`${label}.${key} has an unsupported route format`); + return value; +} + +function repoYamlPathField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (value.startsWith("/") || value.includes("..") || !value.startsWith("config/") || !value.endsWith(".yaml")) { + throw new Error(`${label}.${key} must be a repo-relative config/*.yaml path without ..`); + } + return value; +} + +function repoStatePathField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (value.startsWith("/") || value.includes("..") || !value.startsWith(".state/")) { + throw new Error(`${label}.${key} must be a repo-relative .state/ path without ..`); + } + return value; +} + +function relativePathField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (value.startsWith("/") || value.includes("..")) throw new Error(`${label}.${key} must be a relative path without ..`); + return value; +} + +function absolutePathField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (!value.startsWith("/") || value.includes("..")) throw new Error(`${label}.${key} must be an absolute path without ..`); + return value; +} + +function booleanField(obj: Record, key: string, label: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${label}.${key} must be a boolean`); + return value; +} + +function stringArrayField(obj: Record, key: string, label: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || !value.every((item) => typeof item === "string" && item.trim().length > 0)) { + throw new Error(`${label}.${key} must be an array of non-empty strings`); + } + return value.map((item) => item.trim()); +} + +function enumField(obj: Record, key: string, label: string, values: T): T[number] { + const value = stringField(obj, key, label); + if (!(values as readonly string[]).includes(value)) throw new Error(`${label}.${key} must be one of ${values.join(", ")}`); + return value as T[number]; +} + +function urlField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`${label}.${key} must use http:// or https://`); + if (url.username || url.password || url.hash) throw new Error(`${label}.${key} must not include credentials or hash`); + return value; +} + +function httpsUrlField(obj: Record, key: string, label: string): string { + const value = urlField(obj, key, label); + if (!value.startsWith("https://")) throw new Error(`${label}.${key} must use https://`); + return value; +} + +function hostField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error(`${label}.${key} has an unsupported host/listen format`); + return value; +} + +function privateIpv4(value: string): boolean { + const parts = value.split(".").map((part) => Number(part)); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + return parts[0] === 10 + || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) + || (parts[0] === 192 && parts[1] === 168); +} + +function portField(obj: Record, key: string, label: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > 65535) throw new Error(`${label}.${key} must be a TCP port`); + return value; +} + +function imageField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + if (!/^[A-Za-z0-9._:/@+-]+$/u.test(value) || value.includes("..")) throw new Error(`${label}.${key} has an unsupported image reference format`); + return value; +} + +function listenAddressField(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label); + const [host, portRaw, extra] = value.split(":"); + if (extra !== undefined || host === undefined || portRaw === undefined || host.length === 0) throw new Error(`${label}.${key} must be host:port`); + const port = Number(portRaw); + if (!/^[A-Za-z0-9._-]+$/u.test(host) || !Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${label}.${key} must be host:port`); + return value; +} + +function sha256Field(obj: Record, key: string, label: string): string { + const value = stringField(obj, key, label).toLowerCase(); + if (!/^[0-9a-f]{64}$/u.test(value)) throw new Error(`${label}.${key} must be a sha256 hex digest`); + return value; +} + +function b64(value: string): string { + return Buffer.from(value, "utf8").toString("base64"); +} + +function fingerprint(value: unknown): string { + return `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16)}`; +} + +function text(value: unknown, fallback = "-"): string { + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return fallback; +} + +function shortSha(value: string): string { + return /^[0-9a-f]{64}$/u.test(value) ? value.slice(0, 12) : value; +} + +function table(headers: string[], rows: string[][]): string[] { + const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length))); + const format = (row: string[]) => row.map((cell, index) => (cell ?? "").padEnd(widths[index] ?? 0)).join(" ").trimEnd(); + return [format(headers), format(headers.map((header, index) => "-".repeat(widths[index] ?? header.length))), ...rows.map(format)]; +} diff --git a/scripts/src/platform-infra/manifest.ts b/scripts/src/platform-infra/manifest.ts index bb3cd896..1419a3a1 100644 --- a/scripts/src/platform-infra/manifest.ts +++ b/scripts/src/platform-infra/manifest.ts @@ -117,7 +117,9 @@ export function manifest(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): s const configHash = configHashFor(sub2api, target); const image = targetImage(sub2api, target); const dependencyImages = targetDependencyImages(sub2api, target); + const proxyEnv = sub2ApiProxyEnv(target); return template + .replaceAll("unidesk.ai/runtime-node: G14", `unidesk.ai/runtime-node: ${target.id}`) .replaceAll("__SUB2API_IMAGE__", imageRef(sub2api, target)) .replaceAll("__SUB2API_IMAGE_PULL_POLICY__", image.pullPolicy) .replaceAll("postgres:18-alpine", dependencyImages.postgres) @@ -126,7 +128,22 @@ export function manifest(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): s .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ENABLED__", String(urlAllowlist.enabled)) .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP__", String(urlAllowlist.allowInsecureHttp)) .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS__", String(urlAllowlist.allowPrivateHosts)) - .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS__", urlAllowlist.upstreamHosts.join(",")); + .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS__", urlAllowlist.upstreamHosts.join(",")) + .replaceAll( + 'UPDATE_PROXY_URL: ""', + [ + `UPDATE_PROXY_URL: "${proxyEnv.httpProxy}"`, + ` HTTP_PROXY: "${proxyEnv.httpProxy}"`, + ` HTTPS_PROXY: "${proxyEnv.httpProxy}"`, + ` ALL_PROXY: "${proxyEnv.httpProxy}"`, + ` http_proxy: "${proxyEnv.httpProxy}"`, + ` https_proxy: "${proxyEnv.httpProxy}"`, + ` all_proxy: "${proxyEnv.httpProxy}"`, + ` NO_PROXY: "${proxyEnv.noProxy}"`, + ` no_proxy: "${proxyEnv.noProxy}"`, + ].join("\n"), + ) + .concat("\n", renderPublicExposureManifest(target), renderEgressProxyManifest(target)); } export function configHashFor(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { diff --git a/scripts/src/provider-attach.ts b/scripts/src/provider-attach.ts index b1e53584..09f07372 100644 --- a/scripts/src/provider-attach.ts +++ b/scripts/src/provider-attach.ts @@ -45,6 +45,16 @@ function safeProviderSlug(providerId: string): string { return providerId.replace(/[^a-zA-Z0-9_.-]/g, "-").replace(/^-+|-+$/g, "").toLowerCase() || "provider"; } +function defaultHostSshUser(hostProjectRoot: string): string { + const homeMatch = hostProjectRoot.match(/^\/home\/([^/]+)/u); + if (homeMatch?.[1]) return homeMatch[1]; + return "root"; +} + +function defaultHostRemoteCwd(user: string): string { + return user === "root" ? "/root" : `/home/${user}`; +} + function defaultMasterServer(config: UniDeskConfig): string { return `http://${config.network.publicHost}/`; } @@ -88,11 +98,27 @@ function parseAttachOptions(config: UniDeskConfig, args: string[]): ProviderAtta } function envContent(options: ProviderAttachOptions): string { + const slug = safeProviderSlug(options.providerId); + const hostSshUser = defaultHostSshUser(repoRoot); const lines = [ "# Generated by: bun scripts/cli.ts provider attach", - "# Required attach inputs are intentionally limited to master server and provider id.", + "# Required attach inputs are intentionally limited to master server, provider id, and optional provider token.", `UNIDESK_MASTER_SERVER=${options.masterServer}`, `PROVIDER_ID=${options.providerId}`, + `PROVIDER_NAME=${options.providerId}`, + `PROVIDER_UPGRADE_HOST_PROJECT_ROOT=${repoRoot}`, + "PROVIDER_UPGRADE_WORKSPACE_PATH=/workspace", + `PROVIDER_UPGRADE_COMPOSE_FILE=provider-${options.providerId}.yml`, + `PROVIDER_UPGRADE_ENV_FILE=.state/provider-${options.providerId}.env`, + `PROVIDER_UPGRADE_COMPOSE_PROJECT=unidesk-${slug}`, + "PROVIDER_UPGRADE_SERVICE=provider-gateway", + `PROVIDER_UPGRADE_RUNNER_IMAGE=unidesk_provider-gateway:${slug}`, + "HOST_SSH_HOST=host.docker.internal", + "HOST_SSH_PORT=22", + `HOST_SSH_USER=${hostSshUser}`, + "HOST_SSH_KEY=/run/host-ssh/id_ed25519", + `HOST_REMOTE_CWD=${defaultHostRemoteCwd(hostSshUser)}`, + "HOST_LOGIN_SHELL=/bin/bash", ]; if (options.token !== null) lines.push(`PROVIDER_TOKEN=${options.token}`); return `${lines.join("\n")}\n`; @@ -109,6 +135,8 @@ function composeContent(options: ProviderAttachOptions): string { " build:", " context: .", " dockerfile: src/components/provider-gateway/Dockerfile", + " args:", + ' ALPINE_REPOSITORY: "${PROVIDER_GATEWAY_ALPINE_REPOSITORY:-}"', ` container_name: unidesk-provider-gateway-${slug}`, " restart: always", ' pid: "host"', diff --git a/scripts/src/ssh-file-transfer.ts b/scripts/src/ssh-file-transfer.ts index 4f1edac9..3c823b27 100644 --- a/scripts/src/ssh-file-transfer.ts +++ b/scripts/src/ssh-file-transfer.ts @@ -91,7 +91,7 @@ class SshFileTransferError extends Error { } const fileTransferWriteB64ArgvLimit = 48_000; -const fileTransferWriteRawChunkBytes = 1_048_576; +const fileTransferWriteRawChunkBytes = 131_072; const fileTransferWriteB64ChunkChars = 1_398_104; const fileTransferProgressEveryChunks = 16; @@ -247,7 +247,7 @@ async function writeRemoteFileVerified( const encoded = content.length <= fileTransferWriteB64ArgvLimit ? content.toString("base64") : ""; - if (invocation.route.plane !== "win" && encoded.length <= fileTransferWriteB64ArgvLimit) { + if (invocation.route.plane !== "win" && encoded.length > 0 && encoded.length <= fileTransferWriteB64ArgvLimit) { await checkedFileTransfer(invocation, executor, builders, "write-b64-argv", [remotePath, expectedBytes, expectedSha256, ...chunkString(encoded, fileTransferWriteB64ChunkChars)]); return { strategy: "argv", chunks: encoded.length === 0 ? 0 : Math.ceil(encoded.length / fileTransferWriteB64ChunkChars) }; } @@ -260,14 +260,18 @@ async function writeRemoteFileVerified( } } const token = `${process.pid}-${Date.now()}-${randomBytes(4).toString("hex")}-${expectedSha256.slice(0, 12)}`; + const startedAtMs = Date.now(); let chunks = 0; await checkedFileTransfer(invocation, executor, builders, "write-b64-begin", [remotePath, token]); for (let offset = 0; offset < content.length; offset += fileTransferWriteRawChunkBytes) { - const encodedChunk = content.subarray(offset, Math.min(content.length, offset + fileTransferWriteRawChunkBytes)).toString("base64"); + const nextOffset = Math.min(content.length, offset + fileTransferWriteRawChunkBytes); + const encodedChunk = content.subarray(offset, nextOffset).toString("base64"); await checkedFileTransfer(invocation, executor, builders, "write-b64-append-stdin", [remotePath, token], encodedChunk); chunks += 1; + emitUploadProgress(invocation, remotePath, chunks, nextOffset, content.length, nextOffset - offset, startedAtMs); } await checkedFileTransfer(invocation, executor, builders, "write-b64-commit", [remotePath, token, expectedBytes, expectedSha256]); + emitUploadProgress(invocation, remotePath, Math.max(1, chunks), content.length, content.length, 0, startedAtMs, true); return { strategy: "chunked-stdin", chunks }; } @@ -383,6 +387,37 @@ function emitDownloadProgress( })}\n`); } +function emitUploadProgress( + invocation: ParsedSshInvocation, + remotePath: string, + chunkCount: number, + actualBytes: number, + expectedBytes: number, + lastChunkBytes: number, + startedAtMs: number, + force = false, +): void { + if (!force && chunkCount !== 1 && actualBytes < expectedBytes && chunkCount % fileTransferProgressEveryChunks !== 0) return; + const elapsedMs = Math.max(1, Date.now() - startedAtMs); + process.stderr.write(`${JSON.stringify({ + event: "unidesk.ssh.upload.progress", + at: new Date().toISOString(), + route: invocation.route.raw, + providerId: invocation.providerId, + remotePath, + strategy: "chunked-stdin", + chunks: chunkCount, + bytes: actualBytes, + totalBytes: expectedBytes, + actualBytes, + expectedBytes, + lastChunkBytes, + remainingBytes: Math.max(0, expectedBytes - actualBytes), + elapsedMs, + throughputBytesPerSecond: Math.round((actualBytes * 1000) / elapsedMs), + })}\n`); +} + async function statRemoteFile( invocation: ParsedSshInvocation, executor: SshRemoteCommandExecutor, diff --git a/scripts/web-probe-sentinel-scheduler.ts b/scripts/web-probe-sentinel-scheduler.ts index 01ed3417..5890bddf 100644 --- a/scripts/web-probe-sentinel-scheduler.ts +++ b/scripts/web-probe-sentinel-scheduler.ts @@ -157,10 +157,10 @@ function sentinelSchedules(spec: ReturnType, } return selectedRows.map((row) => { const sentinel = resolveWebProbeSentinel(spec, row.id); - const publicExposure = record(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure); - const runtime = record(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime); - const cicd = record(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd); - const scenarios = scenarioRows(readConfigRefTarget(sentinel.configRefs.scenarios)); + const publicExposure = record(readConfigRefTarget(sentinel.configRefs.publicExposure, spec), sentinel.configRefs.publicExposure); + const runtime = record(readConfigRefTarget(sentinel.configRefs.runtime, spec), sentinel.configRefs.runtime); + const cicd = record(readConfigRefTarget(sentinel.configRefs.cicd, spec), sentinel.configRefs.cicd); + const scenarios = scenarioRows(readConfigRefTarget(sentinel.configRefs.scenarios, spec)); const enabledScenarios = scenarios.filter((scenario) => scenario.enabled !== false); const scenarioCadences = enabledScenarios .map((scenario) => typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null) diff --git a/src/components/provider-gateway/Dockerfile b/src/components/provider-gateway/Dockerfile index df5bc329..9f694b4f 100644 --- a/src/components/provider-gateway/Dockerfile +++ b/src/components/provider-gateway/Dockerfile @@ -1,4 +1,6 @@ FROM oven/bun:1-alpine +ARG ALPINE_REPOSITORY= +RUN if [ -n "$ALPINE_REPOSITORY" ]; then printf '%s\n' "$ALPINE_REPOSITORY/main" "$ALPINE_REPOSITORY/community" > /etc/apk/repositories; fi RUN apk add --no-cache bash docker-cli docker-cli-compose openssh-client WORKDIR /app/src/components/provider-gateway COPY src/components/provider-gateway/package.json ./package.json