fix: use k8s git mirror source snapshots

This commit is contained in:
Codex
2026-07-01 10:32:05 +00:00
parent 79e419ce98
commit c90ad04bff
19 changed files with 481 additions and 168 deletions
@@ -148,9 +148,9 @@ PipelineRun `gitops-promote` 如果报 git mirror 控制面漂移、refs 不一
node-scoped lane 可能在本次 PR 合并后又被后续 PR 推进。`control-plane status --pipeline-run <name>` 是定点观察某个 PipelineRun,但输出里的当前 `sourceHead` / `summary.sourceCommit` 可能已经是最新 branch tip,而不是该 PipelineRun 名称对应的 merge commit。closeout 必须同时记录 PR merge commit、PipelineRun 名称/状态、Argo sync revision、当前 branch tip,并用 `git merge-base --is-ancestor <merge-commit> HEAD` 或等价证据说明最新 tip 包含本次 PR;不要只凭当前 source head 判断本次 rollout。
`trigger-current --node D601 --lane v03 --confirm --wait` 可能先解析到一个 source commit,随后在 control-plane apply 的 `source-render` / `local-git-clone-worktree` 阶段因 `v0.3` 被并行 PR 推进而 clone 到更新 head,触发 `rev-parse HEAD == source_commit` 校验失败。这不是业务代码构建失败;先执行 `hwlab nodes git-mirror sync --node D601 --lane v03 --confirm --wait`,再快进固定 worktree,确认原 PR merge commit 是最新 `origin/v0.3` 的祖先,然后按最新 branch tip 重新运行 `trigger-current`。收口时同时记录原 PR merge commit、最新 source head 和 ancestor 证据
`trigger-current --node D601 --lane v03 --confirm --wait` 的 source selection 必须走 k8s git-mirror source snapshotconfirmed trigger 先执行受控 `git-mirror sync`,再从 mirror cache 读取 `refs/mirror-stage/heads/<branch>` 作为本轮 source commit。旧 `source-render` / `local-git-clone-worktree` 追 branch tip 的问题不得再用固定 worktree fetch/pull 修复;如果 mirror 缺对象或 GitHub SSH transient 耗尽,命令应以 `source-snapshot-missing` 或 git-mirror retry exhausted 类故障停止,并给出受控 sync/status 下一步
D601 `v0.3` 固定 worktree 的 fetch remote 是 node-local git mirror。GitHub PR 合并后,如果 `D601:/home/ubuntu/workspace/hwlab-v03``git fetch origin v0.3` 仍看不到最新 merge commit,先执行 `hwlab nodes git-mirror sync --node D601 --lane v03 --confirm --wait`,再在固定 worktree `git fetch origin v0.3 && git pull --ff-only origin v0.3`D601/v03 `git-mirror` 的 GitHub upstream 标准传输固定为 YAML 声明的 SSH:`githubTransport.mode=ssh`,脚本通过 `GIT_SSH` wrapper 访问 `ssh://git@ssh.github.com:443/...`node-global HTTP proxy 只作为 SSH CONNECT tunnel,不是 GitHub HTTPS auth/token transport。若 CLI 输出 `transport=https``GITHUB_TOKEN``git-mirror-github-token` 或 HTTPS token sourceRef,按 control-plane drift/配置回归处理:先修 `config/hwlab-node-control-plane.yaml` 并执行 `hwlab nodes control-plane apply --node D601 --lane v03 --confirm`,不要改走 HTTPS、不要增加 fallback、不要用 host workspace repair。`sync/flush` 的 retry 只消费 SSH upstream transient,并在耗尽后输出 stopped/exhausted`trigger-current --lane v03` 会为 PipelineRun 做 mirror pre-sync,但不替代固定 worktree 的 fetch hygiene。promotion 后若 node-local `git-mirror status` 显示 `pendingFlush=true`,执行 node-local flush 并等到 `pendingFlush=false``githubInSync=true`
D601/v03 `git-mirror` 的 GitHub upstream 标准传输固定为 YAML 声明的 SSH:`githubTransport.mode=ssh`,脚本通过 `GIT_SSH` wrapper 访问 `ssh://git@ssh.github.com:443/...`node-global HTTP proxy 只作为 SSH CONNECT tunnel,不是 GitHub HTTPS auth/token transport。若 CLI 输出 `transport=https``GITHUB_TOKEN``git-mirror-github-token` 或 HTTPS token sourceRef,按 control-plane drift/配置回归处理:先修 `config/hwlab-node-control-plane.yaml` 并执行 `hwlab nodes control-plane apply --node D601 --lane v03 --confirm`,不要改走 HTTPS、不要增加 fallback、不要用 host workspace repair。`sync/flush` 的 retry 只消费 SSH upstream transient,并在耗尽后输出 stopped/exhaustedpromotion 后若 node-local `git-mirror status` 显示 `pendingFlush=true`,执行 node-local flush 并等到 `pendingFlush=false``githubInSync=true`
D601/node-scoped mirror status 的 `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...``status` 输出应通过 `refSources.githubFieldsAreMirrorStageCache=true` 显示这一点。`hwlab nodes git-mirror flush --node D601 --lane v03 --confirm --wait` 如果已经显示 `v0.3-gitops -> v0.3-gitops` 推送成功,但随后因 GitHub SSH `kex_exchange_identification` 或 fetch 确认失败导致命令非零退出,会标记 `partialSuccess=push-succeeded-fetch-failed`。当前 CLI 会自动做一次受控 sync/recheck;恢复后输出 `partialSuccessRecovered=true``postPushRecovery` 且整体 `ok=true`,未恢复时才把下一步指向 `hwlab nodes git-mirror sync --node D601 --lane v03 --confirm --wait`。不要连续盲目 flush;先刷新 mirror-stage,再用 status 确认 `localGitops=githubGitops``pendingFlush=false``githubInSync=true`
+1
View File
@@ -31,6 +31,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台。本文
- P0: 用户最新明确要求优先于旧测试、旧门禁、旧预检、旧断言和旧 guard;阻碍最新目标的旧入口应删除,不做兼容保留。
- P0: 可见性问题优先修复;状态、耗时、失败原因、trace、命令结果或关键证据不可见时,先补 CLI/日志/状态输出再继续。细则见 `docs/reference/observability.md`
- P0: CLI 改动默认不做单元测试、合同测试或新增测试脚本;除非用户明确要求,最多做语法检查和必要命令形态确认。
- P0: 不得用裸 `tsc`/`bun --bun tsc` 当语法验证;它会长时间卡住。CLI/TS 轻量验证用 `bun --check <具体文件>``bun scripts/cli.ts check` 默认 syntax transpile 和必要命令形态确认;细则见 `docs/reference/cli.md`
- P0: CLI 默认输出应是 Kubernetes 风格的简洁表格、短摘要和 drill-down 命令;JSON 只用于 `--json``--raw``--full``-o json` 或机器消费。超长输出 dump 是兜底,不是长期交互入口。细则见 `docs/reference/cli.md`
## P0: 配置、Secret 与 YAML-first
@@ -22,6 +22,16 @@ sentinel:
- package.json
- bun.lock
- bun.lockb
sourceAuthority:
mode: gitMirrorSnapshot
resolver: k8s-git-mirror
allowHostGit: false
allowGithubDirectInPipeline: false
sourceSnapshot:
stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch}
missingObjectPolicy: fail-fast
refreshPolicy: sync-before-snapshot
cacheRef: source.gitMirrorReadUrl
builder:
namespace: devops-infra
sourceMode: sparse-git-checkout
@@ -22,6 +22,16 @@ sentinel:
- package.json
- bun.lock
- bun.lockb
sourceAuthority:
mode: gitMirrorSnapshot
resolver: k8s-git-mirror
allowHostGit: false
allowGithubDirectInPipeline: false
sourceSnapshot:
stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch}
missingObjectPolicy: fail-fast
refreshPolicy: sync-before-snapshot
cacheRef: source.gitMirrorReadUrl
builder:
namespace: devops-infra
sourceMode: sparse-git-checkout
@@ -22,6 +22,16 @@ sentinel:
- package.json
- bun.lock
- bun.lockb
sourceAuthority:
mode: gitMirrorSnapshot
resolver: k8s-git-mirror
allowHostGit: false
allowGithubDirectInPipeline: false
sourceSnapshot:
stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch}
missingObjectPolicy: fail-fast
refreshPolicy: sync-before-snapshot
cacheRef: source.gitMirrorReadUrl
builder:
namespace: devops-infra
sourceMode: sparse-git-checkout
@@ -23,6 +23,16 @@ sentinel:
- package.json
- bun.lock
- bun.lockb
sourceAuthority:
mode: gitMirrorSnapshot
resolver: k8s-git-mirror
allowHostGit: false
allowGithubDirectInPipeline: false
sourceSnapshot:
stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch}
missingObjectPolicy: fail-fast
refreshPolicy: sync-before-snapshot
cacheRef: source.gitMirrorReadUrl
builder:
namespace: devops-infra
sourceMode: sparse-git-checkout
@@ -22,6 +22,16 @@ sentinel:
- package.json
- bun.lock
- bun.lockb
sourceAuthority:
mode: gitMirrorSnapshot
resolver: k8s-git-mirror
allowHostGit: false
allowGithubDirectInPipeline: false
sourceSnapshot:
stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch}
missingObjectPolicy: fail-fast
refreshPolicy: sync-before-snapshot
cacheRef: source.gitMirrorReadUrl
builder:
namespace: devops-infra
sourceMode: sparse-git-checkout
@@ -57,6 +57,16 @@ baselines:
- package.json
- bun.lock
- bun.lockb
sourceAuthority: &cicd-source-authority
mode: gitMirrorSnapshot
resolver: k8s-git-mirror
allowHostGit: false
allowGithubDirectInPipeline: false
sourceSnapshot: &cicd-source-snapshot
stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch}
missingObjectPolicy: fail-fast
refreshPolicy: sync-before-snapshot
cacheRef: source.gitMirrorReadUrl
builder: &cicd-builder
namespace: devops-infra
sourceMode: sparse-git-checkout
@@ -159,6 +169,10 @@ nodes:
controlPlaneConfigRef: config/hwlab-node-control-plane.yaml#targets[1]
source:
<<: *cicd-source
sourceAuthority:
<<: *cicd-source-authority
sourceSnapshot:
<<: *cicd-source-snapshot
argo: &jd01-argo
namespace: argocd
projectName: hwlab-jd01
+2 -2
View File
@@ -34,7 +34,7 @@ G14/D601 v03 的 bootstrap admin password 是 HWLAB runtime Secret 生命周期
`hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile <profile> --confirm` 是 HWLAB v0.3 k3s CI/CD 全量无缓存构建出网测速入口,profile、cache policy、独立 catalog path 模板、PipelineRun prefix、必须输出的 timing 阶段和失败族都来自 `config/hwlab-node-control-plane.yaml`。confirmed benchmark 只创建一次唯一 PipelineRun,使用 node-lane YAML 中的实际 HWLAB v0.3 service set、git mirror read/write URL、registry prefix、base image 和 Tekton pipeline`forbidBuildkitCache=true` 时会向 PipelineRun 传 `build-cache-mode=disabled`。status/logs 通过短连接轮询 PipelineRun/TaskRun 摘要和有界日志。成功的 benchmark 必须出现每个 `build-<service>` TaskRun;如果 PipelineRun 成功但缺少任一 service build taskCLI 必须把该 service 报为 `cache-hit-forbidden`,不能把 catalog/env reuse 当作 #1010 这类性能验收的通过证据。
`hwlab nodes git-mirror status|sync|flush --node <node> --lane <lane>` 是 node-scoped runtime lane 的 Git mirror 维护入口。`status``githubSource` / `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...`,不是实时 GitHub API;输出中的 `refSources.githubFieldsAreMirrorStageCache=true``refSources.cacheRefresh` 给出这一来源和刷新命令。`sync --confirm --wait` 的 k3s Job 遇到 GitHub SSH transient 时,应通过目标 workspace fallback 拉取 GitHub source/gitops 并写回 node-local mirror,输出只披露 commit、mirror write URL 和 fallback 状态`flush --confirm --wait` 如果已经把 GitOps ref push 到 GitHub,但 post-push fetch/recheck 因 transient SSH 失败而无法刷新 mirror-stage,会标记 `partialSuccess=push-succeeded-fetch-failed`;CLI 应自动执行一次受控 sync 刷新 mirror-stage,若恢复后 `pendingFlush=false``githubInSync=true`,结果应为 `ok=true` 并输出 `partialSuccessRecovered` / `postPushRecovery`,否则才保留 `degradedReason=node-runtime-git-mirror-flush-post-push-fetch-failed` 和下一步 `sync --confirm --wait`。不要把这种 partial success 解读为需要连续盲目 flush。`hwlab nodes control-plane trigger-current --node <node> --lane <lane> --confirm --wait`在 source sync 后自动执行必要的 pre-flush,在 PipelineRun terminal 后自动执行必要的 post-flushprogress 事件必须显式输出 `git-mirror-pre-flush` / `git-mirror-post-flush` 的 executed/skipped、jobName、local/github source、local/github GitOps、`pendingFlush``githubInSync`,且已恢复的 partial success 不能让顶层 trigger-current false-fail。`control-plane status` 仍是只读入口,只暴露 compact `gitMirror` 摘要和下一步 flush 命令,不隐式执行写操作。
`hwlab nodes git-mirror status|sync|flush --node <node> --lane <lane>` 是 node-scoped runtime lane 的 Git mirror 维护入口。`status``githubSource` / `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...`,不是实时 GitHub API;输出中的 `refSources.githubFieldsAreMirrorStageCache=true``refSources.cacheRefresh` 给出这一来源和刷新命令。`sync --confirm --wait` 的 k8s Job 是 upstream GitHub fetch 的唯一正式入口;遇到 GitHub SSH transient 时按受控 retry/backoff 停止并输出下一步,不回退到 operator host git、目标 host fixed workspace 或第二套 source resolver`flush --confirm --wait` 如果已经把 GitOps ref push 到 GitHub,但 post-push fetch/recheck 因 transient SSH 失败而无法刷新 mirror-stage,会标记 `partialSuccess=push-succeeded-fetch-failed`;CLI 应自动执行一次受控 sync 刷新 mirror-stage,若恢复后 `pendingFlush=false``githubInSync=true`,结果应为 `ok=true` 并输出 `partialSuccessRecovered` / `postPushRecovery`,否则才保留 `degradedReason=node-runtime-git-mirror-flush-post-push-fetch-failed` 和下一步 `sync --confirm --wait`。不要把这种 partial success 解读为需要连续盲目 flush。`hwlab nodes control-plane trigger-current --node <node> --lane <lane> --confirm --wait`先执行 k8s git-mirror source snapshot sync,再从 mirror cache 选择 source commit,随后自动执行必要的 pre-flush在 PipelineRun terminal 后自动执行必要的 post-flushprogress 事件必须显式输出 `git-mirror-pre-flush` / `git-mirror-post-flush` 的 executed/skipped、jobName、local/github source、local/github GitOps、`pendingFlush``githubInSync`,且已恢复的 partial success 不能让顶层 trigger-current false-fail。`control-plane status` 仍是只读入口,只读 k8s mirror cache 并暴露 compact `gitMirror` 摘要和下一步 flush 命令,不隐式执行写操作。
PR 合并后触发 node-scoped runtime lane 时,`control-plane status --pipeline-run <name>` 是某次 PipelineRun 的定点观察入口,但同一输出中的 `sourceHead` / `summary.sourceCommit` 仍可能反映当前分支最新 head;如果触发后又有后续 PR 合并,当前 head 可能已经不是该 PipelineRun 名称中的短 SHA。closeout 证据必须同时写明:PR merge commit、定点 PipelineRun 名称和状态、最终 runtime/GitOps revision、当前 branch tip,以及当前 branch tip 是否包含本次 PR merge commit。不要只凭 `summary.sourceCommit` 反推某个旧 PipelineRun 的源码身份。
@@ -54,7 +54,7 @@ PipelineRun 失败或长时间未完成时,先按定点 `control-plane status
- 每个 CLI 命名空间必须支持 `help``--help``-h` 并返回 JSON,不得为了打印帮助而访问 runtime 服务、拉起交互会话或执行长时任务。
- `--main-server-ip <ip> <command>` 默认通过公网 frontend 登录态调用主 server 的同源 API 代理,不要求计算节点持有主 server SSH key;显式提供 `--main-server-key``--main-server-transport ssh` 时才使用旧 SSH 传输。远程 frontend 传输下的 `ssh <route> ...` 必须复用同一套结构化 route parser,支持 `D601``G14`、host workspace、`D601:win``D601:win/c/test``D601:k3s``D601:k3s:<namespace>:<workload>` 这类定位路径;它不向调用容器下发 provider token,也不要求调用容器能解析 backend-core 内网 DNS。
- `config show` 读取并校验根目录 `config.json`,不从环境变量、默认值或隐藏文件静默补配置。
- `check` 默认只执行轻量配置校验、Bun 版本检查和 Bun Transpiler 语法解析(覆盖 CLI 入口、主要 `scripts/` 模块和核心组件入口,不做类型推导)。除非用户明确要求,CLI 改动不运行单元测试、合同测试或新增测试脚本;默认最多做语法检查和必要的帮助/命令形态人工确认。关键文件存在性、`scripts/` TypeScript 类型检查、`src/components/` TypeScript 类型检查、Docker Compose config、日志轮转策略扫描和 D601 recovery guardrails 默认不启用,分别通过 `--files``--scripts-typecheck``--components``--compose``--logs``--recovery-guardrails` 开启,或用 `--full` 一次性开启。`--scripts-typecheck` 只跑 scripts TypeScript 类型检查,不触发测试脚本或 GitHub issue/PR live API check。长命令项必须在 stderr 输出 `unidesk.check.progress` JSON linesstdout 保持最终 JSON 结果,避免 post-task 或人工运行时长时间无可见进度。`typescript:scripts` 固定通过 `bun --bun tsc -p scripts/tsconfig.json --noEmit --pretty false` 执行,默认 `--scripts-typecheck-timeout-ms 120000`,可按目标运行面显式调小或调大但 CLI 会封顶;`--check-heartbeat-ms` 控制运行中心跳间隔,默认 `15000`。所有命令项的最终 item detail 必须包含 `durationMs``timeoutMs``heartbeatMs``exitCode``signal``timedOut`、stdout/stderr byte count、truncation flag 和有界 tail;超时必须返回 `timedOut=true`,不得只留下被外层命令杀死的空输出。不要把 `bun --check scripts/cli.ts` 当作低噪声 CLI 自检入口;它可能执行根 CLI help 并触发长 help dump。CLI 入口级自检使用 `bun scripts/cli.ts check`,单文件语法确认只针对具体模块文件运行。`check recovery-guardrails` 是同一诊断的低噪声直接入口,报告 malformed `/proc/mounts`、kubelet validation risk、stale CRI sandbox count、Code Queue worktree/symlink、Code Queue/MDTODO hostPath 和 `ContainerCreating` 分类;它不得重启 k3s、删除 CRI sandbox、修改 hostPath、deploy/rollout 或 prune/reset。`--rust` 只允许在 D601 CI/dev execution 中配合 `UNIDESK_D601_RUST_CHECK=1` 使用,长期规则见 `docs/reference/dev-environment.md``docs/reference/devops-hygiene.md`
- `check` 默认只执行轻量配置校验、Bun 版本检查和 Bun Transpiler 语法解析(覆盖 CLI 入口、主要 `scripts/` 模块和核心组件入口,不做类型推导)。除非用户明确要求,CLI 改动不运行单元测试、合同测试或新增测试脚本;默认最多做语法检查和必要的帮助/命令形态人工确认。关键文件存在性、`scripts/` TypeScript 类型检查、`src/components/` TypeScript 类型检查、Docker Compose config、日志轮转策略扫描和 D601 recovery guardrails 默认不启用,分别通过 `--files``--scripts-typecheck``--components``--compose``--logs``--recovery-guardrails` 开启,或用 `--full` 一次性开启。不得把裸 `tsc` / `bun --bun tsc``check --scripts-typecheck` 当作语法验证;它是重型类型检查,可能长时间卡住或超时。单文件语法确认使用 `bun --check <具体文件>`,入口级轻量自检使用 `bun scripts/cli.ts check` 默认 syntax transpile。`--scripts-typecheck` 只跑 scripts TypeScript 类型检查,不触发测试脚本或 GitHub issue/PR live API check。长命令项必须在 stderr 输出 `unidesk.check.progress` JSON linesstdout 保持最终 JSON 结果,避免 post-task 或人工运行时长时间无可见进度。`typescript:scripts` 固定通过 `bun --bun tsc -p scripts/tsconfig.json --noEmit --pretty false` 执行,默认 `--scripts-typecheck-timeout-ms 120000`,可按目标运行面显式调小或调大但 CLI 会封顶;`--check-heartbeat-ms` 控制运行中心跳间隔,默认 `15000`。所有命令项的最终 item detail 必须包含 `durationMs``timeoutMs``heartbeatMs``exitCode``signal``timedOut`、stdout/stderr byte count、truncation flag 和有界 tail;超时必须返回 `timedOut=true`,不得只留下被外层命令杀死的空输出。不要把 `bun --check scripts/cli.ts` 当作低噪声 CLI 自检入口;它可能执行根 CLI help 并触发长 help dump。CLI 入口级自检使用 `bun scripts/cli.ts check`,单文件语法确认只针对具体模块文件运行。`check recovery-guardrails` 是同一诊断的低噪声直接入口,报告 malformed `/proc/mounts`、kubelet validation risk、stale CRI sandbox count、Code Queue worktree/symlink、Code Queue/MDTODO hostPath 和 `ContainerCreating` 分类;它不得重启 k3s、删除 CRI sandbox、修改 hostPath、deploy/rollout 或 prune/reset。`--rust` 只允许在 D601 CI/dev execution 中配合 `UNIDESK_D601_RUST_CHECK=1` 使用,长期规则见 `docs/reference/dev-environment.md``docs/reference/devops-hygiene.md`
- `server start` 创建异步 job,在后台执行 Docker 构建和启动;命令默认只返回低噪声 async job 摘要、stdout/stderr 路径和 `job status` 后续命令,完整 JSON 只能通过 `--full`/`--raw` 显式展开。
- `server stop` 创建异步 job,在后台停止固定 Compose project 中的全部 UniDesk 服务;默认输出同样是 async job 摘要。
- `server status` 查询公开端口、受限宿主端口、内部端口、主机 swap 摘要、Compose 容器、core/frontend/dev-frontend/provider/database 健康检查和访问 URLD601 Code Queue 使用的 PostgreSQL/OA Event Flow host mapping 必须出现在受限宿主端口而不是无条件公开入口中。低内存主 server 上 `swap.warning` 非空时,先执行 `server swap status``server swap ensure`
@@ -22,7 +22,7 @@
| 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) |
| 上级规格 | [PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md) |
| 关联规格 | [PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md)、[PJ2026-010602 源码同步](PJ2026-010602-source-sync.md)、[PJ2026-010604 公开入口](PJ2026-010604-public-entry.md)、[PJ2026-010605 运维监控](PJ2026-010605-observability-monitoring.md) |
| 实现引用版本 | draft-2026-06-25-p0 |
| 实现引用版本 | draft-2026-06-25-p0; draft-2026-07-01-cicd-source-snapshot |
| 规格治理索引 | [规格治理](spec-governance.md) |
本文采用 ISO/IEC/IEEE 29148 需求规格模板的项目裁剪版:正文只保留 CI/CD、YAML-first 和平台运维 CLI 控制面源码拆分的稳定目标、边界、架构、阶段和验收口径。
@@ -31,7 +31,7 @@
### 2.1 目的
控制面模块化负责把 UniDesk CI/CD、YAML-first 和平台运维 CLI 中的超大 TypeScript 入口拆成可维护的领域模块,使命令路由、配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、Tekton/Argo、status summary 和 bounded output 各有明确归属。
控制面模块化负责把 UniDesk CI/CD、YAML-first 和平台运维 CLI 中的超大 TypeScript 入口拆成可维护的领域模块,使命令路由、配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、source snapshot resolver、Tekton/Argo、status summary 和 bounded output 各有明确归属。
本规格的直接治理对象是超过 3000 行的控制面入口文件。第一阶段先做机械拆分和兼容入口保留,第二阶段再收紧 import/export、模块边界和共享 helper;不得在规格未落地前直接进行行为重写。
@@ -39,7 +39,7 @@
- `scripts/src/hwlab-g14.ts``scripts/src/hwlab-node-impl.ts``scripts/src/platform-infra-sub2api-codex.ts``scripts/src/agentrun.ts``scripts/src/platform-infra.ts``scripts/src/deploy.ts``scripts/src/artifact-registry.ts``scripts/src/ci.ts``scripts/src/platform-infra-observability.ts` 的职责拆分。
- 领域子目录的建立、兼容 re-export 入口、机械搬迁追溯标注和 public API 保持。
- 配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、Tekton/Argo、status summary、bounded output 和低层 helper 的模块边界。
- 配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、source snapshot resolver、Tekton/Argo、status summary、bounded output 和低层 helper 的模块边界。
- 与 YAML-first source of truth、发布流水、源码同步、公开入口和运维监控规格的一致性。
- `bun scripts/cli.ts <domain> --help`、plan/status/validate/dry-run 等原入口 smoke 验证。
@@ -67,7 +67,7 @@
| 边界项 | 内容 |
| --- | --- |
| 外部使用者 | 平台维护者、发布操作人员、自动化任务和后续代码维护者。 |
| 外部输入 | CLI 命令、YAML 配置、node/lane/target 参数、Secret sourceRef、Git commit 和运行面状态查询请求。 |
| 外部输入 | CLI 命令、YAML 配置、node/lane/target 参数、Secret sourceRef、source snapshot、Git commit 和运行面状态查询请求。 |
| 受控资源 | `scripts/src` 控制面源码、领域子目录、共享 helper、兼容入口、CLI 输出摘要和原入口命令。 |
| 外部输出 | 原 CLI 命令结果、help、plan/status/validate 摘要、dry-run 计划和 redacted 错误。 |
| 用户接口 | 既有 `bun scripts/cli.ts hwlab g14 ...``hwlab nodes ...``agentrun ...``platform-infra ...``ci ...``deploy ...``artifact-registry ...`。 |
@@ -86,10 +86,12 @@ flowchart LR
DomainIndex --> Actions[plan apply status actions]
Actions --> Render[manifest and summary renderers]
Actions --> Remote[remote script builders and capture]
Actions --> Source[git-mirror source snapshot resolver]
Actions --> Secret[Secret/public exposure/git-mirror helpers]
Config --> Ops[common ops primitives]
Render --> Ops
Remote --> Ops
Source --> Ops
Secret --> Ops
Ops --> Runtime[target runtime or dry-run evidence]
```
@@ -163,7 +165,7 @@ CI/CD、YAML-first 和平台运维控制面文件超过 3000 行时,必须进
| --- | --- | --- | --- |
| OPS-MOD-REQ-004 | 模块边界 | PJ2026-01060307 控制面模块化 | [公开入口](PJ2026-010604-public-entry.md)、[运维监控](PJ2026-010605-observability-monitoring.md) |
第二阶段应把 import/export 和依赖方向收敛到稳定边界:options/config 只解析输入,manifest/render 只生成输出,remote/script builder 只封装远端执行文本,Secret/public exposure/git-mirror/Tekton/Argo/status summary 分别归属领域模块或公共 helper。
第二阶段应把 import/export 和依赖方向收敛到稳定边界:options/config 只解析输入,manifest/render 只生成输出,remote/script builder 只封装远端执行文本,source snapshot resolver、Secret/public exposure/git-mirror/Tekton/Argo/status summary 分别归属领域模块或公共 helper。
若多个领域重复实现同一机制,应优先抽到公共 helper;若差异是业务领域真实差异,应保留在领域 adapter 中并在 closeout 说明为 `keep-domain-special`
@@ -187,6 +189,18 @@ Secret 相关模块只能输出对象名、key 名、sourceRef、presence、fing
回滚策略是通过 Git revert 恢复模块边界,不通过复制旧文件、手工 patch 运行面或保留并行 legacy 入口实现长期双路径。若模块化后发现行为回归,应先修复 import/export 或职责边界,再决定是否回退。
### 6.7 OPS-MOD-REQ-007 source plane 单一路径
| 编号 | 短名 | 主责模块 | 关联模块 |
| --- | --- | --- | --- |
| OPS-MOD-REQ-007 | source plane | PJ2026-01060307 控制面模块化 | [YAML目标治理](PJ2026-01060308-cicd-yaml-first-target-governance.md)、[源码同步](PJ2026-010602-source-sync.md) |
控制面模块必须把 source selection、mirror sync、snapshot creation、PipelineRun source acquisition 和 GitOps publish 明确拆分并串成单一路径。正式入口只能先通过 YAML 解析目标,再通过 k8s git-mirror resolver 得到 source snapshot,随后把同一 snapshot 注入 PipelineRun、status 和 closeout。
不得在同一正式流程中并存 operator host `git ls-remote`、固定 host worktree clone/fetch、node-host shared worktree、PipelineRun mutable branch fetch 和 GitHub direct fetch 等多个 source resolver。历史 local render 或 host workspace 逻辑若暂时保留,必须标注为显式 legacy/debug adapter,不得由 `trigger-current`、Web 哨兵 publish 或 runtime rollout 默认调用。
PipelineRun/status 模块必须把 selected snapshot 与 latest branch/mirror head 分开展示。source snapshot 缺对象、mirror 追新、GitOps flush pending、Argo progressing 和 runtime readiness 是不同阶段,不得用一个 `sourceHead` 字段覆盖全部判断。
## 7. 过程控制
本规格的执行 issue 为 [#903](https://github.com/pikasTech/unidesk/issues/903)。源码文件头部应标注 `SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0`,并说明文件是薄入口、领域模块还是公共 helper。纯配置、锁文件、生成产物和无法承载注释头的二进制文件可例外。
@@ -22,7 +22,7 @@
| 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) |
| 上级规格 | [PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md) |
| 关联规格 | [PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md)、[PJ2026-010602 源码同步](PJ2026-010602-source-sync.md)、[PJ2026-01060307 控制面模块化](PJ2026-01060307-control-plane-modularity.md)、[PJ2026-010604 公开入口](PJ2026-010604-public-entry.md)、[PJ2026-010605 运维监控](PJ2026-010605-observability-monitoring.md) |
| 实现引用版本 | draft-2026-06-25-cicd-yaml-targets |
| 实现引用版本 | draft-2026-06-25-cicd-yaml-targets; draft-2026-07-01-cicd-source-snapshot |
| 规格治理索引 | [规格治理](spec-governance.md) |
本文采用 ISO/IEC/IEEE 29148 需求规格模板的项目裁剪版:正文只保留 CI/CD、YAML-first 和平台运维控制面中 target、lane、Secret、public exposure、PipelineRun/Argo/git-mirror 与公共 ops primitive 的稳定治理口径。
@@ -58,6 +58,8 @@ YAML目标治理负责把 CI、deploy、artifact-registry、HWLAB node/lane、Ag
| lane | YAML 中描述一条 runtime 或 control-plane 运行线的对象,通常包含 node、source truth、runtime namespace、gitops、CI、Secret 和 publicExposure。 |
| configRef | `path/to/file.yaml#object.path` 形式的跨 YAML 引用,解析器只校验引用存在性、类型和摘要,不把合并结果写成第二真相。 |
| sourceRef | Secret 来源声明。输出只能显示来源标识、key 名、presence、fingerprint、字节数和 `valuesPrinted=false`。 |
| source snapshot | CI/CD 触发时由 k8s 托管 git-mirror 解析出的不可变源码快照,至少包含 repository、branch、selectedCommit、stageRef、mirrorCommit、objectPresent、resolver job 和 YAML 来源。 |
| host workspace | operator host 或目标节点上的固定源码 checkout。正式 CI/CD 只能把它作为开发便利或显式 legacy/debug adapter,不得作为发布 source truth。 |
| hidden default | 代码在未显式参数或未读取 YAML 时自动选择固定目标、namespace、route、registry、lane 或工作目录。 |
| legacy adapter | 为历史入口保留的隔离兼容路径。它可以拒绝 mutation 或提示替代命令,但不得污染主 runtime lane 解析。 |
| ops primitive | 跨领域复用的配置引用、目标解析、Secret redaction、K8s apply/status、public exposure 和 bounded output helper。 |
@@ -161,6 +163,26 @@ sequenceDiagram
状态输出必须区分 PR merge commit、current source head、PipelineRun、Argo revision、git-mirror pendingFlush 和 runtime readiness;不得只用一个当前 head 推断所有阶段完成。
### 5.6 k8s source snapshot 链路
```mermaid
sequenceDiagram
participant CLI
participant Target as YAML target
participant Resolver as k8s git-mirror resolver
participant Mirror as git-mirror cache
participant Tekton
CLI->>Target: resolve node/lane/sourceAuthority
CLI->>Resolver: ensure or read source snapshot
Resolver->>Mirror: sync/read selected commit and create stageRef
Mirror-->>Resolver: selectedCommit + stageRef + objectPresent
Resolver-->>CLI: sourceSnapshot summary
CLI->>Tekton: create PipelineRun with sourceSnapshot
Tekton->>Mirror: fetch/checkout immutable stageRef or snapshot artifact
```
正式 CI/CD 的源码身份以 source snapshot 为准。`trigger-current`、image publish、GitOps render 和 closeout 不得再从 operator host 的 dirty/stale worktree、当前 branch tip 或 GitHub direct fetch 推导本轮 source identity。upstream fetch 只允许发生在受控 k8s git-mirror resolver/sync 组件内;PipelineRun 只消费 node-local mirror 的不可变 stageRef、PVC 或 bundle artifact。
## 6. 原子需求
### 6.1 OPS-TARGET-REQ-001 YAML target 真相
@@ -217,6 +239,18 @@ D601 maintenance deploy、G14 DEV/PROD retirement、v02-only runtime migration
验收至少覆盖 `ci plan/status --target <id>``artifact-registry plan/status --target <id>``deploy apply --dry-run` 目标来源说明、`hwlab nodes control-plane status --node <node> --lane <lane>` 的 YAML-declared lane、`platform-infra observability plan/status --full` 的 configRef 解析链和 `secrets status --config ... --full` 的脱敏输出。
### 6.7 OPS-TARGET-REQ-007 source snapshot 与 host worktree 禁用
| 编号 | 短名 | 主责模块 | 关联模块 |
| --- | --- | --- | --- |
| OPS-TARGET-REQ-007 | 源码快照 | PJ2026-01060308 YAML目标治理 | [源码同步](PJ2026-010602-source-sync.md)、[发布流水](PJ2026-010601-controlled-release.md) |
CI/CD source authority 必须由 owning YAML 声明。主路径应支持 `sourceAuthority.mode=gitMirrorSnapshot` 或等价字段,使受控 CLI 通过 k8s git-mirror resolver 选择本轮 `selectedCommit` 并创建不可变 snapshot/stageRef。`sourceAuthority.allowHostGit=false` 时,正式 `trigger-current`、image publish、GitOps render 和 closeout 不得执行 operator host `git ls-remote``git fetch``git clone``git rev-parse HEAD` 或读写固定 host workspace。
PipelineRun source step 不得以 mutable `refs/heads/<branch>` 作为本轮源码身份。它只能从 source snapshot 的 `stageRef`、commit-pinned mirror ref、PVC 或 bundle artifact checkout;若对象缺失,应返回 `source-snapshot-missing`,并输出受控 `git-mirror sync/ensure-snapshot` 下一步,不得回退到 GitHub direct clone、host workspace repair 或第二套 source resolver。
status/closeout 必须同时展示 selected snapshot、PipelineRun snapshot、latest mirror head、GitOps revision 和 runtime provenance。branch 在发布过程中前进只能标记为 `latest-drift``superseded`,不得把 selected commit 的成功发布误判为本轮失败。
## 7. 过程控制
本规格的执行 issue 为 [#911](https://github.com/pikasTech/unidesk/issues/911)。源码文件头部应标注 `SPEC: PJ2026-01060308 cicd-yaml-targets draft-2026-06-25-cicd-yaml-targets`;自动生成文件、纯配置、锁文件和无法承载注释头的二进制产物可例外。
@@ -25,6 +25,7 @@
| Monitor Web 聚合实现引用版本 | draft-2026-06-26-p10-monitor-web-aggregation |
| Monitor Web 观察面板治理实现引用版本 | draft-2026-06-27-p11-monitor-web-observability-dashboard; draft-2026-06-27-p12-cadence-scheduler-monitor-web |
| Cadence/OTel 稳定性实现引用版本 | draft-2026-07-01-p15-cadence-otel |
| CI/CD source snapshot 实现引用版本 | draft-2026-07-01-p16-cicd-source-snapshot |
| 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) |
| 上级规格 | [PJ2026-010605 运维监控](PJ2026-010605-observability-monitoring.md) |
| 关联规格 | [PJ2026-010401 Web工作台](PJ2026-010401-web-workbench.md)、[PJ2026-0104010803 Workbench唯一投影](PJ2026-0104010803-workbench-unique-projection.md)、[PJ2026-010403 API契约](PJ2026-010403-api-contract.md)、[PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md)、[PJ2026-010602 源码同步](PJ2026-010602-source-sync.md)、[PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md)、[PJ2026-010604 公开入口](PJ2026-010604-public-entry.md)、[PJ2026-01060505 Workbench性能](PJ2026-01060505-workbench-performance.md) |
@@ -49,6 +50,7 @@ Web哨兵必须遵循 UniDesk YAML-first ops。目标 node/lane、public origin
- 常驻 TypeScript 单 Pod wrapper 服务、scheduler、scenario runner、PVC/SQLite index、health、metrics、maintenance API 和 dashboard。
- `sentinel plan|apply|status|validate|report|maintenance``sentinel image|control-plane` 等受控 CLI 入口。
- 发布流水 maintenance start/stop、quick verify、targetValidation、GitOps/Argo/git-mirror closeout 和 public exposure 验证。
- 哨兵自身 CI/CD 的 k8s git-mirror source snapshot、PipelineRun source acquisition、GitOps publish 和 selected/latest closeout。
- Dashboard 信息架构、规范化 API、前端组件分层、自动刷新、筛选、深链和 trace/turn 两层阅读视图。
- Vue `monitor-web` 观察面板的趋势曲线、固定视口三栏、运行时间线、cadence freshness、root cause 可见性和配套 CI/CD。
- `workbench-dsflash-go-tool-call-10x` 生产 canary 和 24 小时 dry-run 收口。
@@ -93,6 +95,7 @@ Web哨兵必须遵循 UniDesk YAML-first ops。目标 node/lane、public origin
| auto refresh | Dashboard 的受控刷新能力;刷新只读 API/report/index,不发送 control command、不启动采样、不制造第二事实源。 |
| public exposure | YAML 声明的 `monitor.pikapython.com` HTTPS 暴露,通过共享 PK01 Caddy + FRP managed-block helper 到达 ClusterIP Service。 |
| targetValidation | 发布流水中的目标验证结果;对 HWLAB Web 恢复的判定必须来自 observe/analyze 对 public origin 的观察,不得只看 Argo green。 |
| source snapshot | 哨兵自身 CI/CD 触发时由 k8s git-mirror 解析出的不可变源码快照,绑定 selected commit、mirror ref、PipelineRun、GitOps revision 和 runtime image。 |
| monitor-web | 独立于 sentinel-runner 的 Vue 3 + TypeScript + Vite 展示和聚合层,承载 `monitor.pikapython.com` root、多哨兵总览、趋势曲线、时间线、单哨兵详情和受控截图验收。 |
| cadence freshness | 根据 YAML cadence、scheduler heartbeat、latest run age、active/planned run 和 analyzed report 更新时间计算的运行新鲜度;它默认是非阻塞值守告警,只有真正导致 run/report 不产生或业务链路不可用时才升级为 blocker。 |
| env reuse | CI/CD 复用既有 env image、依赖缓存、BuildKit 层和未受影响服务的发布产物,以避免无关重建;小范围变更应在 status/closeout 中暴露 build/reuse 摘要。 |
@@ -515,6 +518,10 @@ P8 恢复判定必须把 Workbench 业务失败继续 drill-down 到运行面依
Web哨兵自身必须纳入受控且独立的 sentinel control-planesource 来自 UniDesk `master`,镜像、GitOps path、Argo Application、publicExposure 和 targetValidation 由 Web 哨兵 owning YAML 声明。D601/v03 当前可通过 `web-probe sentinel image|control-plane` 的独立 publish Job 实现构建、推送、GitOps 写回和 Argo 收敛;后续也可以切换到 Tekton Pipeline,但 builder 类型必须来自 YAML,不得依赖 operator 本地 dirty worktree。
哨兵自身 CI/CD 的 source identity 必须来自 k8s git-mirror source snapshot。`trigger-current` 应先通过 YAML 声明的 git-mirror cache/service 在 k8s 内解析并同步 selected commit,再把同一 source snapshot 注入 publish PipelineRun、GitOps 写回、status 和 closeout。operator host 上的 `git ls-remote``git fetch``git clone``git rev-parse HEAD` 和固定 host worktree 读写不得参与正式 source selection。
publish PipelineRun 的 source step 只能消费 source snapshot 的 commit-pinned mirror ref、stageRef、PVC 或 bundle artifact;不得把 mutable `refs/heads/<branch>` 作为本轮 source identity,也不得在 mirror 对象缺失时回退到 GitHub direct clone 或 host workspace repair。mirror 缺对象时必须结构化失败为 `source-snapshot-missing` 或等价故障码,并输出受控 source-mirror sync/ensure 命令。
哨兵 rollout 与 HWLAB runtime rollout 不是同一个滚动单元。哨兵 dashboard/API/服务代码变更应通过 Web 哨兵独立 control-plane 滚动;HWLAB runtime 发布流水只调用当前已部署哨兵的 `maintenance/start``maintenance/stop` 和 quick verify 作为恢复判定。哨兵 control-plane 的顶层状态只表达哨兵自身 source、镜像、GitOps、Argo、runtime、metrics 和 dashboard 是否发布成功;HWLAB quick verify 必须作为独立 `targetValidation` 状态、warning 和 report 证据输出。哨兵 validate、maintenance 和 quick verify 控制路径必须优先走 k3s 内部 Service DNS,不绕 `monitor.pikapython.com` 公网入口。
哨兵镜像构建应使用 YAML 声明的 tools image、base image、registry、egress proxy 和 env-reuse 配方。Node/Bun/Playwright/Chromium 依赖不得在 runtime Pod 中临时下载。Secret 与 env 复用只走 sourceRef/keyMapping;日志、status、dashboard 和 issue closeout 只输出 object/key/presence/fingerprint/digest。
@@ -525,6 +532,8 @@ HWLAB runtime 发布 Pipeline 应在 Argo sync 前调用当前哨兵 `maintenanc
`web-probe sentinel control-plane trigger-current --confirm --wait` 只等待 source mirror、publish、flush、publicExposure、Argo 和 runtime observed 收敛;CI/CD confirm-wait 超过 YAML `confirmWait.maxSeconds` 时必须输出 warning,并先优化等待阶段耗时,不得继续把长业务验证塞在部署同步路径里死等。`sentinel validate --quick-verify --confirm --wait` 和 maintenance stop quick verify 才执行 targetValidation 业务验证;业务 quick verify 的等待预算由 YAML `targetValidation.maxSeconds` 控制。计时超限本身只作为非阻塞告警;只有真正影响 Code Agent 多轮业务链路、submit/command 执行、trace/final 可见性或 session 连续性的失败才构成 targetValidation blocker。不得通过减少业务轮次、吞掉 submit 失败、fallback 到第二执行路径或读侧 repair 来消除红灯。
status/closeout 必须同时展示 selected source snapshot、latest mirror head、PipelineRun source snapshot、GitOps revision、runtime image digest 和 runtime observed 状态。若 `master` 在发布过程中继续前进,只能标记为 `latest-drift``superseded`;不得把 selected commit 已发布成功的结果覆盖成“追最新失败”。
### 6.6 OPS-SENTINEL-REQ-006 dsflash-go 十轮 canary
| 编号 | 短名 | 主责模块 | 关联模块 |
+242 -57
View File
@@ -7,6 +7,7 @@
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot.
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
import { createHash, randomUUID } from "node:crypto";
import { existsSync, readFileSync } from "node:fs";
@@ -61,6 +62,7 @@ export type WebProbeSentinelOptions =
readonly confirm: boolean;
readonly wait: boolean;
readonly timeoutSeconds: number;
readonly rerun: boolean;
}
| {
readonly kind: "publish";
@@ -72,6 +74,7 @@ export type WebProbeSentinelOptions =
readonly confirm: boolean;
readonly wait: boolean;
readonly timeoutSeconds: number;
readonly rerun: boolean;
}
| {
readonly kind: "maintenance";
@@ -156,7 +159,10 @@ interface SourceHead {
readonly repository: string;
readonly branch: string;
readonly commit: string | null;
readonly localHead: string | null;
readonly stageRef: string | null;
readonly mirrorCommit: string | null;
readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot";
readonly latestDrift: boolean;
readonly result: CompactCommandResult;
}
@@ -218,12 +224,14 @@ export interface ChildCliResult {
readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string };
}
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel";
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot";
type SourceResolveMode = "cached" | "sync";
export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId));
requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`);
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds);
const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options));
if (options.kind === "image") return runSentinelImage(state, options);
if (options.kind === "control-plane") return runSentinelControlPlane(state, options);
if (options.kind === "publish") return runSentinelPublishCurrent(state, options);
@@ -233,6 +241,13 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
return runSentinelReport(state, options);
}
function sentinelSourceResolveMode(options: WebProbeSentinelOptions): SourceResolveMode {
if (options.kind === "image" && options.action === "build" && options.confirm && options.wait) return "sync";
if (options.kind === "control-plane" && options.action === "trigger-current" && options.confirm && options.wait) return "sync";
if (options.kind === "publish" && options.confirm && options.wait) return "sync";
return "cached";
}
function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
const command = `web-probe sentinel image ${options.action}`;
if (options.action === "build" && options.confirm) {
@@ -280,7 +295,7 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
const observed = options.action === "status" ? collectSentinelObservedStatus(state, options.timeoutSeconds) : null;
const observedReady = options.action !== "status" || sentinelObservedReady(record(observed));
const observedWarnings = options.action === "status" ? sentinelObservedWarnings(record(observed)) : [];
const pipelineRun = sentinelPipelineRunName(state);
const pipelineRun = sentinelPipelineRunName(state, options.rerun);
const result = {
ok: state.configReady && state.sourceHead.ok && observedReady,
command,
@@ -354,7 +369,7 @@ function runSentinelPublishCurrent(state: SentinelCicdState, options: Extract<We
specRef: SPEC_REF,
source: state.sourceHead,
image: state.image,
pipelineRun: sentinelPipelineRunName(state),
pipelineRun: sentinelPipelineRunName(state, options.rerun),
gitops: {
path: stringAt(state.cicd, "gitopsPath"),
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
@@ -408,6 +423,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
confirm: true,
wait: true,
timeoutSeconds: Math.max(1, remainingBudgetSeconds()),
rerun: options.rerun,
});
let health: Record<string, unknown>;
let healthElapsedMs: number | null = null;
@@ -439,7 +455,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E
specRef: SPEC_REF,
source: state.sourceHead,
image: state.image,
pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state),
pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun),
controlPlane: controlResult,
health,
budget,
@@ -571,7 +587,7 @@ function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed:
};
}
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState {
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number, sourceResolveMode: SourceResolveMode): SentinelCicdState {
const sentinel = resolveWebProbeSentinel(spec, sentinelId);
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime);
@@ -585,7 +601,8 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef));
const nodeId = stringField(controlPlaneTarget, "node");
const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
const sourceHead = resolveSourceHead(cicd, timeoutSeconds);
validateSentinelSourceAuthority(cicd);
const sourceHead = resolveSourceHead(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode);
const image = sentinelImagePlan(spec, cicd, sourceHead);
const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image);
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
@@ -609,23 +626,145 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
};
}
function resolveSourceHead(cicd: Record<string, unknown>, timeoutSeconds: number): SourceHead {
function resolveSourceHead(
spec: HwlabRuntimeLaneSpec,
cicd: Record<string, unknown>,
controlPlaneTarget: Record<string, unknown>,
controlPlaneNode: Record<string, unknown>,
timeoutSeconds: number,
mode: SourceResolveMode,
): SourceHead {
const repository = stringAt(cicd, "source.repository");
const branch = stringAt(cicd, "source.branch");
const remote = runCommand(["git", "ls-remote", "origin", `refs/heads/${branch}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
const local = runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 10_000 });
const commit = /^[0-9a-f]{40}\b/iu.exec(remote.stdout.trim())?.[0].toLowerCase() ?? null;
const localHead = /^[0-9a-f]{40}$/iu.test(local.stdout.trim()) ? local.stdout.trim().toLowerCase() : null;
const resolved = mode === "sync"
? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds)
: probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null);
const probe = record(resolved.probe);
const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit);
const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit));
const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit);
return {
ok: remote.exitCode === 0 && commit !== null,
ok: resolved.ok === true && commit !== null,
repository,
branch,
commit,
localHead,
result: compactCommand(remote),
stageRef,
mirrorCommit,
sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache",
latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit,
result: compactCommand(resolved.result),
};
}
function resolveSourceHeadWithK8sSnapshot(
spec: HwlabRuntimeLaneSpec,
cicd: Record<string, unknown>,
controlPlaneTarget: Record<string, unknown>,
controlPlaneNode: Record<string, unknown>,
timeoutSeconds: number,
): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
const namespace = stringAt(cicd, "builder.namespace");
const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`;
const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName);
const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created };
const startedAt = Date.now();
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000));
let lastCapture = created;
while (Date.now() - startedAt < timeoutMs) {
const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
lastCapture = probeCapture;
const probe = parseJsonObject(probeCapture.stdout) ?? {};
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
if (probe.succeeded === true) return { ok: payload.ok === true, probe: payload, result: probeCapture };
if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, valuesRedacted: true } : payload, result: probeCapture };
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
}
return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture };
}
function sentinelSourceMirrorResolveJobManifest(
spec: HwlabRuntimeLaneSpec,
cicd: Record<string, unknown>,
controlPlaneTarget: Record<string, unknown>,
controlPlaneNode: Record<string, unknown>,
jobName: string,
): Record<string, unknown> {
const namespace = stringAt(cicd, "builder.namespace");
const labels = {
"app.kubernetes.io/name": "web-probe-sentinel-source-resolve",
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
"unidesk.ai/spec-ref": "PJ2026-01060508",
"unidesk.ai/node": spec.nodeId,
"unidesk.ai/lane": spec.lane,
};
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: { name: jobName, namespace, labels },
spec: {
backoffLimit: 0,
activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"),
ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"),
template: {
metadata: { labels },
spec: {
restartPolicy: "Never",
volumes: [
sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget),
{ name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
],
containers: [{
name: "resolve",
image: sentinelSourceResolverImage(spec, cicd),
imagePullPolicy: "IfNotPresent",
command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)],
volumeMounts: [
{ name: "cache", mountPath: "/cache" },
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
],
}],
},
},
},
};
}
function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>): string {
const baseImageRef = stringAt(cicd, "image.baseImageRef");
return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
}
function validateSentinelSourceAuthority(cicd: Record<string, unknown>): void {
const mode = stringAt(cicd, "sourceAuthority.mode");
const resolver = stringAt(cicd, "sourceAuthority.resolver");
const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit");
const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline");
const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy");
if (mode !== "gitMirrorSnapshot") throw new Error("sourceAuthority.mode must be gitMirrorSnapshot");
if (resolver !== "k8s-git-mirror") throw new Error("sourceAuthority.resolver must be k8s-git-mirror");
if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false");
if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false");
if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast");
sentinelSourceSnapshotStageRefPrefix(cicd);
}
function sentinelSourceSnapshotStageRefPrefix(cicd: Record<string, unknown>): string {
const branch = stringAt(cicd, "source.branch");
const repository = stringAt(cicd, "source.repository");
const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix")
.replaceAll("{branch}", branch)
.replaceAll("{repository}", repository)
.replace(/\/+$/u, "");
if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix");
return prefix;
}
function sentinelSourceSnapshotRef(cicd: Record<string, unknown>, commit: string): string {
return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`;
}
function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>, sourceHead: SourceHead): SentinelImagePlan {
const repository = stringAt(cicd, "image.repository");
const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12);
@@ -1409,7 +1548,7 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra
const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
const sourceMirrorReady = sourceMirrorSync.ok === true;
const publish = sourceMirrorReady
? runSentinelPublishJob(state, false, options.timeoutSeconds)
? runSentinelPublishJob(state, false, options.timeoutSeconds, false)
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
const registry = probeImageRegistry(state, options.timeoutSeconds);
const registryReady = record(registry.probe).present === true;
@@ -1474,7 +1613,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
const publish = applyOnly
? null
: sourceMirrorReady
? runSentinelPublishJob(state, true, remainingCommandSeconds())
? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun)
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8;
const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true
@@ -1523,6 +1662,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
? "one or more YAML-declared runtime Secrets were not synced from sourceRef"
: "one or more publicExposure, Argo or runtime observation checks did not pass",
};
const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun);
const result = {
ok,
command,
@@ -1533,7 +1673,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options:
specRef: SPEC_REF,
source: state.sourceHead,
image: state.image,
pipelineRun: sentinelPipelineRunName(state),
pipelineRun: publishPipelineRun,
gitops: {
path: stringAt(state.cicd, "gitopsPath"),
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
@@ -1732,19 +1872,26 @@ function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObser
}
function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
const namespace = stringAt(state.cicd, "builder.namespace");
const repository = stringAt(state.cicd, "source.repository");
const branch = stringAt(state.cicd, "source.branch");
const expectedCommit = state.sourceHead.commit;
const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit);
return { ...result, result: compactCommand(result.result) };
}
function probeSourceMirrorCache(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record<string, unknown>; result: CommandResult } {
const namespace = stringAt(cicd, "builder.namespace");
const repository = stringAt(cicd, "source.repository");
const branch = stringAt(cicd, "source.branch");
const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit);
const script = [
"set +e",
`repo_path=${shellQuote(`/cache/${repository}.git`)}`,
`branch=${shellQuote(branch)}`,
`expected=${shellQuote(expectedCommit ?? "")}`,
`stage_ref=${shellQuote(stageRef)}`,
"commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)",
"rc=$?",
"object_rc=1",
"expected_object_rc=1",
"stage_object_rc=1",
"contains_rc=1",
"if [ \"$rc\" -eq 0 ]; then",
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
@@ -1754,22 +1901,27 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1",
" expected_object_rc=$?",
"fi",
"if [ -n \"$stage_ref\" ]; then",
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1",
" stage_object_rc=$?",
"fi",
"if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then",
" kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1",
" contains_rc=$?",
"fi",
"node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$repo_path\" \"$branch\" <<'NODE'",
"const [rc, objectRc, expectedObjectRc, containsRc, commit, expected, repoPath, branch] = process.argv.slice(2);",
"node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'",
"const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);",
"const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');",
"const objectPresent = present && Number(objectRc) === 0;",
"const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;",
"const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;",
"const containsExpected = !expected || commit === expected || Number(containsRc) === 0;",
"const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';",
"console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror', present, objectPresent, expectedObjectPresent, containsExpected, relation, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
"console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
"NODE",
].join("\n");
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) };
const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout) ?? {}, result };
}
function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record<string, unknown> {
@@ -2130,18 +2282,22 @@ function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName:
}
function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit);
}
function sentinelSourceMirrorSyncShellFromConfig(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, jobName: string, selectedCommit: string | null): string {
return [
"set -eu",
`job_name=${shellQuote(jobName)}`,
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`,
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
`source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`,
`source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`,
`source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`,
`source_commit=${shellQuote(selectedCommit ?? "")}`,
`source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`,
"started_ms=$(node -e 'console.log(Date.now())')",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
"trap emit_failed EXIT",
"test -n \"$source_commit\"",
...sentinelSourceMirrorSshSetupShellLines(state),
...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode),
"repo=\"/cache/${source_repository}.git\"",
"mkdir -p \"$(dirname \"$repo\")\"",
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
@@ -2164,20 +2320,28 @@ function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string
"done",
"test \"$fetch_ok\" = 1",
"mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
"if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi",
"git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"",
"test \"$mirror_commit\" = \"$source_commit\"",
"stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"",
"git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
"git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"",
"git --git-dir=\"$repo\" update-server-info",
"finished_ms=$(node -e 'console.log(Date.now())')",
"node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
"const [jobName, repository, branch, sourceCommit, mirrorCommit, startedMs, finishedMs] = process.argv.slice(2);",
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
"node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
"const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);",
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
"NODE",
"trap - EXIT",
].join("\n");
}
function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
const proxy = record(valueAtPath(state.controlPlaneNode, "egressProxy"));
return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode);
}
function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record<string, unknown>): string[] {
const proxy = record(valueAtPath(controlPlaneNode, "egressProxy"));
const serviceName = nonEmptyString(proxy.serviceName);
const namespace = nonEmptyString(proxy.namespace);
const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
@@ -2299,8 +2463,8 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin
];
}
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult {
const pipelineRunName = sentinelPipelineRunName(state);
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult {
const pipelineRunName = sentinelPipelineRunName(state, rerun);
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops);
const namespace = stringAt(state.cicd, "builder.namespace");
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
@@ -2371,7 +2535,9 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu
namespace,
labels,
annotations: {
"unidesk.ai/source-commit": state.sourceHead.commit,
"unidesk.ai/source-commit": state.sourceHead.commit ?? "",
"unidesk.ai/source-authority": state.sourceHead.sourceAuthority,
"unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "",
"unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"),
"unidesk.ai/publish-gitops": publishGitops ? "true" : "false",
},
@@ -2459,9 +2625,13 @@ function tektonShellScript(body: string): string {
}
function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string, unknown> {
const hostPath = nonEmptyString(valueAtPath(state.controlPlaneTarget, "gitMirror.cacheHostPath"));
return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget);
}
function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record<string, unknown>): Record<string, unknown> {
const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath"));
if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } };
return { name: "cache", persistentVolumeClaim: { claimName: stringAt(state.controlPlaneTarget, "gitMirror.cachePvcName") } };
return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } };
}
function sentinelBuildkitStateVolume(state: SentinelCicdState): Record<string, unknown> {
@@ -2524,6 +2694,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
`source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`,
`checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
`dockerfile_b64=${shellQuote(dockerfileB64)}`,
`env_reuse_mode=${shellQuote(envReuseMode)}`,
@@ -2542,6 +2713,9 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
"started_ms=$(now_ms)",
"write_meta started_ms \"$started_ms\"",
"write_meta source_commit \"$source_commit\"",
"write_meta source_stage_ref \"$source_stage_ref\"",
"test -n \"$source_commit\"",
"test -n \"$source_stage_ref\"",
"mkdir -p /root/.ssh",
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
"chmod 0400 /root/.ssh/id_rsa",
@@ -2561,7 +2735,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
"source_fetch_started_ms=$(now_ms)",
"write_meta source_fetch_started_ms \"$source_fetch_started_ms\"",
"emit_stage source-fetch running \"$source_fetch_started_ms\"",
"git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"",
"git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
"git checkout --detach \"$source_commit\"",
"mirror_commit=$(git rev-parse HEAD)",
"test \"$mirror_commit\" = \"$source_commit\"",
@@ -2720,6 +2894,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
"started_ms=$(read_meta started_ms)",
"source_commit=$(read_meta source_commit)",
"source_stage_ref=$(read_meta source_stage_ref)",
"mirror_commit=$(read_meta mirror_commit)",
"source_fetch_started_ms=$(read_meta source_fetch_started_ms)",
"source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)",
@@ -2781,11 +2956,11 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
"fi",
"gitops_finished_ms=$(now_ms)",
"finished_ms=$(now_ms)",
"node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
"const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
"node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
"const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
"const elapsed = (start, finish) => Number(finish) - Number(start);",
"const cacheHits = Number(imageBuildCacheHits || 0);",
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
"NODE",
"trap - EXIT",
].join("\n");
@@ -2862,7 +3037,7 @@ function probeK8sJobScript(namespace: string, jobName: string): string {
"pod_phase=''",
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
"logs_tail=''",
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null || true; done; } | tail -c 16000 | base64 | tr -d '\\n'); fi",
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
"const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
@@ -2878,12 +3053,15 @@ function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string
"condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)",
"condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)",
"condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)",
"if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi",
"if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi",
"if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi",
"task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
"pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
"pod_phase=''",
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi",
"logs_tail=''",
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=180 2>/dev/null || true; } | tail -c 24000 | base64 | tr -d '\\n'); fi",
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
"node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
"const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);",
"const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');",
@@ -3082,7 +3260,9 @@ function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, prob
ok: true,
status: "already-present",
sourceCommit: state.sourceHead.commit,
mirrorCommit: state.sourceHead.commit,
mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit,
stageRef: state.sourceHead.stageRef,
sourceAuthority: state.sourceHead.sourceAuthority,
valuesRedacted: true,
},
polls: 0,
@@ -3653,9 +3833,12 @@ export function displayPath(pathValue: string): string {
return pathValue;
}
function sentinelPipelineRunName(state: SentinelCicdState): string {
function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string {
const commit = state.sourceHead.commit ?? "source";
return `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`;
const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`;
if (!rerun) return base;
const suffix = `-r${Date.now().toString(36)}`;
return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`;
}
export function sentinelCliSuffix(state: SentinelCicdState): string {
@@ -3795,9 +3978,11 @@ function renderPublishCurrentResult(result: Record<string, unknown>): string {
finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000),
]]),
"",
table(["SOURCE", "COMMIT", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[
`${source.repository ?? "-"}@${source.branch ?? "-"}`,
short(source.commit),
source.sourceAuthority ?? "-",
short(source.stageRef),
image.ref ?? "-",
short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest),
result.pipelineRun ?? publish.jobName ?? "-",
@@ -3902,7 +4087,7 @@ function renderImageResult(result: Record<string, unknown>): string {
"",
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]),
"",
table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]),
table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]),
"",
Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]),
"",
@@ -3912,7 +4097,7 @@ function renderImageResult(result: Record<string, unknown>): string {
"",
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
"",
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
"",
Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish),
"",
@@ -3958,7 +4143,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
"",
table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]),
"",
table(["SOURCE", "COMMIT", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), image.ref, short(gitops.manifestSha256)]]),
table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]),
"",
table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]),
"",
@@ -3968,7 +4153,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
"",
renderObservedStatus(observed),
"",
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
"",
Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[
targetValidation.ok,
@@ -138,6 +138,14 @@ const REQUIRED_TARGET_SHAPES: Record<HwlabRuntimeWebProbeSentinelConfigRefKey, R
"source.gitMirrorReadUrl",
"source.buildContext",
"source.entrypoint",
"sourceAuthority.mode",
"sourceAuthority.resolver",
"sourceAuthority.allowHostGit",
"sourceAuthority.allowGithubDirectInPipeline",
"sourceSnapshot.stageRefPrefix",
"sourceSnapshot.missingObjectPolicy",
"sourceSnapshot.refreshPolicy",
"sourceSnapshot.cacheRef",
"gitopsPath",
"argo.namespace",
"argo.projectName",
+65 -4
View File
@@ -103,10 +103,71 @@ export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
}
export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 });
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
const match = /[0-9a-f]{40}/iu.exec(statusText(result));
return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
const mirror = nodeRuntimeSourceMirrorTarget(spec);
const script = [
"set +e",
`namespace=${shellQuote(mirror.namespace)}`,
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
`repo_path=${shellQuote(`/cache/${mirror.sourceRepository}.git`)}`,
`source_branch=${shellQuote(mirror.sourceBranch)}`,
"read_ref() {",
" ref=\"$1\"",
" kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'repo_path=$1; ref=$2; git --git-dir=\"$repo_path\" rev-parse --verify \"$ref^{commit}\" 2>/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null",
"}",
"mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")",
"mirror_stage_rc=$?",
"local_head=$(read_ref \"refs/heads/$source_branch\")",
"local_head_rc=$?",
"source_commit=\"$mirror_stage\"",
"source_ref=\"refs/mirror-stage/heads/$source_branch\"",
"if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then",
" source_commit=\"$local_head\"",
" source_ref=\"refs/heads/$source_branch\"",
"fi",
"node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'",
"const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);",
"const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');",
"const ok = isSha(sourceCommit);",
"console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));",
"NODE",
].join("\n");
const result = runNodeK3sScript(spec, script, 45);
const payload = parseJsonObject(result.stdout);
const payloadCommit = typeof payload.sourceCommit === "string" && /^[0-9a-f]{40}$/iu.test(payload.sourceCommit) ? payload.sourceCommit.toLowerCase() : null;
const match = payloadCommit ?? /[0-9a-f]{40}/iu.exec(statusText(result))?.[0].toLowerCase() ?? null;
return { sourceCommit: result.exitCode === 0 ? match : null, result };
}
interface NodeRuntimeSourceMirrorTarget {
readonly namespace: string;
readonly serviceReadName: string;
readonly sourceRepository: string;
readonly sourceBranch: string;
}
function nodeRuntimeSourceMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeSourceMirrorTarget {
const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
const parsed = runtimeRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")));
const targets = Array.isArray(parsed.targets) ? parsed.targets.map((item) => runtimeRecord(item)) : [];
const target = targets.find((item) => item.node === spec.nodeId && item.lane === spec.lane);
if (target === undefined) throw new Error(`no control-plane target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
const gitMirror = runtimeRecord(target.gitMirror);
const source = runtimeRecord(target.source);
return {
namespace: runtimeString(gitMirror.namespace, "gitMirror.namespace"),
serviceReadName: runtimeString(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
sourceRepository: runtimeString(source.repository, "source.repository"),
sourceBranch: runtimeString(source.branch, "source.branch"),
};
}
function runtimeRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
function runtimeString(value: unknown, path: string): string {
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
return value;
}
export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
+1 -1
View File
@@ -52,7 +52,7 @@ export type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cle
export type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
export type NodeRuntimeRenderLocation = "node-host" | "local";
export type NodeRuntimeRenderLocation = "node-host";
export type WebProbeBrowserProxyMode = "auto" | "direct";
+29 -1
View File
@@ -34,7 +34,7 @@ import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WAR
import { parseNodeScopedDelegatedOptions } from "./plan";
import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render";
import { compactRuntimeCommand } from "./runtime-common";
import { compactNodeRuntimeGitMirrorObservation, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
import { compactNodeRuntimeGitMirrorObservation, compactNodeRuntimeGitMirrorRun, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
import { record } from "./utils";
import { webObserveTable } from "./web-observe-render";
import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe";
@@ -51,6 +51,28 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
const triggerStartedAt = Date.now();
const triggerElapsedMs = () => Date.now() - triggerStartedAt;
const sourceSnapshotSync = scoped.dryRun
? null
: nodeRuntimeGitMirrorRun({
...scoped,
domain: "git-mirror",
action: "sync",
confirm: true,
dryRun: false,
wait: true,
discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
});
if (sourceSnapshotSync !== null && sourceSnapshotSync.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "source-snapshot-sync",
degradedReason: "node-runtime-source-snapshot-sync-failed",
sourceSnapshotSync: nodeScopedFullOutput(scoped) ? sourceSnapshotSync : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
const head = resolveNodeRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
@@ -64,6 +86,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
phase: "source-head",
degradedReason: "node-runtime-source-head-unresolved",
headProbe: compactRuntimeCommand(head.result),
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
};
}
const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
@@ -86,6 +109,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
rerun: scoped.rerun,
before,
gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
@@ -122,6 +146,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
pipelineWait,
pipelineFailureSummary,
postFlush,
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
skipped: true,
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
skipPolicy: "source-commit-only",
@@ -149,6 +174,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
pipelineRun,
before,
gitMirror,
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
degradedReason: "node-runtime-git-mirror-pre-sync-failed",
};
}
@@ -175,6 +201,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
sourceCommit,
pipelineRun,
refresh,
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
next: {
@@ -220,6 +247,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
mutation: createOk,
sourceCommit,
pipelineRun,
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
rerunOf: scoped.rerun ? basePipelineRun : null,
rerun: scoped.rerun,
before,
+1 -92
View File
@@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -971,14 +971,9 @@ export function nodeRuntimeRenderToken(): string {
}
export function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
}
export function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
}
export function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
const retryCount = Math.max(0, Math.floor(retries));
@@ -1138,92 +1133,6 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec,
return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
}
export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
const token = nodeRuntimeRenderToken();
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
const script = [
"set -eu",
`source_url=${shellQuote(spec.gitUrl)}`,
`source_branch=${shellQuote(spec.sourceBranch)}`,
`source_commit=${shellQuote(sourceCommit)}`,
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
`overlay_b64=${shellQuote(overlay)}`,
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
"rm -rf \"$render_dir\" \"$worktree_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
"echo \"phase=local-git-clone-worktree\" >&2",
"run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
"cd \"$worktree_dir\"",
"echo \"phase=local-install-yaml\" >&2",
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
"node - \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
"const path = 'deploy/deploy.yaml';",
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const downloadStack = {",
" ...(lane.envRecipe?.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
"echo \"phase=local-gitops-render\" >&2",
[
"node scripts/run-bun.mjs \"$render_script\"",
`--lane ${shellQuote(spec.lane)}`,
`--node ${shellQuote(spec.nodeId)}`,
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
`--catalog-path ${shellQuote(spec.catalogPath)}`,
"--image-tag-mode full",
`--source-revision ${shellQuote(sourceCommit)}`,
`--source-repo ${shellQuote(spec.gitUrl)}`,
`--source-branch ${shellQuote(spec.sourceBranch)}`,
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
`--out ${shellQuote(renderDir)}`,
].join(" "),
...nodeRuntimePipelinePostprocessScript(),
].join("\n");
return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
}
export function nodeRuntimePipelinePostprocessScript(): string[] {
return [
"node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",
+3 -3
View File
@@ -72,7 +72,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
"--timeout-ms",
"--wait-timeout-ms",
"--command-timeout-seconds",
]), new Set(["--dry-run", "--confirm", "--wait", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"]));
]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"]));
const node = requiredOption(args, "--node");
assertNodeId(node);
const lane = requiredOption(args, "--lane");
@@ -96,9 +96,9 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
if (controlPlaneAction !== "plan" && controlPlaneAction !== "apply" && controlPlaneAction !== "status" && controlPlaneAction !== "trigger-current") {
throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]");
}
sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds };
sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") };
} else if (sentinelActionRaw === "publish-current") {
sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds };
sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") };
} else if (sentinelActionRaw === "maintenance") {
const maintenanceAction = args[1];
if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") {