From c90ad04bfffff91c6ef4d8f4e14a0a038da0cec0 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 1 Jul 2026 10:32:05 +0000 Subject: [PATCH] fix: use k8s git mirror source snapshots --- .../skills/unidesk-cicd/references/full.md | 4 +- AGENTS.md | 1 + .../cicd.auth-session-switch.d601-v03.yaml | 10 + .../cicd.d518-v03.yaml | 10 + .../cicd.d601-v03.yaml | 10 + .../cicd.fake-echo.d518-v03.yaml | 10 + .../cicd.mdtodo.d601-v03.yaml | 10 + config/hwlab-web-probe-sentinel/profiles.yaml | 14 + docs/reference/cli.md | 4 +- ...J2026-01060307-control-plane-modularity.md | 24 +- ...60308-cicd-yaml-first-target-governance.md | 36 ++- .../PJ2026-01060508-web-probe-sentinel.md | 9 + scripts/src/hwlab-node-web-sentinel-cicd.ts | 299 ++++++++++++++---- scripts/src/hwlab-node-web-sentinel-config.ts | 8 + scripts/src/hwlab-node/cleanup.ts | 69 +++- scripts/src/hwlab-node/entry.ts | 2 +- scripts/src/hwlab-node/git-mirror.ts | 30 +- scripts/src/hwlab-node/render.ts | 93 +----- scripts/src/hwlab-node/web-probe-observe.ts | 6 +- 19 files changed, 481 insertions(+), 168 deletions(-) diff --git a/.agents/skills/unidesk-cicd/references/full.md b/.agents/skills/unidesk-cicd/references/full.md index 693d5c28..b569a179 100644 --- a/.agents/skills/unidesk-cicd/references/full.md +++ b/.agents/skills/unidesk-cicd/references/full.md @@ -148,9 +148,9 @@ PipelineRun `gitops-promote` 如果报 git mirror 控制面漂移、refs 不一 node-scoped lane 可能在本次 PR 合并后又被后续 PR 推进。`control-plane status --pipeline-run ` 是定点观察某个 PipelineRun，但输出里的当前 `sourceHead` / `summary.sourceCommit` 可能已经是最新 branch tip，而不是该 PipelineRun 名称对应的 merge commit。closeout 必须同时记录 PR merge commit、PipelineRun 名称/状态、Argo sync revision、当前 branch tip，并用 `git merge-base --is-ancestor HEAD` 或等价证据说明最新 tip 包含本次 PR；不要只凭当前 source head 判断本次 rollout。 -`trigger-current --node D601 --lane v03 --confirm --wait` 可能先解析到一个 source commit，随后在 control-plane apply 的 `source-render` / `local-git-clone-worktree` 阶段因 `v0.3` 被并行 PR 推进而 clone 到更新 head，触发 `rev-parse HEAD == source_commit` 校验失败。这不是业务代码构建失败；先执行 `hwlab nodes git-mirror sync --node D601 --lane v03 --confirm --wait`，再快进固定 worktree，确认原 PR merge commit 是最新 `origin/v0.3` 的祖先，然后按最新 branch tip 重新运行 `trigger-current`。收口时同时记录原 PR merge commit、最新 source head 和 ancestor 证据。 +`trigger-current --node D601 --lane v03 --confirm --wait` 的 source selection 必须走 k8s git-mirror source snapshot：confirmed trigger 先执行受控 `git-mirror sync`，再从 mirror cache 读取 `refs/mirror-stage/heads/` 作为本轮 source commit。旧 `source-render` / `local-git-clone-worktree` 追 branch tip 的问题不得再用固定 worktree fetch/pull 修复；如果 mirror 缺对象或 GitHub SSH transient 耗尽，命令应以 `source-snapshot-missing` 或 git-mirror retry exhausted 类故障停止，并给出受控 sync/status 下一步。 -D601 `v0.3` 固定 worktree 的 fetch remote 是 node-local git mirror。GitHub PR 合并后，如果 `D601:/home/ubuntu/workspace/hwlab-v03` 中 `git fetch origin v0.3` 仍看不到最新 merge commit，先执行 `hwlab nodes git-mirror sync --node D601 --lane v03 --confirm --wait`，再在固定 worktree `git fetch origin v0.3 && git pull --ff-only origin v0.3`。D601/v03 `git-mirror` 的 GitHub upstream 标准传输固定为 YAML 声明的 SSH：`githubTransport.mode=ssh`，脚本通过 `GIT_SSH` wrapper 访问 `ssh://git@ssh.github.com:443/...`；node-global HTTP proxy 只作为 SSH CONNECT tunnel，不是 GitHub HTTPS auth/token transport。若 CLI 输出 `transport=https`、`GITHUB_TOKEN`、`git-mirror-github-token` 或 HTTPS token sourceRef，按 control-plane drift/配置回归处理：先修 `config/hwlab-node-control-plane.yaml` 并执行 `hwlab nodes control-plane apply --node D601 --lane v03 --confirm`，不要改走 HTTPS、不要增加 fallback、不要用 host workspace repair。`sync/flush` 的 retry 只消费 SSH upstream transient，并在耗尽后输出 stopped/exhausted；`trigger-current --lane v03` 会为 PipelineRun 做 mirror pre-sync，但不替代固定 worktree 的 fetch hygiene。promotion 后若 node-local `git-mirror status` 显示 `pendingFlush=true`，执行 node-local flush 并等到 `pendingFlush=false`、`githubInSync=true`。 +D601/v03 `git-mirror` 的 GitHub upstream 标准传输固定为 YAML 声明的 SSH：`githubTransport.mode=ssh`，脚本通过 `GIT_SSH` wrapper 访问 `ssh://git@ssh.github.com:443/...`；node-global HTTP proxy 只作为 SSH CONNECT tunnel，不是 GitHub HTTPS auth/token transport。若 CLI 输出 `transport=https`、`GITHUB_TOKEN`、`git-mirror-github-token` 或 HTTPS token sourceRef，按 control-plane drift/配置回归处理：先修 `config/hwlab-node-control-plane.yaml` 并执行 `hwlab nodes control-plane apply --node D601 --lane v03 --confirm`，不要改走 HTTPS、不要增加 fallback、不要用 host workspace repair。`sync/flush` 的 retry 只消费 SSH upstream transient，并在耗尽后输出 stopped/exhausted；promotion 后若 node-local `git-mirror status` 显示 `pendingFlush=true`，执行 node-local flush 并等到 `pendingFlush=false`、`githubInSync=true`。 D601/node-scoped mirror status 的 `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...`；`status` 输出应通过 `refSources.githubFieldsAreMirrorStageCache=true` 显示这一点。`hwlab nodes git-mirror flush --node D601 --lane v03 --confirm --wait` 如果已经显示 `v0.3-gitops -> v0.3-gitops` 推送成功，但随后因 GitHub SSH `kex_exchange_identification` 或 fetch 确认失败导致命令非零退出，会标记 `partialSuccess=push-succeeded-fetch-failed`。当前 CLI 会自动做一次受控 sync/recheck；恢复后输出 `partialSuccessRecovered=true`、`postPushRecovery` 且整体 `ok=true`，未恢复时才把下一步指向 `hwlab nodes git-mirror sync --node D601 --lane v03 --confirm --wait`。不要连续盲目 flush；先刷新 mirror-stage，再用 status 确认 `localGitops=githubGitops`、`pendingFlush=false`、`githubInSync=true`。 diff --git a/AGENTS.md b/AGENTS.md index 993d7d35..9d67fe8e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台。本文 - P0: 用户最新明确要求优先于旧测试、旧门禁、旧预检、旧断言和旧 guard；阻碍最新目标的旧入口应删除，不做兼容保留。 - P0: 可见性问题优先修复；状态、耗时、失败原因、trace、命令结果或关键证据不可见时，先补 CLI/日志/状态输出再继续。细则见 `docs/reference/observability.md`。 - P0: CLI 改动默认不做单元测试、合同测试或新增测试脚本；除非用户明确要求，最多做语法检查和必要命令形态确认。 +- P0: 不得用裸 `tsc`/`bun --bun tsc` 当语法验证；它会长时间卡住。CLI/TS 轻量验证用 `bun --check <具体文件>`、`bun scripts/cli.ts check` 默认 syntax transpile 和必要命令形态确认；细则见 `docs/reference/cli.md`。 - P0: CLI 默认输出应是 Kubernetes 风格的简洁表格、短摘要和 drill-down 命令；JSON 只用于 `--json`、`--raw`、`--full`、`-o json` 或机器消费。超长输出 dump 是兜底，不是长期交互入口。细则见 `docs/reference/cli.md`。 ## P0: 配置、Secret 与 YAML-first diff --git a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml index 4ea50718..60c8cfde 100644 --- a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml @@ -22,6 +22,16 @@ sentinel: - package.json - bun.lock - bun.lockb + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + cacheRef: source.gitMirrorReadUrl builder: namespace: devops-infra sourceMode: sparse-git-checkout diff --git a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml index 8e242a9e..121cb967 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml @@ -22,6 +22,16 @@ sentinel: - package.json - bun.lock - bun.lockb + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + cacheRef: source.gitMirrorReadUrl builder: namespace: devops-infra sourceMode: sparse-git-checkout diff --git a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml index 7c140648..70c49746 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml @@ -22,6 +22,16 @@ sentinel: - package.json - bun.lock - bun.lockb + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + cacheRef: source.gitMirrorReadUrl builder: namespace: devops-infra sourceMode: sparse-git-checkout diff --git a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml index 6fad8450..4a26d686 100644 --- a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml @@ -23,6 +23,16 @@ sentinel: - package.json - bun.lock - bun.lockb + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + cacheRef: source.gitMirrorReadUrl builder: namespace: devops-infra sourceMode: sparse-git-checkout diff --git a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml index 7cedfe22..e9291ce9 100644 --- a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml @@ -22,6 +22,16 @@ sentinel: - package.json - bun.lock - bun.lockb + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + cacheRef: source.gitMirrorReadUrl builder: namespace: devops-infra sourceMode: sparse-git-checkout diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml index ca72b576..84250e26 100644 --- a/config/hwlab-web-probe-sentinel/profiles.yaml +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -57,6 +57,16 @@ baselines: - package.json - bun.lock - bun.lockb + sourceAuthority: &cicd-source-authority + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowGithubDirectInPipeline: false + sourceSnapshot: &cicd-source-snapshot + stageRefPrefix: refs/unidesk/snapshots/web-probe-sentinel/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot + cacheRef: source.gitMirrorReadUrl builder: &cicd-builder namespace: devops-infra sourceMode: sparse-git-checkout @@ -159,6 +169,10 @@ nodes: controlPlaneConfigRef: config/hwlab-node-control-plane.yaml#targets[1] source: <<: *cicd-source + sourceAuthority: + <<: *cicd-source-authority + sourceSnapshot: + <<: *cicd-source-snapshot argo: &jd01-argo namespace: argocd projectName: hwlab-jd01 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index aba7650a..92d13da9 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -34,7 +34,7 @@ G14/D601 v03 的 bootstrap admin password 是 HWLAB runtime Secret 生命周期 `hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile --confirm` 是 HWLAB v0.3 k3s CI/CD 全量无缓存构建出网测速入口，profile、cache policy、独立 catalog path 模板、PipelineRun prefix、必须输出的 timing 阶段和失败族都来自 `config/hwlab-node-control-plane.yaml`。confirmed benchmark 只创建一次唯一 PipelineRun，使用 node-lane YAML 中的实际 HWLAB v0.3 service set、git mirror read/write URL、registry prefix、base image 和 Tekton pipeline；`forbidBuildkitCache=true` 时会向 PipelineRun 传 `build-cache-mode=disabled`。status/logs 通过短连接轮询 PipelineRun/TaskRun 摘要和有界日志。成功的 benchmark 必须出现每个 `build-` TaskRun；如果 PipelineRun 成功但缺少任一 service build task，CLI 必须把该 service 报为 `cache-hit-forbidden`，不能把 catalog/env reuse 当作 #1010 这类性能验收的通过证据。 -`hwlab nodes git-mirror status|sync|flush --node --lane ` 是 node-scoped runtime lane 的 Git mirror 维护入口。`status` 的 `githubSource` / `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...`，不是实时 GitHub API；输出中的 `refSources.githubFieldsAreMirrorStageCache=true` 和 `refSources.cacheRefresh` 给出这一来源和刷新命令。`sync --confirm --wait` 的 k3s Job 遇到 GitHub SSH transient 时，应通过目标 workspace fallback 拉取 GitHub source/gitops 并写回 node-local mirror，输出只披露 commit、mirror write URL 和 fallback 状态。`flush --confirm --wait` 如果已经把 GitOps ref push 到 GitHub，但 post-push fetch/recheck 因 transient SSH 失败而无法刷新 mirror-stage，会标记 `partialSuccess=push-succeeded-fetch-failed`；CLI 应自动执行一次受控 sync 刷新 mirror-stage，若恢复后 `pendingFlush=false` 且 `githubInSync=true`，结果应为 `ok=true` 并输出 `partialSuccessRecovered` / `postPushRecovery`，否则才保留 `degradedReason=node-runtime-git-mirror-flush-post-push-fetch-failed` 和下一步 `sync --confirm --wait`。不要把这种 partial success 解读为需要连续盲目 flush。`hwlab nodes control-plane trigger-current --node --lane --confirm --wait` 会在 source sync 后自动执行必要的 pre-flush，在 PipelineRun terminal 后自动执行必要的 post-flush；progress 事件必须显式输出 `git-mirror-pre-flush` / `git-mirror-post-flush` 的 executed/skipped、jobName、local/github source、local/github GitOps、`pendingFlush` 和 `githubInSync`，且已恢复的 partial success 不能让顶层 trigger-current false-fail。`control-plane status` 仍是只读入口，只暴露 compact `gitMirror` 摘要和下一步 flush 命令，不隐式执行写操作。 +`hwlab nodes git-mirror status|sync|flush --node --lane ` 是 node-scoped runtime lane 的 Git mirror 维护入口。`status` 的 `githubSource` / `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...`，不是实时 GitHub API；输出中的 `refSources.githubFieldsAreMirrorStageCache=true` 和 `refSources.cacheRefresh` 给出这一来源和刷新命令。`sync --confirm --wait` 的 k8s Job 是 upstream GitHub fetch 的唯一正式入口；遇到 GitHub SSH transient 时按受控 retry/backoff 停止并输出下一步，不回退到 operator host git、目标 host fixed workspace 或第二套 source resolver。`flush --confirm --wait` 如果已经把 GitOps ref push 到 GitHub，但 post-push fetch/recheck 因 transient SSH 失败而无法刷新 mirror-stage，会标记 `partialSuccess=push-succeeded-fetch-failed`；CLI 应自动执行一次受控 sync 刷新 mirror-stage，若恢复后 `pendingFlush=false` 且 `githubInSync=true`，结果应为 `ok=true` 并输出 `partialSuccessRecovered` / `postPushRecovery`，否则才保留 `degradedReason=node-runtime-git-mirror-flush-post-push-fetch-failed` 和下一步 `sync --confirm --wait`。不要把这种 partial success 解读为需要连续盲目 flush。`hwlab nodes control-plane trigger-current --node --lane --confirm --wait` 会先执行 k8s git-mirror source snapshot sync，再从 mirror cache 选择 source commit，随后自动执行必要的 pre-flush，并在 PipelineRun terminal 后自动执行必要的 post-flush；progress 事件必须显式输出 `git-mirror-pre-flush` / `git-mirror-post-flush` 的 executed/skipped、jobName、local/github source、local/github GitOps、`pendingFlush` 和 `githubInSync`，且已恢复的 partial success 不能让顶层 trigger-current false-fail。`control-plane status` 仍是只读入口，只读 k8s mirror cache 并暴露 compact `gitMirror` 摘要和下一步 flush 命令，不隐式执行写操作。 PR 合并后触发 node-scoped runtime lane 时，`control-plane status --pipeline-run ` 是某次 PipelineRun 的定点观察入口，但同一输出中的 `sourceHead` / `summary.sourceCommit` 仍可能反映当前分支最新 head；如果触发后又有后续 PR 合并，当前 head 可能已经不是该 PipelineRun 名称中的短 SHA。closeout 证据必须同时写明：PR merge commit、定点 PipelineRun 名称和状态、最终 runtime/GitOps revision、当前 branch tip，以及当前 branch tip 是否包含本次 PR merge commit。不要只凭 `summary.sourceCommit` 反推某个旧 PipelineRun 的源码身份。 @@ -54,7 +54,7 @@ PipelineRun 失败或长时间未完成时，先按定点 `control-plane status - 每个 CLI 命名空间必须支持 `help`、`--help` 或 `-h` 并返回 JSON，不得为了打印帮助而访问 runtime 服务、拉起交互会话或执行长时任务。 - `--main-server-ip ` 默认通过公网 frontend 登录态调用主 server 的同源 API 代理，不要求计算节点持有主 server SSH key；显式提供 `--main-server-key` 或 `--main-server-transport ssh` 时才使用旧 SSH 传输。远程 frontend 传输下的 `ssh ...` 必须复用同一套结构化 route parser，支持 `D601`、`G14`、host workspace、`D601:win`、`D601:win/c/test`、`D601:k3s` 和 `D601:k3s::` 这类定位路径；它不向调用容器下发 provider token，也不要求调用容器能解析 backend-core 内网 DNS。 - `config show` 读取并校验根目录 `config.json`，不从环境变量、默认值或隐藏文件静默补配置。 -- `check` 默认只执行轻量配置校验、Bun 版本检查和 Bun Transpiler 语法解析（覆盖 CLI 入口、主要 `scripts/` 模块和核心组件入口，不做类型推导）。除非用户明确要求，CLI 改动不运行单元测试、合同测试或新增测试脚本；默认最多做语法检查和必要的帮助/命令形态人工确认。关键文件存在性、`scripts/` TypeScript 类型检查、`src/components/` TypeScript 类型检查、Docker Compose config、日志轮转策略扫描和 D601 recovery guardrails 默认不启用，分别通过 `--files`、`--scripts-typecheck`、`--components`、`--compose`、`--logs`、`--recovery-guardrails` 开启，或用 `--full` 一次性开启。`--scripts-typecheck` 只跑 scripts TypeScript 类型检查，不触发测试脚本或 GitHub issue/PR live API check。长命令项必须在 stderr 输出 `unidesk.check.progress` JSON lines，stdout 保持最终 JSON 结果，避免 post-task 或人工运行时长时间无可见进度。`typescript:scripts` 固定通过 `bun --bun tsc -p scripts/tsconfig.json --noEmit --pretty false` 执行，默认 `--scripts-typecheck-timeout-ms 120000`，可按目标运行面显式调小或调大但 CLI 会封顶；`--check-heartbeat-ms` 控制运行中心跳间隔，默认 `15000`。所有命令项的最终 item detail 必须包含 `durationMs`、`timeoutMs`、`heartbeatMs`、`exitCode`、`signal`、`timedOut`、stdout/stderr byte count、truncation flag 和有界 tail；超时必须返回 `timedOut=true`，不得只留下被外层命令杀死的空输出。不要把 `bun --check scripts/cli.ts` 当作低噪声 CLI 自检入口；它可能执行根 CLI help 并触发长 help dump。CLI 入口级自检使用 `bun scripts/cli.ts check`，单文件语法确认只针对具体模块文件运行。`check recovery-guardrails` 是同一诊断的低噪声直接入口，报告 malformed `/proc/mounts`、kubelet validation risk、stale CRI sandbox count、Code Queue worktree/symlink、Code Queue/MDTODO hostPath 和 `ContainerCreating` 分类；它不得重启 k3s、删除 CRI sandbox、修改 hostPath、deploy/rollout 或 prune/reset。`--rust` 只允许在 D601 CI/dev execution 中配合 `UNIDESK_D601_RUST_CHECK=1` 使用，长期规则见 `docs/reference/dev-environment.md` 和 `docs/reference/devops-hygiene.md`。 +- `check` 默认只执行轻量配置校验、Bun 版本检查和 Bun Transpiler 语法解析（覆盖 CLI 入口、主要 `scripts/` 模块和核心组件入口，不做类型推导）。除非用户明确要求，CLI 改动不运行单元测试、合同测试或新增测试脚本；默认最多做语法检查和必要的帮助/命令形态人工确认。关键文件存在性、`scripts/` TypeScript 类型检查、`src/components/` TypeScript 类型检查、Docker Compose config、日志轮转策略扫描和 D601 recovery guardrails 默认不启用，分别通过 `--files`、`--scripts-typecheck`、`--components`、`--compose`、`--logs`、`--recovery-guardrails` 开启，或用 `--full` 一次性开启。不得把裸 `tsc` / `bun --bun tsc` 或 `check --scripts-typecheck` 当作语法验证；它是重型类型检查，可能长时间卡住或超时。单文件语法确认使用 `bun --check <具体文件>`，入口级轻量自检使用 `bun scripts/cli.ts check` 默认 syntax transpile。`--scripts-typecheck` 只跑 scripts TypeScript 类型检查，不触发测试脚本或 GitHub issue/PR live API check。长命令项必须在 stderr 输出 `unidesk.check.progress` JSON lines，stdout 保持最终 JSON 结果，避免 post-task 或人工运行时长时间无可见进度。`typescript:scripts` 固定通过 `bun --bun tsc -p scripts/tsconfig.json --noEmit --pretty false` 执行，默认 `--scripts-typecheck-timeout-ms 120000`，可按目标运行面显式调小或调大但 CLI 会封顶；`--check-heartbeat-ms` 控制运行中心跳间隔，默认 `15000`。所有命令项的最终 item detail 必须包含 `durationMs`、`timeoutMs`、`heartbeatMs`、`exitCode`、`signal`、`timedOut`、stdout/stderr byte count、truncation flag 和有界 tail；超时必须返回 `timedOut=true`，不得只留下被外层命令杀死的空输出。不要把 `bun --check scripts/cli.ts` 当作低噪声 CLI 自检入口；它可能执行根 CLI help 并触发长 help dump。CLI 入口级自检使用 `bun scripts/cli.ts check`，单文件语法确认只针对具体模块文件运行。`check recovery-guardrails` 是同一诊断的低噪声直接入口，报告 malformed `/proc/mounts`、kubelet validation risk、stale CRI sandbox count、Code Queue worktree/symlink、Code Queue/MDTODO hostPath 和 `ContainerCreating` 分类；它不得重启 k3s、删除 CRI sandbox、修改 hostPath、deploy/rollout 或 prune/reset。`--rust` 只允许在 D601 CI/dev execution 中配合 `UNIDESK_D601_RUST_CHECK=1` 使用，长期规则见 `docs/reference/dev-environment.md` 和 `docs/reference/devops-hygiene.md`。 - `server start` 创建异步 job，在后台执行 Docker 构建和启动；命令默认只返回低噪声 async job 摘要、stdout/stderr 路径和 `job status` 后续命令，完整 JSON 只能通过 `--full`/`--raw` 显式展开。 - `server stop` 创建异步 job，在后台停止固定 Compose project 中的全部 UniDesk 服务；默认输出同样是 async job 摘要。 - `server status` 查询公开端口、受限宿主端口、内部端口、主机 swap 摘要、Compose 容器、core/frontend/dev-frontend/provider/database 健康检查和访问 URL；D601 Code Queue 使用的 PostgreSQL/OA Event Flow host mapping 必须出现在受限宿主端口而不是无条件公开入口中。低内存主 server 上 `swap.warning` 非空时，先执行 `server swap status` 或 `server swap ensure`。 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060307-control-plane-modularity.md b/project-management/PJ2026-01/specs/PJ2026-01060307-control-plane-modularity.md index fa288e91..d7969a17 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060307-control-plane-modularity.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060307-control-plane-modularity.md @@ -22,7 +22,7 @@ | 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) | | 上级规格 | [PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md) | | 关联规格 | [PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md)、[PJ2026-010602 源码同步](PJ2026-010602-source-sync.md)、[PJ2026-010604 公开入口](PJ2026-010604-public-entry.md)、[PJ2026-010605 运维监控](PJ2026-010605-observability-monitoring.md) | -| 实现引用版本 | draft-2026-06-25-p0 | +| 实现引用版本 | draft-2026-06-25-p0; draft-2026-07-01-cicd-source-snapshot | | 规格治理索引 | [规格治理](spec-governance.md) | 本文采用 ISO/IEC/IEEE 29148 需求规格模板的项目裁剪版：正文只保留 CI/CD、YAML-first 和平台运维 CLI 控制面源码拆分的稳定目标、边界、架构、阶段和验收口径。 @@ -31,7 +31,7 @@ ### 2.1 目的 -控制面模块化负责把 UniDesk CI/CD、YAML-first 和平台运维 CLI 中的超大 TypeScript 入口拆成可维护的领域模块，使命令路由、配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、Tekton/Argo、status summary 和 bounded output 各有明确归属。 +控制面模块化负责把 UniDesk CI/CD、YAML-first 和平台运维 CLI 中的超大 TypeScript 入口拆成可维护的领域模块，使命令路由、配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、source snapshot resolver、Tekton/Argo、status summary 和 bounded output 各有明确归属。 本规格的直接治理对象是超过 3000 行的控制面入口文件。第一阶段先做机械拆分和兼容入口保留，第二阶段再收紧 import/export、模块边界和共享 helper；不得在规格未落地前直接进行行为重写。 @@ -39,7 +39,7 @@ - `scripts/src/hwlab-g14.ts`、`scripts/src/hwlab-node-impl.ts`、`scripts/src/platform-infra-sub2api-codex.ts`、`scripts/src/agentrun.ts`、`scripts/src/platform-infra.ts`、`scripts/src/deploy.ts`、`scripts/src/artifact-registry.ts`、`scripts/src/ci.ts` 和 `scripts/src/platform-infra-observability.ts` 的职责拆分。 - 领域子目录的建立、兼容 re-export 入口、机械搬迁追溯标注和 public API 保持。 -- 配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、Tekton/Argo、status summary、bounded output 和低层 helper 的模块边界。 +- 配置解析、manifest 渲染、远端脚本、Secret/public exposure、git-mirror、source snapshot resolver、Tekton/Argo、status summary、bounded output 和低层 helper 的模块边界。 - 与 YAML-first source of truth、发布流水、源码同步、公开入口和运维监控规格的一致性。 - `bun scripts/cli.ts --help`、plan/status/validate/dry-run 等原入口 smoke 验证。 @@ -67,7 +67,7 @@ | 边界项 | 内容 | | --- | --- | | 外部使用者 | 平台维护者、发布操作人员、自动化任务和后续代码维护者。 | -| 外部输入 | CLI 命令、YAML 配置、node/lane/target 参数、Secret sourceRef、Git commit 和运行面状态查询请求。 | +| 外部输入 | CLI 命令、YAML 配置、node/lane/target 参数、Secret sourceRef、source snapshot、Git commit 和运行面状态查询请求。 | | 受控资源 | `scripts/src` 控制面源码、领域子目录、共享 helper、兼容入口、CLI 输出摘要和原入口命令。 | | 外部输出 | 原 CLI 命令结果、help、plan/status/validate 摘要、dry-run 计划和 redacted 错误。 | | 用户接口 | 既有 `bun scripts/cli.ts hwlab g14 ...`、`hwlab nodes ...`、`agentrun ...`、`platform-infra ...`、`ci ...`、`deploy ...` 和 `artifact-registry ...`。 | @@ -86,10 +86,12 @@ flowchart LR DomainIndex --> Actions[plan apply status actions] Actions --> Render[manifest and summary renderers] Actions --> Remote[remote script builders and capture] + Actions --> Source[git-mirror source snapshot resolver] Actions --> Secret[Secret/public exposure/git-mirror helpers] Config --> Ops[common ops primitives] Render --> Ops Remote --> Ops + Source --> Ops Secret --> Ops Ops --> Runtime[target runtime or dry-run evidence] ``` @@ -163,7 +165,7 @@ CI/CD、YAML-first 和平台运维控制面文件超过 3000 行时，必须进 | --- | --- | --- | --- | | OPS-MOD-REQ-004 | 模块边界 | PJ2026-01060307 控制面模块化 | [公开入口](PJ2026-010604-public-entry.md)、[运维监控](PJ2026-010605-observability-monitoring.md) | -第二阶段应把 import/export 和依赖方向收敛到稳定边界：options/config 只解析输入，manifest/render 只生成输出，remote/script builder 只封装远端执行文本，Secret/public exposure/git-mirror/Tekton/Argo/status summary 分别归属领域模块或公共 helper。 +第二阶段应把 import/export 和依赖方向收敛到稳定边界：options/config 只解析输入，manifest/render 只生成输出，remote/script builder 只封装远端执行文本，source snapshot resolver、Secret/public exposure/git-mirror/Tekton/Argo/status summary 分别归属领域模块或公共 helper。 若多个领域重复实现同一机制，应优先抽到公共 helper；若差异是业务领域真实差异，应保留在领域 adapter 中并在 closeout 说明为 `keep-domain-special`。 @@ -187,6 +189,18 @@ Secret 相关模块只能输出对象名、key 名、sourceRef、presence、fing 回滚策略是通过 Git revert 恢复模块边界，不通过复制旧文件、手工 patch 运行面或保留并行 legacy 入口实现长期双路径。若模块化后发现行为回归，应先修复 import/export 或职责边界，再决定是否回退。 +### 6.7 OPS-MOD-REQ-007 source plane 单一路径 + +| 编号 | 短名 | 主责模块 | 关联模块 | +| --- | --- | --- | --- | +| OPS-MOD-REQ-007 | source plane | PJ2026-01060307 控制面模块化 | [YAML目标治理](PJ2026-01060308-cicd-yaml-first-target-governance.md)、[源码同步](PJ2026-010602-source-sync.md) | + +控制面模块必须把 source selection、mirror sync、snapshot creation、PipelineRun source acquisition 和 GitOps publish 明确拆分并串成单一路径。正式入口只能先通过 YAML 解析目标，再通过 k8s git-mirror resolver 得到 source snapshot，随后把同一 snapshot 注入 PipelineRun、status 和 closeout。 + +不得在同一正式流程中并存 operator host `git ls-remote`、固定 host worktree clone/fetch、node-host shared worktree、PipelineRun mutable branch fetch 和 GitHub direct fetch 等多个 source resolver。历史 local render 或 host workspace 逻辑若暂时保留，必须标注为显式 legacy/debug adapter，不得由 `trigger-current`、Web 哨兵 publish 或 runtime rollout 默认调用。 + +PipelineRun/status 模块必须把 selected snapshot 与 latest branch/mirror head 分开展示。source snapshot 缺对象、mirror 追新、GitOps flush pending、Argo progressing 和 runtime readiness 是不同阶段，不得用一个 `sourceHead` 字段覆盖全部判断。 + ## 7. 过程控制 本规格的执行 issue 为 [#903](https://github.com/pikasTech/unidesk/issues/903)。源码文件头部应标注 `SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0`，并说明文件是薄入口、领域模块还是公共 helper。纯配置、锁文件、生成产物和无法承载注释头的二进制文件可例外。 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060308-cicd-yaml-first-target-governance.md b/project-management/PJ2026-01/specs/PJ2026-01060308-cicd-yaml-first-target-governance.md index 2a0741a6..527f12bd 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060308-cicd-yaml-first-target-governance.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060308-cicd-yaml-first-target-governance.md @@ -22,7 +22,7 @@ | 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) | | 上级规格 | [PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md) | | 关联规格 | [PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md)、[PJ2026-010602 源码同步](PJ2026-010602-source-sync.md)、[PJ2026-01060307 控制面模块化](PJ2026-01060307-control-plane-modularity.md)、[PJ2026-010604 公开入口](PJ2026-010604-public-entry.md)、[PJ2026-010605 运维监控](PJ2026-010605-observability-monitoring.md) | -| 实现引用版本 | draft-2026-06-25-cicd-yaml-targets | +| 实现引用版本 | draft-2026-06-25-cicd-yaml-targets; draft-2026-07-01-cicd-source-snapshot | | 规格治理索引 | [规格治理](spec-governance.md) | 本文采用 ISO/IEC/IEEE 29148 需求规格模板的项目裁剪版：正文只保留 CI/CD、YAML-first 和平台运维控制面中 target、lane、Secret、public exposure、PipelineRun/Argo/git-mirror 与公共 ops primitive 的稳定治理口径。 @@ -58,6 +58,8 @@ YAML目标治理负责把 CI、deploy、artifact-registry、HWLAB node/lane、Ag | lane | YAML 中描述一条 runtime 或 control-plane 运行线的对象，通常包含 node、source truth、runtime namespace、gitops、CI、Secret 和 publicExposure。 | | configRef | `path/to/file.yaml#object.path` 形式的跨 YAML 引用，解析器只校验引用存在性、类型和摘要，不把合并结果写成第二真相。 | | sourceRef | Secret 来源声明。输出只能显示来源标识、key 名、presence、fingerprint、字节数和 `valuesPrinted=false`。 | +| source snapshot | CI/CD 触发时由 k8s 托管 git-mirror 解析出的不可变源码快照，至少包含 repository、branch、selectedCommit、stageRef、mirrorCommit、objectPresent、resolver job 和 YAML 来源。 | +| host workspace | operator host 或目标节点上的固定源码 checkout。正式 CI/CD 只能把它作为开发便利或显式 legacy/debug adapter，不得作为发布 source truth。 | | hidden default | 代码在未显式参数或未读取 YAML 时自动选择固定目标、namespace、route、registry、lane 或工作目录。 | | legacy adapter | 为历史入口保留的隔离兼容路径。它可以拒绝 mutation 或提示替代命令，但不得污染主 runtime lane 解析。 | | ops primitive | 跨领域复用的配置引用、目标解析、Secret redaction、K8s apply/status、public exposure 和 bounded output helper。 | @@ -161,6 +163,26 @@ sequenceDiagram 状态输出必须区分 PR merge commit、current source head、PipelineRun、Argo revision、git-mirror pendingFlush 和 runtime readiness；不得只用一个当前 head 推断所有阶段完成。 +### 5.6 k8s source snapshot 链路 + +```mermaid +sequenceDiagram + participant CLI + participant Target as YAML target + participant Resolver as k8s git-mirror resolver + participant Mirror as git-mirror cache + participant Tekton + CLI->>Target: resolve node/lane/sourceAuthority + CLI->>Resolver: ensure or read source snapshot + Resolver->>Mirror: sync/read selected commit and create stageRef + Mirror-->>Resolver: selectedCommit + stageRef + objectPresent + Resolver-->>CLI: sourceSnapshot summary + CLI->>Tekton: create PipelineRun with sourceSnapshot + Tekton->>Mirror: fetch/checkout immutable stageRef or snapshot artifact +``` + +正式 CI/CD 的源码身份以 source snapshot 为准。`trigger-current`、image publish、GitOps render 和 closeout 不得再从 operator host 的 dirty/stale worktree、当前 branch tip 或 GitHub direct fetch 推导本轮 source identity。upstream fetch 只允许发生在受控 k8s git-mirror resolver/sync 组件内；PipelineRun 只消费 node-local mirror 的不可变 stageRef、PVC 或 bundle artifact。 + ## 6. 原子需求 ### 6.1 OPS-TARGET-REQ-001 YAML target 真相 @@ -217,6 +239,18 @@ D601 maintenance deploy、G14 DEV/PROD retirement、v02-only runtime migration 验收至少覆盖 `ci plan/status --target `、`artifact-registry plan/status --target `、`deploy apply --dry-run` 目标来源说明、`hwlab nodes control-plane status --node --lane ` 的 YAML-declared lane、`platform-infra observability plan/status --full` 的 configRef 解析链和 `secrets status --config ... --full` 的脱敏输出。 +### 6.7 OPS-TARGET-REQ-007 source snapshot 与 host worktree 禁用 + +| 编号 | 短名 | 主责模块 | 关联模块 | +| --- | --- | --- | --- | +| OPS-TARGET-REQ-007 | 源码快照 | PJ2026-01060308 YAML目标治理 | [源码同步](PJ2026-010602-source-sync.md)、[发布流水](PJ2026-010601-controlled-release.md) | + +CI/CD source authority 必须由 owning YAML 声明。主路径应支持 `sourceAuthority.mode=gitMirrorSnapshot` 或等价字段，使受控 CLI 通过 k8s git-mirror resolver 选择本轮 `selectedCommit` 并创建不可变 snapshot/stageRef。`sourceAuthority.allowHostGit=false` 时，正式 `trigger-current`、image publish、GitOps render 和 closeout 不得执行 operator host `git ls-remote`、`git fetch`、`git clone`、`git rev-parse HEAD` 或读写固定 host workspace。 + +PipelineRun source step 不得以 mutable `refs/heads/` 作为本轮源码身份。它只能从 source snapshot 的 `stageRef`、commit-pinned mirror ref、PVC 或 bundle artifact checkout；若对象缺失，应返回 `source-snapshot-missing`，并输出受控 `git-mirror sync/ensure-snapshot` 下一步，不得回退到 GitHub direct clone、host workspace repair 或第二套 source resolver。 + +status/closeout 必须同时展示 selected snapshot、PipelineRun snapshot、latest mirror head、GitOps revision 和 runtime provenance。branch 在发布过程中前进只能标记为 `latest-drift` 或 `superseded`，不得把 selected commit 的成功发布误判为本轮失败。 + ## 7. 过程控制 本规格的执行 issue 为 [#911](https://github.com/pikasTech/unidesk/issues/911)。源码文件头部应标注 `SPEC: PJ2026-01060308 cicd-yaml-targets draft-2026-06-25-cicd-yaml-targets`；自动生成文件、纯配置、锁文件和无法承载注释头的二进制产物可例外。 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md b/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md index b2b26c8a..c83d9984 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md @@ -25,6 +25,7 @@ | Monitor Web 聚合实现引用版本 | draft-2026-06-26-p10-monitor-web-aggregation | | Monitor Web 观察面板治理实现引用版本 | draft-2026-06-27-p11-monitor-web-observability-dashboard; draft-2026-06-27-p12-cadence-scheduler-monitor-web | | Cadence/OTel 稳定性实现引用版本 | draft-2026-07-01-p15-cadence-otel | +| CI/CD source snapshot 实现引用版本 | draft-2026-07-01-p16-cicd-source-snapshot | | 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) | | 上级规格 | [PJ2026-010605 运维监控](PJ2026-010605-observability-monitoring.md) | | 关联规格 | [PJ2026-010401 Web工作台](PJ2026-010401-web-workbench.md)、[PJ2026-0104010803 Workbench唯一投影](PJ2026-0104010803-workbench-unique-projection.md)、[PJ2026-010403 API契约](PJ2026-010403-api-contract.md)、[PJ2026-010601 发布流水](PJ2026-010601-controlled-release.md)、[PJ2026-010602 源码同步](PJ2026-010602-source-sync.md)、[PJ2026-010603 YAML运维](PJ2026-010603-yaml-first-ops.md)、[PJ2026-010604 公开入口](PJ2026-010604-public-entry.md)、[PJ2026-01060505 Workbench性能](PJ2026-01060505-workbench-performance.md) | @@ -49,6 +50,7 @@ Web哨兵必须遵循 UniDesk YAML-first ops。目标 node/lane、public origin - 常驻 TypeScript 单 Pod wrapper 服务、scheduler、scenario runner、PVC/SQLite index、health、metrics、maintenance API 和 dashboard。 - `sentinel plan|apply|status|validate|report|maintenance` 与 `sentinel image|control-plane` 等受控 CLI 入口。 - 发布流水 maintenance start/stop、quick verify、targetValidation、GitOps/Argo/git-mirror closeout 和 public exposure 验证。 +- 哨兵自身 CI/CD 的 k8s git-mirror source snapshot、PipelineRun source acquisition、GitOps publish 和 selected/latest closeout。 - Dashboard 信息架构、规范化 API、前端组件分层、自动刷新、筛选、深链和 trace/turn 两层阅读视图。 - Vue `monitor-web` 观察面板的趋势曲线、固定视口三栏、运行时间线、cadence freshness、root cause 可见性和配套 CI/CD。 - `workbench-dsflash-go-tool-call-10x` 生产 canary 和 24 小时 dry-run 收口。 @@ -93,6 +95,7 @@ Web哨兵必须遵循 UniDesk YAML-first ops。目标 node/lane、public origin | auto refresh | Dashboard 的受控刷新能力；刷新只读 API/report/index，不发送 control command、不启动采样、不制造第二事实源。 | | public exposure | YAML 声明的 `monitor.pikapython.com` HTTPS 暴露，通过共享 PK01 Caddy + FRP managed-block helper 到达 ClusterIP Service。 | | targetValidation | 发布流水中的目标验证结果；对 HWLAB Web 恢复的判定必须来自 observe/analyze 对 public origin 的观察，不得只看 Argo green。 | +| source snapshot | 哨兵自身 CI/CD 触发时由 k8s git-mirror 解析出的不可变源码快照，绑定 selected commit、mirror ref、PipelineRun、GitOps revision 和 runtime image。 | | monitor-web | 独立于 sentinel-runner 的 Vue 3 + TypeScript + Vite 展示和聚合层，承载 `monitor.pikapython.com` root、多哨兵总览、趋势曲线、时间线、单哨兵详情和受控截图验收。 | | cadence freshness | 根据 YAML cadence、scheduler heartbeat、latest run age、active/planned run 和 analyzed report 更新时间计算的运行新鲜度；它默认是非阻塞值守告警，只有真正导致 run/report 不产生或业务链路不可用时才升级为 blocker。 | | env reuse | CI/CD 复用既有 env image、依赖缓存、BuildKit 层和未受影响服务的发布产物，以避免无关重建；小范围变更应在 status/closeout 中暴露 build/reuse 摘要。 | @@ -515,6 +518,10 @@ P8 恢复判定必须把 Workbench 业务失败继续 drill-down 到运行面依 Web哨兵自身必须纳入受控且独立的 sentinel control-plane：source 来自 UniDesk `master`，镜像、GitOps path、Argo Application、publicExposure 和 targetValidation 由 Web 哨兵 owning YAML 声明。D601/v03 当前可通过 `web-probe sentinel image|control-plane` 的独立 publish Job 实现构建、推送、GitOps 写回和 Argo 收敛；后续也可以切换到 Tekton Pipeline，但 builder 类型必须来自 YAML，不得依赖 operator 本地 dirty worktree。 +哨兵自身 CI/CD 的 source identity 必须来自 k8s git-mirror source snapshot。`trigger-current` 应先通过 YAML 声明的 git-mirror cache/service 在 k8s 内解析并同步 selected commit，再把同一 source snapshot 注入 publish PipelineRun、GitOps 写回、status 和 closeout。operator host 上的 `git ls-remote`、`git fetch`、`git clone`、`git rev-parse HEAD` 和固定 host worktree 读写不得参与正式 source selection。 + +publish PipelineRun 的 source step 只能消费 source snapshot 的 commit-pinned mirror ref、stageRef、PVC 或 bundle artifact；不得把 mutable `refs/heads/` 作为本轮 source identity，也不得在 mirror 对象缺失时回退到 GitHub direct clone 或 host workspace repair。mirror 缺对象时必须结构化失败为 `source-snapshot-missing` 或等价故障码，并输出受控 source-mirror sync/ensure 命令。 + 哨兵 rollout 与 HWLAB runtime rollout 不是同一个滚动单元。哨兵 dashboard/API/服务代码变更应通过 Web 哨兵独立 control-plane 滚动；HWLAB runtime 发布流水只调用当前已部署哨兵的 `maintenance/start`、`maintenance/stop` 和 quick verify 作为恢复判定。哨兵 control-plane 的顶层状态只表达哨兵自身 source、镜像、GitOps、Argo、runtime、metrics 和 dashboard 是否发布成功；HWLAB quick verify 必须作为独立 `targetValidation` 状态、warning 和 report 证据输出。哨兵 validate、maintenance 和 quick verify 控制路径必须优先走 k3s 内部 Service DNS，不绕 `monitor.pikapython.com` 公网入口。 哨兵镜像构建应使用 YAML 声明的 tools image、base image、registry、egress proxy 和 env-reuse 配方。Node/Bun/Playwright/Chromium 依赖不得在 runtime Pod 中临时下载。Secret 与 env 复用只走 sourceRef/keyMapping；日志、status、dashboard 和 issue closeout 只输出 object/key/presence/fingerprint/digest。 @@ -525,6 +532,8 @@ HWLAB runtime 发布 Pipeline 应在 Argo sync 前调用当前哨兵 `maintenanc `web-probe sentinel control-plane trigger-current --confirm --wait` 只等待 source mirror、publish、flush、publicExposure、Argo 和 runtime observed 收敛；CI/CD confirm-wait 超过 YAML `confirmWait.maxSeconds` 时必须输出 warning，并先优化等待阶段耗时，不得继续把长业务验证塞在部署同步路径里死等。`sentinel validate --quick-verify --confirm --wait` 和 maintenance stop quick verify 才执行 targetValidation 业务验证；业务 quick verify 的等待预算由 YAML `targetValidation.maxSeconds` 控制。计时超限本身只作为非阻塞告警；只有真正影响 Code Agent 多轮业务链路、submit/command 执行、trace/final 可见性或 session 连续性的失败才构成 targetValidation blocker。不得通过减少业务轮次、吞掉 submit 失败、fallback 到第二执行路径或读侧 repair 来消除红灯。 +status/closeout 必须同时展示 selected source snapshot、latest mirror head、PipelineRun source snapshot、GitOps revision、runtime image digest 和 runtime observed 状态。若 `master` 在发布过程中继续前进，只能标记为 `latest-drift` 或 `superseded`；不得把 selected commit 已发布成功的结果覆盖成“追最新失败”。 + ### 6.6 OPS-SENTINEL-REQ-006 dsflash-go 十轮 canary | 编号 | 短名 | 主责模块 | 关联模块 | diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 6c23af31..c63feb5b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -7,6 +7,7 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot. // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel. import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; @@ -61,6 +62,7 @@ export type WebProbeSentinelOptions = readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; + readonly rerun: boolean; } | { readonly kind: "publish"; @@ -72,6 +74,7 @@ export type WebProbeSentinelOptions = readonly confirm: boolean; readonly wait: boolean; readonly timeoutSeconds: number; + readonly rerun: boolean; } | { readonly kind: "maintenance"; @@ -156,7 +159,10 @@ interface SourceHead { readonly repository: string; readonly branch: string; readonly commit: string | null; - readonly localHead: string | null; + readonly stageRef: string | null; + readonly mirrorCommit: string | null; + readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot"; + readonly latestDrift: boolean; readonly result: CompactCommandResult; } @@ -218,12 +224,14 @@ export interface ChildCliResult { readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string }; } -const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel"; +const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot"; + +type SourceResolveMode = "cached" | "sync"; export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult { if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId)); requireSentinelIdForRegistry(spec, options.sentinelId, `web-probe sentinel ${options.kind}`); - const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds); + const state = loadSentinelCicdState(spec, options.sentinelId, options.timeoutSeconds, sentinelSourceResolveMode(options)); if (options.kind === "image") return runSentinelImage(state, options); if (options.kind === "control-plane") return runSentinelControlPlane(state, options); if (options.kind === "publish") return runSentinelPublishCurrent(state, options); @@ -233,6 +241,13 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: return runSentinelReport(state, options); } +function sentinelSourceResolveMode(options: WebProbeSentinelOptions): SourceResolveMode { + if (options.kind === "image" && options.action === "build" && options.confirm && options.wait) return "sync"; + if (options.kind === "control-plane" && options.action === "trigger-current" && options.confirm && options.wait) return "sync"; + if (options.kind === "publish" && options.confirm && options.wait) return "sync"; + return "cached"; +} + function runSentinelImage(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel image ${options.action}`; if (options.action === "build" && options.confirm) { @@ -280,7 +295,7 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract; let healthElapsedMs: number | null = null; @@ -439,7 +455,7 @@ function runSentinelPublishCurrentConfirmed(state: SentinelCicdState, options: E specRef: SPEC_REF, source: state.sourceHead, image: state.image, - pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state), + pipelineRun: record(controlResult).pipelineRun ?? sentinelPipelineRunName(state, options.rerun), controlPlane: controlResult, health, budget, @@ -571,7 +587,7 @@ function sentinelAlreadyCurrentControlResult(state: SentinelCicdState, observed: }; } -function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number): SentinelCicdState { +function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | null, timeoutSeconds: number, sourceResolveMode: SourceResolveMode): SentinelCicdState { const sentinel = resolveWebProbeSentinel(spec, sentinelId); const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime); @@ -585,7 +601,8 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef)); const nodeId = stringField(controlPlaneTarget, "node"); const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`); - const sourceHead = resolveSourceHead(cicd, timeoutSeconds); + validateSentinelSourceAuthority(cicd); + const sourceHead = resolveSourceHead(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds, sourceResolveMode); const image = sentinelImagePlan(spec, cicd, sourceHead); const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; @@ -609,23 +626,145 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | }; } -function resolveSourceHead(cicd: Record, timeoutSeconds: number): SourceHead { +function resolveSourceHead( + spec: HwlabRuntimeLaneSpec, + cicd: Record, + controlPlaneTarget: Record, + controlPlaneNode: Record, + timeoutSeconds: number, + mode: SourceResolveMode, +): SourceHead { const repository = stringAt(cicd, "source.repository"); const branch = stringAt(cicd, "source.branch"); - const remote = runCommand(["git", "ls-remote", "origin", `refs/heads/${branch}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const local = runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 10_000 }); - const commit = /^[0-9a-f]{40}\b/iu.exec(remote.stdout.trim())?.[0].toLowerCase() ?? null; - const localHead = /^[0-9a-f]{40}$/iu.test(local.stdout.trim()) ? local.stdout.trim().toLowerCase() : null; + const resolved = mode === "sync" + ? resolveSourceHeadWithK8sSnapshot(spec, cicd, controlPlaneTarget, controlPlaneNode, timeoutSeconds) + : probeSourceMirrorCache(cicd, controlPlaneNode, timeoutSeconds, null); + const probe = record(resolved.probe); + const commit = nonEmptyString(probe.sourceCommit) ?? nonEmptyString(probe.commit) ?? nonEmptyString(probe.mirrorCommit); + const stageRef = nonEmptyString(probe.stageRef) ?? (commit === null ? null : sentinelSourceSnapshotRef(cicd, commit)); + const mirrorCommit = nonEmptyString(probe.mirrorCommit) ?? nonEmptyString(probe.commit); return { - ok: remote.exitCode === 0 && commit !== null, + ok: resolved.ok === true && commit !== null, repository, branch, commit, - localHead, - result: compactCommand(remote), + stageRef, + mirrorCommit, + sourceAuthority: mode === "sync" ? "git-mirror-snapshot" : "git-mirror-cache", + latestDrift: commit !== null && mirrorCommit !== null && commit !== mirrorCommit, + result: compactCommand(resolved.result), }; } +function resolveSourceHeadWithK8sSnapshot( + spec: HwlabRuntimeLaneSpec, + cicd: Record, + controlPlaneTarget: Record, + controlPlaneNode: Record, + timeoutSeconds: number, +): { ok: boolean; probe: Record; result: CommandResult } { + const namespace = stringAt(cicd, "builder.namespace"); + const prefix = `${stringAt(cicd, "builder.jobPrefix")}-source-resolve`; + const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); + const manifest = sentinelSourceMirrorResolveJobManifest(spec, cicd, controlPlaneTarget, controlPlaneNode, jobName); + const created = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + if (created.exitCode !== 0) return { ok: false, probe: { ok: false, status: "create-failed", jobName, valuesRedacted: true }, result: created }; + const startedAt = Date.now(); + const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, 120_000)); + let lastCapture = created; + while (Date.now() - startedAt < timeoutMs) { + const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + lastCapture = probeCapture; + const probe = parseJsonObject(probeCapture.stdout) ?? {}; + const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); + if (probe.succeeded === true) return { ok: payload.ok === true, probe: payload, result: probeCapture }; + if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, valuesRedacted: true } : payload, result: probeCapture }; + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + } + return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture }; +} + +function sentinelSourceMirrorResolveJobManifest( + spec: HwlabRuntimeLaneSpec, + cicd: Record, + controlPlaneTarget: Record, + controlPlaneNode: Record, + jobName: string, +): Record { + const namespace = stringAt(cicd, "builder.namespace"); + const labels = { + "app.kubernetes.io/name": "web-probe-sentinel-source-resolve", + "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", + "unidesk.ai/spec-ref": "PJ2026-01060508", + "unidesk.ai/node": spec.nodeId, + "unidesk.ai/lane": spec.lane, + }; + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { name: jobName, namespace, labels }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: numberAt(cicd, "builder.activeDeadlineSeconds"), + ttlSecondsAfterFinished: numberAt(cicd, "builder.ttlSecondsAfterFinished"), + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + volumes: [ + sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget), + { name: "git-ssh", secret: { secretName: stringAt(cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, + ], + containers: [{ + name: "resolve", + image: sentinelSourceResolverImage(spec, cicd), + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShellFromConfig(cicd, controlPlaneNode, jobName, null)], + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }], + }, + }, + }, + }; +} + +function sentinelSourceResolverImage(spec: HwlabRuntimeLaneSpec, cicd: Record): string { + const baseImageRef = stringAt(cicd, "image.baseImageRef"); + return stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef); +} + +function validateSentinelSourceAuthority(cicd: Record): void { + const mode = stringAt(cicd, "sourceAuthority.mode"); + const resolver = stringAt(cicd, "sourceAuthority.resolver"); + const allowHostGit = booleanAt(cicd, "sourceAuthority.allowHostGit"); + const allowGithubDirectInPipeline = booleanAt(cicd, "sourceAuthority.allowGithubDirectInPipeline"); + const missingObjectPolicy = stringAt(cicd, "sourceSnapshot.missingObjectPolicy"); + if (mode !== "gitMirrorSnapshot") throw new Error("sourceAuthority.mode must be gitMirrorSnapshot"); + if (resolver !== "k8s-git-mirror") throw new Error("sourceAuthority.resolver must be k8s-git-mirror"); + if (allowHostGit !== false) throw new Error("sourceAuthority.allowHostGit must be false"); + if (allowGithubDirectInPipeline !== false) throw new Error("sourceAuthority.allowGithubDirectInPipeline must be false"); + if (missingObjectPolicy !== "fail-fast") throw new Error("sourceSnapshot.missingObjectPolicy must be fail-fast"); + sentinelSourceSnapshotStageRefPrefix(cicd); +} + +function sentinelSourceSnapshotStageRefPrefix(cicd: Record): string { + const branch = stringAt(cicd, "source.branch"); + const repository = stringAt(cicd, "source.repository"); + const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix") + .replaceAll("{branch}", branch) + .replaceAll("{repository}", repository) + .replace(/\/+$/u, ""); + if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix"); + return prefix; +} + +function sentinelSourceSnapshotRef(cicd: Record, commit: string): string { + return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`; +} + function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record, sourceHead: SourceHead): SentinelImagePlan { const repository = stringAt(cicd, "image.repository"); const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12); @@ -1409,7 +1548,7 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra const sourceMirrorSync = record(sourceMirrorProbe).ok === true ? sentinelSourceMirrorAlreadyPresentResult(state, sourceMirrorProbe) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); const sourceMirrorReady = sourceMirrorSync.ok === true; const publish = sourceMirrorReady - ? runSentinelPublishJob(state, false, options.timeoutSeconds) + ? runSentinelPublishJob(state, false, options.timeoutSeconds, false) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const registry = probeImageRegistry(state, options.timeoutSeconds); const registryReady = record(registry.probe).present === true; @@ -1474,7 +1613,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: const publish = applyOnly ? null : sourceMirrorReady - ? runSentinelPublishJob(state, true, remainingCommandSeconds()) + ? runSentinelPublishJob(state, true, remainingCommandSeconds(), options.rerun) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const publishWaitBudgetExhausted = !applyOnly && sourceMirrorReady && record(publish).ok !== true && remainingCicdWaitSeconds() <= 8; const flush = !applyOnly && !publishWaitBudgetExhausted && record(publish).ok === true @@ -1523,6 +1662,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: ? "one or more YAML-declared runtime Secrets were not synced from sourceRef" : "one or more publicExposure, Argo or runtime observation checks did not pass", }; + const publishPipelineRun = applyOnly ? sentinelPipelineRunName(state, options.rerun) : record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun); const result = { ok, command, @@ -1533,7 +1673,7 @@ function sentinelControlPlaneConfirmedResult(state: SentinelCicdState, options: specRef: SPEC_REF, source: state.sourceHead, image: state.image, - pipelineRun: sentinelPipelineRunName(state), + pipelineRun: publishPipelineRun, gitops: { path: stringAt(state.cicd, "gitopsPath"), targetRevision: stringAt(state.cicd, "argo.targetRevision"), @@ -1732,19 +1872,26 @@ function sentinelObservedWarnings(value: Record | SentinelObser } function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record { - const namespace = stringAt(state.cicd, "builder.namespace"); - const repository = stringAt(state.cicd, "source.repository"); - const branch = stringAt(state.cicd, "source.branch"); - const expectedCommit = state.sourceHead.commit; + const result = probeSourceMirrorCache(state.cicd, state.controlPlaneNode, timeoutSeconds, state.sourceHead.commit); + return { ...result, result: compactCommand(result.result) }; +} + +function probeSourceMirrorCache(cicd: Record, controlPlaneNode: Record, timeoutSeconds: number, expectedCommit: string | null): { ok: boolean; probe: Record; result: CommandResult } { + const namespace = stringAt(cicd, "builder.namespace"); + const repository = stringAt(cicd, "source.repository"); + const branch = stringAt(cicd, "source.branch"); + const stageRef = expectedCommit === null ? "" : sentinelSourceSnapshotRef(cicd, expectedCommit); const script = [ "set +e", `repo_path=${shellQuote(`/cache/${repository}.git`)}`, `branch=${shellQuote(branch)}`, `expected=${shellQuote(expectedCommit ?? "")}`, + `stage_ref=${shellQuote(stageRef)}`, "commit=$(kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse \\\"refs/heads/$branch\\\" 2>/dev/null\" 2>/dev/null)", "rc=$?", "object_rc=1", "expected_object_rc=1", + "stage_object_rc=1", "contains_rc=1", "if [ \"$rc\" -eq 0 ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$commit^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1", @@ -1754,22 +1901,27 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" cat-file -e \\\"$expected^{commit}\\\" 2>/dev/null\" >/dev/null 2>&1", " expected_object_rc=$?", "fi", + "if [ -n \"$stage_ref\" ]; then", + " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" rev-parse --verify \\\"$stage_ref^{commit}\\\" >/dev/null 2>&1\" >/dev/null 2>&1", + " stage_object_rc=$?", + "fi", "if [ \"$rc\" -eq 0 ] && [ \"$expected_object_rc\" -eq 0 ]; then", " kubectl -n " + shellQuote(namespace) + " exec deploy/git-mirror-http -- sh -lc \"git --git-dir=\\\"$repo_path\\\" merge-base --is-ancestor \\\"$expected\\\" \\\"$commit\\\" 2>/dev/null\" >/dev/null 2>&1", " contains_rc=$?", "fi", - "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$repo_path\" \"$branch\" <<'NODE'", - "const [rc, objectRc, expectedObjectRc, containsRc, commit, expected, repoPath, branch] = process.argv.slice(2);", + "node - \"$rc\" \"$object_rc\" \"$expected_object_rc\" \"$stage_object_rc\" \"$contains_rc\" \"$commit\" \"$expected\" \"$stage_ref\" \"$repo_path\" \"$branch\" <<'NODE'", + "const [rc, objectRc, expectedObjectRc, stageObjectRc, containsRc, commit, expected, stageRef, repoPath, branch] = process.argv.slice(2);", "const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');", "const objectPresent = present && Number(objectRc) === 0;", "const expectedObjectPresent = !expected || Number(expectedObjectRc) === 0;", + "const stageObjectPresent = !stageRef || Number(stageObjectRc) === 0;", "const containsExpected = !expected || commit === expected || Number(containsRc) === 0;", "const relation = !expected ? 'unconstrained' : commit === expected ? 'equal' : containsExpected ? 'mirror-ahead' : expectedObjectPresent ? 'diverged-or-behind' : 'expected-object-missing';", - "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror', present, objectPresent, expectedObjectPresent, containsExpected, relation, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));", + "console.log(JSON.stringify({ ok: objectPresent && expectedObjectPresent && containsExpected, mode: 'internal-git-mirror-cache', present, objectPresent, expectedObjectPresent, stageObjectPresent, containsExpected, relation, commit: present ? commit : null, sourceCommit: expected || (present ? commit : null), mirrorCommit: present ? commit : null, expectedCommit: expected || null, stageRef: stageRef || null, branch, repoPath, persistentMirrorPresent: objectPresent && expectedObjectPresent, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));", "NODE", ].join("\n"); - const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) }; + const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout) ?? {}, result }; } function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record { @@ -2130,18 +2282,22 @@ function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: } function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string { + return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit); +} + +function sentinelSourceMirrorSyncShellFromConfig(cicd: Record, controlPlaneNode: Record, jobName: string, selectedCommit: string | null): string { return [ "set -eu", `job_name=${shellQuote(jobName)}`, - `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`, - `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, - `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`, - `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, + `source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`, + `source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`, + `source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`, + `source_commit=${shellQuote(selectedCommit ?? "")}`, + `source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`, "started_ms=$(node -e 'console.log(Date.now())')", "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", "trap emit_failed EXIT", - "test -n \"$source_commit\"", - ...sentinelSourceMirrorSshSetupShellLines(state), + ...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode), "repo=\"/cache/${source_repository}.git\"", "mkdir -p \"$(dirname \"$repo\")\"", "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", @@ -2164,20 +2320,28 @@ function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string "done", "test \"$fetch_ok\" = 1", "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")", + "if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi", + "git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"", "test \"$mirror_commit\" = \"$source_commit\"", + "stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"", "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"", + "git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"", "git --git-dir=\"$repo\" update-server-info", "finished_ms=$(node -e 'console.log(Date.now())')", - "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$started_ms\" \"$finished_ms\" <<'NODE'", - "const [jobName, repository, branch, sourceCommit, mirrorCommit, startedMs, finishedMs] = process.argv.slice(2);", - "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));", + "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'", + "const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));", "NODE", "trap - EXIT", ].join("\n"); } function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] { - const proxy = record(valueAtPath(state.controlPlaneNode, "egressProxy")); + return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode); +} + +function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record): string[] { + const proxy = record(valueAtPath(controlPlaneNode, "egressProxy")); const serviceName = nonEmptyString(proxy.serviceName); const namespace = nonEmptyString(proxy.namespace); const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null; @@ -2299,8 +2463,8 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin ]; } -function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult { - const pipelineRunName = sentinelPipelineRunName(state); +function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult { + const pipelineRunName = sentinelPipelineRunName(state, rerun); const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops); const namespace = stringAt(state.cicd, "builder.namespace"); sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); @@ -2371,7 +2535,9 @@ function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRu namespace, labels, annotations: { - "unidesk.ai/source-commit": state.sourceHead.commit, + "unidesk.ai/source-commit": state.sourceHead.commit ?? "", + "unidesk.ai/source-authority": state.sourceHead.sourceAuthority, + "unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "", "unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"), "unidesk.ai/publish-gitops": publishGitops ? "true" : "false", }, @@ -2459,9 +2625,13 @@ function tektonShellScript(body: string): string { } function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record { - const hostPath = nonEmptyString(valueAtPath(state.controlPlaneTarget, "gitMirror.cacheHostPath")); + return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget); +} + +function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record): Record { + const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath")); if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } }; - return { name: "cache", persistentVolumeClaim: { claimName: stringAt(state.controlPlaneTarget, "gitMirror.cachePvcName") } }; + return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } }; } function sentinelBuildkitStateVolume(state: SentinelCicdState): Record { @@ -2524,6 +2694,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`, `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, + `source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`, `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`, `dockerfile_b64=${shellQuote(dockerfileB64)}`, `env_reuse_mode=${shellQuote(envReuseMode)}`, @@ -2542,6 +2713,9 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): "started_ms=$(now_ms)", "write_meta started_ms \"$started_ms\"", "write_meta source_commit \"$source_commit\"", + "write_meta source_stage_ref \"$source_stage_ref\"", + "test -n \"$source_commit\"", + "test -n \"$source_stage_ref\"", "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", @@ -2561,7 +2735,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): "source_fetch_started_ms=$(now_ms)", "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"", "emit_stage source-fetch running \"$source_fetch_started_ms\"", - "git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"", + "git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"", "git checkout --detach \"$source_commit\"", "mirror_commit=$(git rev-parse HEAD)", "test \"$mirror_commit\" = \"$source_commit\"", @@ -2720,6 +2894,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", "started_ms=$(read_meta started_ms)", "source_commit=$(read_meta source_commit)", + "source_stage_ref=$(read_meta source_stage_ref)", "mirror_commit=$(read_meta mirror_commit)", "source_fetch_started_ms=$(read_meta source_fetch_started_ms)", "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)", @@ -2781,11 +2956,11 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "fi", "gitops_finished_ms=$(now_ms)", "finished_ms=$(now_ms)", - "node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'", - "const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);", + "node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'", + "const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);", "const elapsed = (start, finish) => Number(finish) - Number(start);", "const cacheHits = Number(imageBuildCacheHits || 0);", - "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", "NODE", "trap - EXIT", ].join("\n"); @@ -2862,7 +3037,7 @@ function probeK8sJobScript(namespace: string, jobName: string): string { "pod_phase=''", "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", "logs_tail=''", - "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null || true; done; } | tail -c 16000 | base64 | tr -d '\\n'); fi", + "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi", "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", @@ -2878,12 +3053,15 @@ function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string "condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)", "condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)", "condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)", + "if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi", + "if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi", + "if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi", "task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", "pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", "pod_phase=''", "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi", "logs_tail=''", - "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=180 2>/dev/null || true; } | tail -c 24000 | base64 | tr -d '\\n'); fi", + "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi", "node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", "const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);", "const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');", @@ -3082,7 +3260,9 @@ function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, prob ok: true, status: "already-present", sourceCommit: state.sourceHead.commit, - mirrorCommit: state.sourceHead.commit, + mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit, + stageRef: state.sourceHead.stageRef, + sourceAuthority: state.sourceHead.sourceAuthority, valuesRedacted: true, }, polls: 0, @@ -3653,9 +3833,12 @@ export function displayPath(pathValue: string): string { return pathValue; } -function sentinelPipelineRunName(state: SentinelCicdState): string { +function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string { const commit = state.sourceHead.commit ?? "source"; - return `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`; + const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`; + if (!rerun) return base; + const suffix = `-r${Date.now().toString(36)}`; + return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`; } export function sentinelCliSuffix(state: SentinelCicdState): string { @@ -3795,9 +3978,11 @@ function renderPublishCurrentResult(result: Record): string { finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000), ]]), "", - table(["SOURCE", "COMMIT", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[ + table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[ `${source.repository ?? "-"}@${source.branch ?? "-"}`, short(source.commit), + source.sourceAuthority ?? "-", + short(source.stageRef), image.ref ?? "-", short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest), result.pipelineRun ?? publish.jobName ?? "-", @@ -3902,7 +4087,7 @@ function renderImageResult(result: Record): string { "", table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]), "", - table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]), + table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]), "", Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]), "", @@ -3912,7 +4097,7 @@ function renderImageResult(result: Record): string { "", Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), "", - Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]), + Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), "", Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), "", @@ -3958,7 +4143,7 @@ function renderControlPlaneResult(result: Record): string { "", table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]), "", - table(["SOURCE", "COMMIT", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), image.ref, short(gitops.manifestSha256)]]), + table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]), "", table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]), "", @@ -3968,7 +4153,7 @@ function renderControlPlaneResult(result: Record): string { "", renderObservedStatus(observed), "", - Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]), + Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), "", Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ targetValidation.ok, diff --git a/scripts/src/hwlab-node-web-sentinel-config.ts b/scripts/src/hwlab-node-web-sentinel-config.ts index 23280877..119f35f5 100644 --- a/scripts/src/hwlab-node-web-sentinel-config.ts +++ b/scripts/src/hwlab-node-web-sentinel-config.ts @@ -138,6 +138,14 @@ const REQUIRED_TARGET_SHAPES: Record/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null", + "}", + "mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")", + "mirror_stage_rc=$?", + "local_head=$(read_ref \"refs/heads/$source_branch\")", + "local_head_rc=$?", + "source_commit=\"$mirror_stage\"", + "source_ref=\"refs/mirror-stage/heads/$source_branch\"", + "if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then", + " source_commit=\"$local_head\"", + " source_ref=\"refs/heads/$source_branch\"", + "fi", + "node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'", + "const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);", + "const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');", + "const ok = isSha(sourceCommit);", + "console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));", + "NODE", + ].join("\n"); + const result = runNodeK3sScript(spec, script, 45); + const payload = parseJsonObject(result.stdout); + const payloadCommit = typeof payload.sourceCommit === "string" && /^[0-9a-f]{40}$/iu.test(payload.sourceCommit) ? payload.sourceCommit.toLowerCase() : null; + const match = payloadCommit ?? /[0-9a-f]{40}/iu.exec(statusText(result))?.[0].toLowerCase() ?? null; + return { sourceCommit: result.exitCode === 0 ? match : null, result }; +} + +interface NodeRuntimeSourceMirrorTarget { + readonly namespace: string; + readonly serviceReadName: string; + readonly sourceRepository: string; + readonly sourceBranch: string; +} + +function nodeRuntimeSourceMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeSourceMirrorTarget { + const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); + const parsed = runtimeRecord(Bun.YAML.parse(readFileSync(configPath, "utf8"))); + const targets = Array.isArray(parsed.targets) ? parsed.targets.map((item) => runtimeRecord(item)) : []; + const target = targets.find((item) => item.node === spec.nodeId && item.lane === spec.lane); + if (target === undefined) throw new Error(`no control-plane target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); + const gitMirror = runtimeRecord(target.gitMirror); + const source = runtimeRecord(target.source); + return { + namespace: runtimeString(gitMirror.namespace, "gitMirror.namespace"), + serviceReadName: runtimeString(gitMirror.serviceReadName, "gitMirror.serviceReadName"), + sourceRepository: runtimeString(source.repository, "source.repository"), + sourceBranch: runtimeString(source.branch, "source.branch"), + }; +} + +function runtimeRecord(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function runtimeString(value: unknown, path: string): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`); + return value; } export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { diff --git a/scripts/src/hwlab-node/entry.ts b/scripts/src/hwlab-node/entry.ts index 595999a5..2f299161 100644 --- a/scripts/src/hwlab-node/entry.ts +++ b/scripts/src/hwlab-node/entry.ts @@ -52,7 +52,7 @@ export type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cle export type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; -export type NodeRuntimeRenderLocation = "node-host" | "local"; +export type NodeRuntimeRenderLocation = "node-host"; export type WebProbeBrowserProxyMode = "auto" | "direct"; diff --git a/scripts/src/hwlab-node/git-mirror.ts b/scripts/src/hwlab-node/git-mirror.ts index d5e7aa9d..83c40370 100644 --- a/scripts/src/hwlab-node/git-mirror.ts +++ b/scripts/src/hwlab-node/git-mirror.ts @@ -34,7 +34,7 @@ import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WAR import { parseNodeScopedDelegatedOptions } from "./plan"; import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render"; import { compactRuntimeCommand } from "./runtime-common"; -import { compactNodeRuntimeGitMirrorObservation, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status"; +import { compactNodeRuntimeGitMirrorObservation, compactNodeRuntimeGitMirrorRun, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status"; import { record } from "./utils"; import { webObserveTable } from "./web-observe-render"; import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe"; @@ -51,6 +51,28 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType Date.now() - triggerStartedAt; + const sourceSnapshotSync = scoped.dryRun + ? null + : nodeRuntimeGitMirrorRun({ + ...scoped, + domain: "git-mirror", + action: "sync", + confirm: true, + dryRun: false, + wait: true, + discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true, + }); + if (sourceSnapshotSync !== null && sourceSnapshotSync.ok !== true) { + return { + ok: false, + command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + phase: "source-snapshot-sync", + degradedReason: "node-runtime-source-snapshot-sync-failed", + sourceSnapshotSync: nodeScopedFullOutput(scoped) ? sourceSnapshotSync : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync), + }; + } printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" }); const head = resolveNodeRuntimeLaneHead(spec); const sourceCommit = head.sourceCommit; @@ -64,6 +86,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType 0 ? diagnostics : null, degradedReason: "node-runtime-control-plane-apply-before-trigger-failed", next: { @@ -220,6 +247,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", - "rm -rf \"$render_dir\" \"$worktree_dir\"", - "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", - "echo \"phase=local-git-clone-worktree\" >&2", - "run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"", - "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"", - "cd \"$worktree_dir\"", - "echo \"phase=local-install-yaml\" >&2", - ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"), - "node - \"$overlay_b64\" <<'NODE'", - "const fs = require('fs');", - "const YAML = require('yaml');", - "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", - "const path = 'deploy/deploy.yaml';", - "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", - "doc.nodes = doc.nodes || {};", - "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", - "doc.lanes = doc.lanes || {};", - "const lane = doc.lanes[overlay.lane] || {};", - "const downloadStack = {", - " ...(lane.envRecipe?.downloadStack || {}),", - " httpProxy: overlay.dockerProxyHttp,", - " httpsProxy: overlay.dockerProxyHttps,", - " noProxy: overlay.dockerNoProxyList,", - "};", - "doc.lanes[overlay.lane] = {", - " ...lane,", - " node: overlay.nodeId,", - " sourceBranch: overlay.sourceBranch,", - " gitopsBranch: overlay.gitopsBranch,", - " namespace: overlay.runtimeNamespace,", - " endpoint: overlay.publicApiUrl,", - " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", - " artifactCatalog: overlay.catalogPath,", - " runtimePath: overlay.runtimePath,", - " imageTagMode: 'full',", - " sourceRepo: overlay.gitUrl,", - " externalPostgres: overlay.externalPostgres,", - " observability: overlay.observability,", - " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", - "};", - "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", - "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;", - "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;", - "fs.writeFileSync(path, YAML.stringify(doc));", - "NODE", - "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", - "echo \"phase=local-gitops-render\" >&2", - [ - "node scripts/run-bun.mjs \"$render_script\"", - `--lane ${shellQuote(spec.lane)}`, - `--node ${shellQuote(spec.nodeId)}`, - `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, - `--catalog-path ${shellQuote(spec.catalogPath)}`, - "--image-tag-mode full", - `--source-revision ${shellQuote(sourceCommit)}`, - `--source-repo ${shellQuote(spec.gitUrl)}`, - `--source-branch ${shellQuote(spec.sourceBranch)}`, - `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, - `--git-read-url ${shellQuote(spec.gitReadUrl)}`, - `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, - `--registry-prefix ${shellQuote(spec.registryPrefix)}`, - `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, - `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, - `--out ${shellQuote(renderDir)}`, - ].join(" "), - ...nodeRuntimePipelinePostprocessScript(), - ].join("\n"); - return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" }; -} - export function nodeRuntimePipelinePostprocessScript(): string[] { return [ "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'", diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 98a62edb..bbfd76d2 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -72,7 +72,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe "--timeout-ms", "--wait-timeout-ms", "--command-timeout-seconds", - ]), new Set(["--dry-run", "--confirm", "--wait", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"])); + ]), new Set(["--dry-run", "--confirm", "--wait", "--rerun", "--quick-verify", "--raw", "--full", "--latest", "--full-page", "--no-full-page"])); const node = requiredOption(args, "--node"); assertNodeId(node); const lane = requiredOption(args, "--lane"); @@ -96,9 +96,9 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe if (controlPlaneAction !== "plan" && controlPlaneAction !== "apply" && controlPlaneAction !== "status" && controlPlaneAction !== "trigger-current") { throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]"); } - sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds }; + sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, sentinelId, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") }; } else if (sentinelActionRaw === "publish-current") { - sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds }; + sentinel = { kind: "publish", action: "publish-current", node, lane, sentinelId, dryRun: dryRun || !confirm, confirm, wait: args.includes("--wait"), timeoutSeconds, rerun: args.includes("--rerun") }; } else if (sentinelActionRaw === "maintenance") { const maintenanceAction = args[1]; if (maintenanceAction !== "status" && maintenanceAction !== "start" && maintenanceAction !== "stop") {