From d687bdf66879984442ab069790dc8c344404547d Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 1 Jul 2026 11:03:18 +0000 Subject: [PATCH] fix: use k8s git mirror source snapshots --- .agents/skills/unidesk-cicd/SKILL.md | 1 + .../skills/unidesk-cicd/references/full.md | 8 +- config/agentrun.yaml | 34 ++++++++ config/hwlab-node-control-plane.yaml | 30 +++++++ config/hwlab-node-lanes.yaml | 10 +++ scripts/src/agentrun-lanes.ts | 80 ++++++++++++++++++- scripts/src/agentrun/cleanup-scripts.ts | 27 ++++++- scripts/src/agentrun/control-plane.ts | 12 ++- scripts/src/agentrun/public-exposure.ts | 7 +- scripts/src/agentrun/secrets.ts | 14 +++- scripts/src/agentrun/trigger.ts | 38 +++++++-- scripts/src/agentrun/yaml-lane.ts | 44 ++++++++-- scripts/src/hwlab-node-control-plane-model.ts | 17 +++- scripts/src/hwlab-node-control-plane.ts | 61 ++++++++++++-- scripts/src/hwlab-node-lanes.ts | 68 ++++++++++++++++ scripts/src/hwlab-node/cleanup.ts | 23 +++--- scripts/src/hwlab-node/git-mirror.ts | 5 +- scripts/src/hwlab-node/render.ts | 17 +++- scripts/src/hwlab-node/runtime-common.ts | 1 + scripts/src/hwlab-node/status.ts | 25 ++++-- 20 files changed, 465 insertions(+), 57 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 95b204b7..c1a1dbed 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -22,6 +22,7 @@ bun scripts/cli.ts agentrun control-plane status ## P0 边界 - CI/CD、GitOps、rollout、PipelineRun、Argo、git-mirror 和 AgentRun 部署必须走受控 CLI；不要用裸 `kubectl`、`argo`、`tkn`、`curl` 当正式控制入口。 +- CI/CD source authority 只能来自 Kubernetes 托管的 git-mirror snapshot：受控命令先同步 GitHub refs 到 k8s git-mirror，再创建/读取不可变 `refs/unidesk/snapshots/.../` stage ref；build/status/publish 只消费该 snapshot，host worktree、本地 `git fetch/pull`、可变 branch ref 或 Pipeline 内直连 GitHub 都不能作为 authoritative source。 - CI/CD、rollout、publish、image build 和部署链路禁止新引入 Docker 依赖；不得依赖 Docker socket、Docker daemon、host Docker、`docker build`、`docker push` 或等价 Docker-only 路径。 - 正式 CI/CD、publish、image build 和 rollout 必须走 Tekton Task/Pipeline/PipelineRun 承担 CI，并通过 GitOps/Argo 承担部署收敛；普通 Kubernetes Job 只允许用于 bounded helper、source sync、diagnostic、cleanup 或 bootstrap，不得作为正式发布、镜像构建或 rollout 入口。 - 正式 CI/CD 必须提供一键完成入口：同一受控命令应完成 source sync、构建、发布、GitOps/Argo 收敛、runtime provenance 校验和 `/health` 端点验证；不要要求操作者手动串联多个 publish/apply/status 命令才能完成一次交付。 diff --git a/.agents/skills/unidesk-cicd/references/full.md b/.agents/skills/unidesk-cicd/references/full.md index b569a179..793ab2d2 100644 --- a/.agents/skills/unidesk-cicd/references/full.md +++ b/.agents/skills/unidesk-cicd/references/full.md @@ -148,7 +148,7 @@ PipelineRun `gitops-promote` 如果报 git mirror 控制面漂移、refs 不一 node-scoped lane 可能在本次 PR 合并后又被后续 PR 推进。`control-plane status --pipeline-run ` 是定点观察某个 PipelineRun，但输出里的当前 `sourceHead` / `summary.sourceCommit` 可能已经是最新 branch tip，而不是该 PipelineRun 名称对应的 merge commit。closeout 必须同时记录 PR merge commit、PipelineRun 名称/状态、Argo sync revision、当前 branch tip，并用 `git merge-base --is-ancestor HEAD` 或等价证据说明最新 tip 包含本次 PR；不要只凭当前 source head 判断本次 rollout。 -`trigger-current --node D601 --lane v03 --confirm --wait` 的 source selection 必须走 k8s git-mirror source snapshot：confirmed trigger 先执行受控 `git-mirror sync`，再从 mirror cache 读取 `refs/mirror-stage/heads/` 作为本轮 source commit。旧 `source-render` / `local-git-clone-worktree` 追 branch tip 的问题不得再用固定 worktree fetch/pull 修复；如果 mirror 缺对象或 GitHub SSH transient 耗尽，命令应以 `source-snapshot-missing` 或 git-mirror retry exhausted 类故障停止，并给出受控 sync/status 下一步。 +`trigger-current --node D601|D518|JD01 --lane v03 --confirm --wait` 的 source selection 必须走 k8s git-mirror source snapshot：confirmed trigger 先执行受控 `git-mirror sync`，sync 在 mirror cache 中为本轮 branch tip 创建不可变 `refs/unidesk/snapshots/hwlab-node-runtime//`，随后 trigger/status/build 只读取该 snapshot ref 作为 authoritative source。旧 `source-render` / `local-git-clone-worktree` / 可变 branch ref 追 branch tip 的问题不得再用固定 worktree fetch/pull 修复；如果 mirror 缺对象或 snapshot ref 缺失，命令应以 `source-snapshot-missing` 或 git-mirror retry exhausted 类故障停止，并给出受控 sync/status 下一步。 D601/v03 `git-mirror` 的 GitHub upstream 标准传输固定为 YAML 声明的 SSH：`githubTransport.mode=ssh`，脚本通过 `GIT_SSH` wrapper 访问 `ssh://git@ssh.github.com:443/...`；node-global HTTP proxy 只作为 SSH CONNECT tunnel，不是 GitHub HTTPS auth/token transport。若 CLI 输出 `transport=https`、`GITHUB_TOKEN`、`git-mirror-github-token` 或 HTTPS token sourceRef，按 control-plane drift/配置回归处理：先修 `config/hwlab-node-control-plane.yaml` 并执行 `hwlab nodes control-plane apply --node D601 --lane v03 --confirm`，不要改走 HTTPS、不要增加 fallback、不要用 host workspace repair。`sync/flush` 的 retry 只消费 SSH upstream transient，并在耗尽后输出 stopped/exhausted；promotion 后若 node-local `git-mirror status` 显示 `pendingFlush=true`，执行 node-local flush 并等到 `pendingFlush=false`、`githubInSync=true`。 @@ -272,13 +272,13 @@ bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--pipel - `apply`: 按 YAML 渲染并 apply Tekton RBAC/Pipeline、Argo AppProject/Application 和 runtime namespace - `secret-sync`: 按 YAML 的 Secret sourceRef/keyMapping 同步 runtime Secret 和外置 DB Secret，只输出 fingerprint - `restart`: patch manager Deployment 的 restart annotation 并等待 rollout，用于 Secret export/DB 连接串变化后让 workload 读取新 Secret；不要手工删除 Pod -- `trigger-current`: 确保 source branch/workspace，删除新 lane source branch 的 `deploy/deploy.json`，构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，触发 git-mirror sync 和 provenance PipelineRun；confirmed 运行可返回异步 job，必须用 `job status --tail-bytes 12000` 看 `agentrun-yaml-lane-trigger` progress，再用 `status --pipeline-run ` 轮询收口；confirmed 收尾会尝试恢复 fixed source workspace 到 lane branch，恢复失败只作为 warning 披露 +- `trigger-current`: v0.2 lane 的 source authority 只读 k8s git-mirror snapshot。confirmed 运行先触发受控 `git-mirror sync`，为 source branch tip 创建 `refs/unidesk/snapshots/agentrun-yaml-lane//`，再从该 snapshot 构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，flush git-mirror 并创建 provenance PipelineRun；dry-run/status 也只展示 snapshot/sourceStageRef，不把 host workspace 当 source。confirmed 运行可返回异步 job，必须用 `job status --tail-bytes 12000` 看 `agentrun-yaml-lane-trigger` progress，再用 `status --pipeline-run ` 轮询收口。v0.1 兼容入口仍可能使用固定 source workspace；v0.2 不恢复也不修复 host worktree。 - `cleanup-runners`: 只清 YAML 选中 lane runtime namespace 中匹配 `deployment.runner.retention.selectors` 的 runner Job/Pod；runner 上限、最后活跃排序、active heartbeat 窗口、age-based cleanup 开关和 selector 都以 YAML 为准。`dry-run` 必须先看 manager facts、inactive candidates、selected 和 `activeRunRisk`；普通 `--confirm` 只删除 selected inactive runner，不替代 CI PipelineRun 清理。`--force-active` 只用于 operator 明确决定强杀所有匹配 runner pod/job 的资源恢复场景，必须先 dry-run 确认 `criteria.forceActive=true` 和 selection 范围；它会中断 active run/command/session，但仍优先于裸 `kubectl delete pod/job`。 - `status`: 默认返回 compact commander JSON，关键结论在 `.data.summary` 和 `.data.alignment`，完整 YAML target、原始 source/runtime/gitMirror payload 和成功 probe tail 只在 `--full|--raw` 展开 -YAML-only lane 的长步骤必须由 CLI 拆成短提交和状态轮询：source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建不得塞进一个顶层 `trans` 长连接。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败发布留下 dirty/detached/GitOps branch 状态，只清理已知发布残留并恢复到 lane source branch 后再重试。后台步骤的 `status` 和 `ok` 要共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不继续轮询到超时。 +YAML-only lane 的长步骤必须由 CLI 拆成短提交和状态轮询：k8s git-mirror snapshot sync、image build、GitOps publish、git-mirror flush 和 PipelineRun 创建不得塞进一个顶层 `trans` 长连接。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；v0.2 若历史失败发布留下 dirty/detached/GitOps branch 状态，也不得把 host workspace 作为 source 修复入口，只清理已知发布残留并从 git-mirror snapshot 重新触发。后台步骤的 `status` 和 `ok` 要共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不继续轮询到超时。 -AgentRun YAML-only lane closeout 必须同时看当前 source branch tip、目标 PipelineRun、GitOps revision、Argo revision 和 manager source commit。发布过程中如果 source branch 被并行 PR 推进，`status --pipeline-run ` 会通过 `summary.branchDrift` / `alignment.branchDrift` 标记目标 PipelineRun 是否已被当前 branch tip supersede；先确认最新 tip 包含本次修复，再按最新 tip 重新 `trigger-current`。最终只用最新 PipelineRun 的 `status` 中 `aligned=true`、`blockers=[]`、`argoSyncedToGitops=true` 和 `managerSourceMatchesExpected=true` 收口。`trigger-current` 会尝试恢复 YAML 声明的固定 source workspace 到 lane branch；若返回 `source-worktree-restore-failed` warning、workspace dirty，或后续 `status` 仍显示 `summary.source.workspaceDetached=true`，先按 `sourceWorkspaceRestore.failureKind` 修复 fixed workspace，再继续下一轮开发或部署。 +AgentRun YAML-only lane closeout 必须同时看当前 k8s git-mirror source snapshot、目标 PipelineRun、GitOps revision、Argo revision 和 manager source commit。发布过程中如果 source branch 被并行 PR 推进，`status --pipeline-run ` 会通过 `summary.branchDrift` / `alignment.branchDrift` 标记目标 PipelineRun 是否已被当前 snapshot tip supersede；先确认最新 snapshot commit 包含本次修复，再按最新 snapshot 重新 `trigger-current`。最终只用最新 PipelineRun 的 `status` 中 `aligned=true`、`blockers=[]`、`argoSyncedToGitops=true` 和 `managerSourceMatchesExpected=true` 收口。v0.2 `trigger-current` 不再恢复 YAML 声明的固定 source workspace；若看到 `source-worktree-restore-failed`、workspace dirty 或 `summary.source.workspaceDetached=true` 作为 v0.2 blocker，按 source authority 回归处理，修 CLI/配置让状态来自 git-mirror snapshot。 Runner egress proxy 只从 `config/agentrun.yaml` 的 `deployment.runner.egressProxyUrl` 与 `deployment.runner.noProxyExtra` 进入部署；manager Deployment 必须带 `AGENTRUN_RUNNER_EGRESS_PROXY_URL` 与 `AGENTRUN_RUNNER_NO_PROXY_EXTRA`，验收时还要用真实 `create/apply/send` 触发 runner Job，检查 Pod env、event/trace 和 final response。GitOps 已更新但 Argo 仍在旧 revision 时，走 `agentrun control-plane refresh --node --lane --confirm`，不要手工 patch runtime。 diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 7e8412cc..c2f73d23 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -277,8 +277,19 @@ controlPlane: node: D601 version: v0.2 source: + statusMode: k3s-git-mirror repository: pikasTech/agentrun branch: v0.2 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot bootstrapFromBranch: v0.1 bootstrapTimeoutSeconds: 900 bootstrapPollSeconds: 15 @@ -297,6 +308,7 @@ controlPlane: serviceAccountName: agentrun-v02-tekton-runner registryPrefix: 127.0.0.1:5000/agentrun toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless gitops: branch: v0.2-gitops path: deploy/gitops/node/d601/runtime-v02 @@ -571,6 +583,16 @@ controlPlane: statusMode: k3s-git-mirror repository: pikasTech/agentrun branch: v0.2 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot bootstrapFromBranch: v0.1 bootstrapTimeoutSeconds: 900 bootstrapPollSeconds: 15 @@ -850,8 +872,19 @@ controlPlane: node: D518 version: v0.2 source: + statusMode: k3s-git-mirror repository: pikasTech/agentrun branch: v0.2 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/agentrun-yaml-lane/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot bootstrapFromBranch: v0.1 bootstrapTimeoutSeconds: 900 bootstrapPollSeconds: 15 @@ -870,6 +903,7 @@ controlPlane: serviceAccountName: agentrun-d518-v02-tekton-runner registryPrefix: 127.0.0.1:5000/agentrun toolsImage: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + buildkitImage: 127.0.0.1:5000/hwlab/buildkit:rootless gitops: branch: d518-v0.2-gitops path: deploy/gitops/node/d518/runtime-v02 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index bf41b3a2..cbd5d04d 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -276,6 +276,16 @@ targets: source: repository: pikasTech/HWLAB branch: v0.3 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/hwlab-node-runtime/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot gitops: branch: v0.3-gitops path: deploy/gitops/node/d601/runtime-v03 @@ -470,6 +480,16 @@ targets: source: repository: pikasTech/HWLAB branch: v0.3 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/hwlab-node-runtime/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot gitops: branch: v0.3-gitops path: deploy/gitops/node/jd01/runtime-v03 @@ -680,6 +700,16 @@ targets: source: repository: pikasTech/HWLAB branch: v0.3 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/hwlab-node-runtime/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot gitops: branch: v0.3-gitops path: deploy/gitops/node/d518/runtime-v03 diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 5496b06a..301a3c66 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -87,6 +87,16 @@ lanes: minor: 3 version: v0.3 sourceBranch: v0.3 + sourceAuthority: + mode: gitMirrorSnapshot + resolver: k8s-git-mirror + allowHostGit: false + allowHostWorkspace: false + allowGithubDirectInPipeline: false + sourceSnapshot: + stageRefPrefix: refs/unidesk/snapshots/hwlab-node-runtime/{branch} + missingObjectPolicy: fail-fast + refreshPolicy: sync-before-snapshot workspace: /root/hwlab-v03 cicdRepo: /root/hwlab-v03-cicd.git cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 61b3dfac..8001b8f8 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -23,6 +23,20 @@ export interface AgentRunGitMirrorRepositorySpec { readonly gitopsBranch?: string; } +export interface AgentRunSourceAuthoritySpec { + readonly mode: "gitMirrorSnapshot"; + readonly resolver: "k8s-git-mirror"; + readonly allowHostGit: false; + readonly allowHostWorkspace: false; + readonly allowGithubDirectInPipeline: false; +} + +export interface AgentRunSourceSnapshotSpec { + readonly stageRefPrefix: string; + readonly missingObjectPolicy: "fail-fast"; + readonly refreshPolicy: "sync-before-snapshot"; +} + export interface AgentRunSecretRef { readonly namespace: string; readonly name: string; @@ -57,6 +71,8 @@ export interface AgentRunLaneSpec { readonly statusMode: "host-worktree" | "k3s-git-mirror"; readonly repository: string; readonly branch: string; + readonly sourceAuthority: AgentRunSourceAuthoritySpec | null; + readonly sourceSnapshot: AgentRunSourceSnapshotSpec | null; readonly bootstrapFromBranch: string | null; readonly bootstrapTimeoutSeconds: number; readonly bootstrapPollSeconds: number; @@ -278,6 +294,8 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record>(configPath); @@ -492,16 +521,20 @@ function parseLane(lane: string, node: AgentRunNodeSpec, input: Record parseLaneSecret(secret, `${path}.secrets[${index}]`)), }; + validateAgentRunLaneSourceAuthority(spec, path); + return spec; } function sourceStatusModeField(value: string, path: string): "host-worktree" | "k3s-git-mirror" { @@ -559,6 +594,45 @@ function sourceStatusModeField(value: string, path: string): "host-worktree" | " return value; } +function sourceAuthorityConfig(value: unknown, path: string): AgentRunSourceAuthoritySpec | null { + if (value === undefined) return null; + const raw = asRecord(value, path); + return { + mode: enumField(raw, "mode", path, ["gitMirrorSnapshot"]), + resolver: enumField(raw, "resolver", path, ["k8s-git-mirror"]), + allowHostGit: falseBooleanField(raw, "allowHostGit", path), + allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path), + allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path), + }; +} + +function sourceSnapshotConfig(value: unknown, path: string): AgentRunSourceSnapshotSpec | null { + if (value === undefined) return null; + const raw = asRecord(value, path); + const stageRefPrefix = stringField(raw, "stageRefPrefix", path); + if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`); + if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`); + if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`); + return { + stageRefPrefix, + missingObjectPolicy: enumField(raw, "missingObjectPolicy", path, ["fail-fast"]), + refreshPolicy: enumField(raw, "refreshPolicy", path, ["sync-before-snapshot"]), + }; +} + +function falseBooleanField(obj: Record, key: string, path: string): false { + const value = booleanField(obj, key, path); + if (value !== false) throw new Error(`${path}.${key} must be false`); + return false; +} + +function validateAgentRunLaneSourceAuthority(spec: AgentRunLaneSpec, path: string): void { + if (spec.version !== "v0.2") return; + if (spec.source.statusMode !== "k3s-git-mirror") throw new Error(`${path}.source.statusMode must be k3s-git-mirror for AgentRun v0.2`); + if (spec.source.sourceAuthority === null) throw new Error(`${path}.source.sourceAuthority is required for AgentRun v0.2`); + if (spec.source.sourceSnapshot === null) throw new Error(`${path}.source.sourceSnapshot is required for AgentRun v0.2`); +} + function parseDeployment(input: Record, path: string): AgentRunLaneSpec["deployment"] { const argocd = recordField(input, "argocd", path); const manager = recordField(input, "manager", path); diff --git a/scripts/src/agentrun/cleanup-scripts.ts b/scripts/src/agentrun/cleanup-scripts.ts index 2b04db89..fd5f4cbb 100644 --- a/scripts/src/agentrun/cleanup-scripts.ts +++ b/scripts/src/agentrun/cleanup-scripts.ts @@ -201,11 +201,31 @@ export async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig async function resolveTriggerCurrentSource(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise & { ok: boolean; sourceCommit?: string | null; sourcePayload?: Record }> { if (spec.source.statusMode === "k3s-git-mirror") { + progressEvent("agentrun.yaml-lane.source-snapshot.progress", { + node: spec.nodeId, + lane: spec.lane, + statusMode: spec.source.statusMode, + status: "syncing", + }); + const sourceSync = await runYamlLaneGitMirrorSyncJob(config, spec); + if (sourceSync.ok !== true) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-snapshot-sync", + degradedReason: "yaml-lane-k3s-source-snapshot-sync-failed", + result: sourceSync, + valuesPrinted: false, + }; + } progressEvent("agentrun.yaml-lane.source-status.progress", { node: spec.nodeId, lane: spec.lane, statusMode: spec.source.statusMode, - status: "probing", + status: "probing-snapshot", }); const sourceStatus = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)]); const sourcePayload = captureJsonPayload(sourceStatus); @@ -218,13 +238,14 @@ async function resolveTriggerCurrentSource(config: UniDeskConfig, spec: AgentRun configPath, target: agentRunLaneSummary(spec), phase: "source-status", - degradedReason: "yaml-lane-k3s-source-status-failed", + degradedReason: stringOrNull(sourcePayload.degradedReason) ?? "yaml-lane-k3s-source-status-failed", + gitMirrorSync: sourceSync, result: sourcePayload, capture: compactCapture(sourceStatus, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), valuesPrinted: false, }; } - return { ok: true, sourceCommit, sourcePayload, valuesPrinted: false }; + return { ok: true, sourceCommit, sourcePayload: { ...sourcePayload, gitMirrorSync: sourceSync, valuesPrinted: false }, valuesPrinted: false }; } progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { diff --git a/scripts/src/agentrun/control-plane.ts b/scripts/src/agentrun/control-plane.ts index d5fc8f67..7c557ede 100644 --- a/scripts/src/agentrun/control-plane.ts +++ b/scripts/src/agentrun/control-plane.ts @@ -234,7 +234,7 @@ export async function controlPlanePlan(_config: UniDeskConfig, options: StatusOp target: agentRunLaneSummary(spec), plannedChecks: [ "source-branch-exists", - "source-worktree-exists-and-clean", + spec.source.statusMode === "k3s-git-mirror" ? "source-git-mirror-snapshot-present" : "source-worktree-exists-and-clean", "git-mirror-services-ready", "ci-namespace-pipeline-serviceaccount", "argo-application-alignment", @@ -369,9 +369,11 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio ...(sourceWorktreeDetached ? ["source-worktree-detached"] : []), ...(sourceBranchAdvanced ? ["source-branch-advanced-after-target"] : []), ]; + const sourceSnapshotRequired = spec.source.statusMode === "k3s-git-mirror"; const blockers = [ ...(sourceWorkspaceRequired && sourcePayload.workspaceExists !== true ? ["source-worktree-missing"] : []), ...(sourcePayload.remoteBranchExists === true ? [] : ["source-branch-missing"]), + ...(sourceSnapshotRequired && sourcePayload.snapshotPresent !== true ? ["source-snapshot-missing"] : []), ...(sourceWorkspaceRequired && sourcePayload.workspaceClean !== true && sourcePayload.workspaceExists === true ? ["source-worktree-dirty"] : []), ...(mirrorPayload.readReady === true ? [] : ["git-mirror-read-not-ready"]), ...(mirrorPayload.writeReady === true ? [] : ["git-mirror-write-not-ready"]), @@ -426,11 +428,15 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio : { code: "inspect-full-status", summary: "alignment is inconclusive; inspect full status details", command: statusFullCommand }; const compactSourceStatus = { statusMode: spec.source.statusMode, + sourceAuthority: sourcePayload.sourceAuthority ?? spec.source.sourceAuthority?.mode ?? null, workspaceExists: sourcePayload.workspaceExists ?? false, workspaceClean: sourcePayload.workspaceClean ?? null, branch: sourcePayload.branch ?? null, remoteBranchExists: sourcePayload.remoteBranchExists ?? false, remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, + sourceRef: sourcePayload.sourceRef ?? null, + sourceStageRef: sourcePayload.sourceStageRef ?? null, + snapshotPresent: sourcePayload.snapshotPresent ?? null, workspaceDetached: sourceWorktreeDetached, }; const compactGitMirrorStatus = { @@ -499,6 +505,10 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio localHead: sourcePayload.localHead ?? null, remoteBranchExists: sourcePayload.remoteBranchExists ?? false, remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, + sourceAuthority: sourcePayload.sourceAuthority ?? spec.source.sourceAuthority?.mode ?? null, + sourceRef: sourcePayload.sourceRef ?? null, + sourceStageRef: sourcePayload.sourceStageRef ?? null, + snapshotPresent: sourcePayload.snapshotPresent ?? null, }, gitMirror: { readReady: mirrorPayload.readReady ?? false, diff --git a/scripts/src/agentrun/public-exposure.ts b/scripts/src/agentrun/public-exposure.ts index eb5e1338..46109d76 100644 --- a/scripts/src/agentrun/public-exposure.ts +++ b/scripts/src/agentrun/public-exposure.ts @@ -69,6 +69,11 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record): string[] { return Object.values(next) @@ -409,18 +409,34 @@ export async function triggerCurrent(config: UniDeskConfig, options: TriggerOpti export async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOptions, target: { configPath: string; spec: AgentRunLaneSpec }): Promise> { const spec = target.spec; - const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapProbeScript(spec)]); + const probe = spec.source.statusMode === "k3s-git-mirror" + ? await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)]) + : await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneSourceBootstrapProbeScript(spec)]); const source = captureJsonPayload(probe); const sourceCommit = stringOrNull(source.sourceCommit); const remoteBranchExists = source.remoteBranchExists === true; const pipelineRun = sourceCommit !== null && isGitSha(sourceCommit) ? agentRunPipelineRunName(spec, sourceCommit) : null; const placeholderImage = sourceCommit === null ? null : placeholderAgentRunImage(spec, sourceCommit); const renderedFiles = placeholderImage === null ? [] : renderAgentRunGitopsFiles(spec, { sourceCommit, image: placeholderImage }); - const plan = { - node: spec.nodeId, - lane: spec.lane, - version: spec.version, - source: { + const sourcePlan = spec.source.statusMode === "k3s-git-mirror" + ? { + statusMode: spec.source.statusMode, + sourceAuthority: spec.source.sourceAuthority, + sourceSnapshot: spec.source.sourceSnapshot, + repository: spec.source.repository, + branch: spec.source.branch, + remoteBranchExists, + remoteBranchCommit: stringOrNull(source.remoteBranchCommit), + sourceCommit, + sourceRef: stringOrNull(source.sourceRef), + sourceStageRef: stringOrNull(source.sourceStageRef), + snapshotPresent: source.snapshotPresent === true, + degradedReason: stringOrNull(source.degradedReason), + next: source.snapshotPresent === true ? null : `bun scripts/cli.ts agentrun git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, + valuesPrinted: false, + } + : { + statusMode: spec.source.statusMode, workspace: spec.source.workspace, remote: spec.source.remote, branch: spec.source.branch, @@ -429,7 +445,13 @@ export async function triggerCurrentYamlLane(config: UniDeskConfig, options: Tri bootstrapPollSeconds: spec.source.bootstrapPollSeconds, remoteBranchExists, sourceCommit, - }, + valuesPrinted: false, + }; + const plan = { + node: spec.nodeId, + lane: spec.lane, + version: spec.version, + source: sourcePlan, deploymentFormat: spec.deployment.format, deploymentTruth: "config/agentrun.yaml", removedServiceDeployJson: true, diff --git a/scripts/src/agentrun/yaml-lane.ts b/scripts/src/agentrun/yaml-lane.ts index 38be1fbd..7b3725c1 100644 --- a/scripts/src/agentrun/yaml-lane.ts +++ b/scripts/src/agentrun/yaml-lane.ts @@ -18,6 +18,8 @@ import { agentRunLaneSummary, agentRunPipelineRunName, agentRunProviderCredentialRefs, + agentRunSourceSnapshotRef, + agentRunSourceSnapshotStageRefPrefix, resolveAgentRunLaneTarget, type AgentRunCancelLifecycleSpec, type AgentRunLaneSpec, @@ -390,21 +392,41 @@ export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string { `read_deployment=${shQuote(spec.gitMirror.readDeployment)}`, `repository=${shQuote(spec.source.repository)}`, `source_branch=${shQuote(spec.source.branch)}`, + `source_stage_ref_prefix=${shQuote(agentRunSourceSnapshotStageRefPrefix(spec))}`, "repo_path=\"/cache/${repository}.git\"", "kubectl -n \"$namespace\" get deploy \"$read_deployment\" >/tmp/agentrun-source-read-deploy.txt 2>/dev/null", "read_exit=$?", "source_commit=''", + "source_ref=\"refs/mirror-stage/heads/$source_branch\"", + "source_stage_ref=''", + "snapshot_commit=''", "if [ \"$read_exit\" -eq 0 ]; then", - " source_commit=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$source_branch^{commit}\" 2>/dev/null || true' sh \"$repo_path\" \"$source_branch\" 2>/dev/null | tail -n 1 | tr -d '\\r')", + " refs=$(kubectl -n \"$namespace\" exec deploy/\"$read_deployment\" -- sh -lc 'repo_path=\"$1\"; source_branch=\"$2\"; stage_ref_prefix=\"$3\"; source_ref=\"refs/mirror-stage/heads/$source_branch\"; source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_ref^{commit}\" 2>/dev/null || true); source_stage_ref=\"\"; snapshot_commit=\"\"; if printf \"%s\" \"$source_commit\" | grep -Eq \"^[0-9a-fA-F]{40}$\"; then source_stage_ref=\"${stage_ref_prefix%/}/$source_commit\"; snapshot_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"$source_stage_ref^{commit}\" 2>/dev/null || true); fi; printf \"source_commit=%s\\nsource_stage_ref=%s\\nsnapshot_commit=%s\\n\" \"$source_commit\" \"$source_stage_ref\" \"$snapshot_commit\"' sh \"$repo_path\" \"$source_branch\" \"$source_stage_ref_prefix\" 2>/dev/null)", + " source_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_commit=//p' | tail -n 1 | tr -d '\\r')", + " source_stage_ref=$(printf '%s\\n' \"$refs\" | sed -n 's/^source_stage_ref=//p' | tail -n 1 | tr -d '\\r')", + " snapshot_commit=$(printf '%s\\n' \"$refs\" | sed -n 's/^snapshot_commit=//p' | tail -n 1 | tr -d '\\r')", "fi", - "export namespace read_deployment repository source_branch read_exit source_commit", + "export namespace read_deployment repository source_branch read_exit source_commit source_ref source_stage_ref snapshot_commit", "python3 - <<'PY'", - "import json, os", - "source_commit = os.environ.get('source_commit') or None", + "import json, os, re", + "def sha(value):", + " return (value or '').lower() if re.match(r'^[0-9a-f]{40}$', value or '', re.I) else None", + "source_commit = sha(os.environ.get('source_commit'))", + "snapshot_commit = sha(os.environ.get('snapshot_commit'))", + "source_stage_ref = os.environ.get('source_stage_ref') or None", "read_ready = os.environ.get('read_exit') == '0'", + "snapshot_present = source_commit is not None and snapshot_commit == source_commit", + "degraded_reason = None", + "if not read_ready:", + " degraded_reason = 'git-mirror-read-not-ready'", + "elif source_commit is None:", + " degraded_reason = 'source-branch-missing'", + "elif not snapshot_present:", + " degraded_reason = 'source-snapshot-missing'", "print(json.dumps({", - " 'ok': read_ready and source_commit is not None,", + " 'ok': read_ready and snapshot_present,", " 'statusMode': 'k3s-git-mirror',", + " 'sourceAuthority': 'git-mirror-snapshot',", " 'expectedWorkspace': None,", " 'actualWorkspace': None,", " 'workspaceExists': None,", @@ -415,12 +437,18 @@ export function yamlLaneK3sSourceStatusScript(spec: AgentRunLaneSpec): string { " 'remoteBranch': os.environ.get('source_branch'),", " 'remoteBranchExists': source_commit is not None,", " 'remoteBranchCommit': source_commit,", - " 'sourceCommit': source_commit,", + " 'sourceCommit': snapshot_commit if snapshot_present else source_commit,", + " 'sourceRef': os.environ.get('source_ref'),", + " 'sourceStageRef': source_stage_ref,", + " 'snapshotPresent': snapshot_present,", + " 'degradedReason': degraded_reason,", " 'gitMirror': {", " 'namespace': os.environ.get('namespace'),", " 'readDeployment': os.environ.get('read_deployment'),", " 'readReady': read_ready,", " 'repository': os.environ.get('repository'),", + " 'sourceRef': os.environ.get('source_ref'),", + " 'sourceStageRef': source_stage_ref,", " },", " 'statusShort': None,", " 'valuesPrinted': False,", @@ -1168,12 +1196,12 @@ function yamlLaneK3sBuildSourceShell(spec: AgentRunLaneSpec, sourceCommit: strin return [ "set -eu", `read_url=${shQuote(spec.gitMirror.readUrl)}`, - `source_branch=${shQuote(spec.source.branch)}`, `source_commit=${shQuote(sourceCommit)}`, + `source_stage_ref=${shQuote(agentRunSourceSnapshotRef(spec, sourceCommit))}`, "rm -rf /workspace/repo", "git clone --no-checkout \"$read_url\" /workspace/repo", "cd /workspace/repo", - "git fetch origin \"refs/heads/$source_branch:refs/remotes/origin/$source_branch\"", + "git fetch origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"", "git checkout --detach \"$source_commit\"", "actual=$(git rev-parse HEAD)", "test \"$actual\" = \"$source_commit\"", diff --git a/scripts/src/hwlab-node-control-plane-model.ts b/scripts/src/hwlab-node-control-plane-model.ts index 99b52f9a..7902dd05 100644 --- a/scripts/src/hwlab-node-control-plane-model.ts +++ b/scripts/src/hwlab-node-control-plane-model.ts @@ -294,7 +294,22 @@ export interface ControlPlaneTargetSpec { enabled: boolean; ciNamespace: string; runtimeNamespace: string; - source: { repository: string; branch: string }; + source: { + repository: string; + branch: string; + sourceAuthority: { + mode: "gitMirrorSnapshot"; + resolver: "k8s-git-mirror"; + allowHostGit: false; + allowHostWorkspace: false; + allowGithubDirectInPipeline: false; + }; + sourceSnapshot: { + stageRefPrefix: string; + missingObjectPolicy: "fail-fast"; + refreshPolicy: "sync-before-snapshot"; + }; + }; gitops: { branch: string; path: string }; gitMirror: { namespace: string; diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 2249cc01..8f656ae8 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -1898,6 +1898,8 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const ciNamespace = stringField(raw, "ciNamespace", path); const runtimeNamespace = stringField(raw, "runtimeNamespace", path); const argoNamespace = stringField(argo, "namespace", `${path}.argo`); + const sourceAuthority = controlPlaneSourceAuthoritySpec(asRecord(source.sourceAuthority, `${path}.source.sourceAuthority`), `${path}.source.sourceAuthority`); + const sourceSnapshot = controlPlaneSourceSnapshotSpec(asRecord(source.sourceSnapshot, `${path}.source.sourceSnapshot`), `${path}.source.sourceSnapshot`); return { id: stringField(raw, "id", path), node, @@ -1905,7 +1907,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa enabled: booleanField(raw, "enabled", path), ciNamespace, runtimeNamespace, - source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) }, + source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`), sourceAuthority, sourceSnapshot }, gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) }, gitMirror: { namespace: gitMirrorNamespace, @@ -1948,6 +1950,38 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa }; } +function controlPlaneSourceAuthoritySpec(raw: Record, path: string): ControlPlaneTargetSpec["source"]["sourceAuthority"] { + const mode = stringField(raw, "mode", path); + const resolver = stringField(raw, "resolver", path); + if (mode !== "gitMirrorSnapshot") throw new Error(`${path}.mode must be gitMirrorSnapshot`); + if (resolver !== "k8s-git-mirror") throw new Error(`${path}.resolver must be k8s-git-mirror`); + return { + mode, + resolver, + allowHostGit: falseBooleanField(raw, "allowHostGit", path), + allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path), + allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path), + }; +} + +function controlPlaneSourceSnapshotSpec(raw: Record, path: string): ControlPlaneTargetSpec["source"]["sourceSnapshot"] { + const stageRefPrefix = stringField(raw, "stageRefPrefix", path); + if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`); + if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`); + if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`); + const missingObjectPolicy = stringField(raw, "missingObjectPolicy", path); + const refreshPolicy = stringField(raw, "refreshPolicy", path); + if (missingObjectPolicy !== "fail-fast") throw new Error(`${path}.missingObjectPolicy must be fail-fast`); + if (refreshPolicy !== "sync-before-snapshot") throw new Error(`${path}.refreshPolicy must be sync-before-snapshot`); + return { stageRefPrefix, missingObjectPolicy, refreshPolicy }; +} + +function falseBooleanField(obj: Record, key: string, path: string): false { + const value = booleanField(obj, key, path); + if (value !== false) throw new Error(`${path}.${key} must be false`); + return false; +} + function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record[] { const labels = { "app.kubernetes.io/part-of": "hwlab-node-control-plane", @@ -1977,7 +2011,7 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa kind: "ConfigMap", metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, data: { - "repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), + "repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, sourceStageRefPrefix: target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch), gitopsBranch: target.gitops.branch }], null, 2), "server.js": gitMirrorServerJs(), "status.sh": gitMirrorStatusShell(), "sync.sh": gitMirrorSyncShell(_node, target), @@ -2280,7 +2314,7 @@ function service(name: string, namespace: string, labels: Record function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return sha256Short(JSON.stringify({ - repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], + repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, sourceStageRefPrefix: target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch), gitopsBranch: target.gitops.branch }], ports: { read: target.gitMirror.readContainerPort, write: target.gitMirror.writeContainerPort }, githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport), runtimeProxy: runtimeHostProxyConfig(node, gitMirrorRuntimeProxySpec(node, target)), @@ -2467,6 +2501,9 @@ for (const spec of repositories) { const repoPath = '/cache/' + spec.repository + '.git'; const localSource = rev(repoPath, 'refs/heads/' + spec.sourceBranch); const githubSource = rev(repoPath, 'refs/mirror-stage/heads/' + spec.sourceBranch); + const sourceStageRefPrefix = spec.sourceStageRefPrefix || ('refs/unidesk/snapshots/hwlab-node-runtime/' + spec.sourceBranch); + const sourceStageRef = githubSource ? sourceStageRefPrefix.replace(/\/+$/, '') + '/' + githubSource : null; + const sourceSnapshot = sourceStageRef ? rev(repoPath, sourceStageRef) : null; const localGitops = rev(repoPath, 'refs/heads/' + spec.gitopsBranch); const githubGitops = rev(repoPath, 'refs/mirror-stage/heads/' + spec.gitopsBranch); items[spec.key] = { @@ -2474,10 +2511,13 @@ for (const spec of repositories) { sourceBranch: spec.sourceBranch, localSource, githubSource, + sourceAuthority: 'git-mirror-snapshot', + sourceStageRef, + sourceSnapshot, gitopsBranch: spec.gitopsBranch, localGitops, githubGitops, - sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), + sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)), }; @@ -2487,11 +2527,15 @@ const pendingFlush = Object.values(items).some((item) => Boolean(item.pendingFlu console.log(JSON.stringify({ localSource: first.localSource || null, githubSource: first.githubSource || null, + sourceAuthority: first.sourceAuthority || null, + sourceStageRef: first.sourceStageRef || null, + sourceSnapshot: first.sourceSnapshot || null, localGitops: first.localGitops || null, githubGitops: first.githubGitops || null, refSources: { localSource: 'refs/heads/' + (first.sourceBranch || ''), githubSource: 'refs/mirror-stage/heads/' + (first.sourceBranch || ''), + sourceSnapshot: first.sourceStageRef || null, localGitops: 'refs/heads/' + (first.gitopsBranch || ''), githubGitops: 'refs/mirror-stage/heads/' + (first.gitopsBranch || ''), githubFieldsAreMirrorStageCache: true @@ -2557,6 +2601,7 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`, `repository=${shQuote(target.source.repository)}`, `source_branch=${shQuote(target.source.branch)}`, + `source_stage_ref_prefix=${shQuote(target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch))}`, `gitops_branch=${shQuote(target.gitops.branch)}`, "repo=\"/cache/${repository}.git\"", ].filter(Boolean); @@ -2678,6 +2723,8 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg "git --git-dir=\"$repo\" config http.receivepack true", "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", + "source_stage_ref=\"${source_stage_ref_prefix%/}/$source_sha\"", + "git --git-dir=\"$repo\" update-ref \"$source_stage_ref\" \"$source_sha\"", "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", "discarded_stale_gitops=false", "if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", @@ -2694,7 +2741,7 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg " fi", "fi", "git --git-dir=\"$repo\" update-server-info", - "export repository source_branch gitops_branch started_at discarded_stale_gitops", + "export repository source_branch source_stage_ref gitops_branch started_at discarded_stale_gitops", "node <<'NODE' | tee /cache/HWLAB.last-sync.json", "const { execFileSync } = require('node:child_process');", "const repository = process.env.repository;", @@ -2704,10 +2751,12 @@ function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTarg "function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }", "const localSource = rev(`refs/heads/${sourceBranch}`);", "const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);", + "const sourceStageRef = process.env.source_stage_ref;", + "const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;", "const localGitops = rev(`refs/heads/${gitopsBranch}`);", "const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);", "const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));", - "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));", + "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), sourceAuthority: 'git-mirror-snapshot', localSource, githubSource, sourceStageRef, sourceSnapshot, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));", "NODE", "cat /cache/HWLAB.last-sync.json", "", diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 14f076b0..c54acf10 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -323,6 +323,20 @@ export interface HwlabRuntimeImageBuildSpec { readonly dockerNetworkMode: "default" | "host"; } +export interface HwlabRuntimeSourceAuthoritySpec { + readonly mode: "gitMirrorSnapshot"; + readonly resolver: "k8s-git-mirror"; + readonly allowHostGit: false; + readonly allowHostWorkspace: false; + readonly allowGithubDirectInPipeline: false; +} + +export interface HwlabRuntimeSourceSnapshotSpec { + readonly stageRefPrefix: string; + readonly missingObjectPolicy: "fail-fast"; + readonly refreshPolicy: "sync-before-snapshot"; +} + export interface HwlabRuntimePublicExposureFrpcProxySpec { readonly name: string; readonly localIP: string; @@ -450,6 +464,8 @@ export interface HwlabRuntimeLaneSpec { readonly minor: number; readonly version: string; readonly sourceBranch: string; + readonly sourceAuthority?: HwlabRuntimeSourceAuthoritySpec; + readonly sourceSnapshot?: HwlabRuntimeSourceSnapshotSpec; readonly workspace: string; readonly cicdRepo: string; readonly cicdRepoLock: string; @@ -498,6 +514,16 @@ export function hwlabRuntimeActiveExternalPostgres(spec: HwlabRuntimeLaneSpec): return spec.runtimeStore?.postgres?.mode === "platform-service" ? spec.externalPostgres : undefined; } +export function hwlabRuntimeSourceSnapshotStageRefPrefix(spec: HwlabRuntimeLaneSpec): string { + if (spec.sourceSnapshot === undefined) throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH} lanes.${spec.lane}.sourceSnapshot is required for k8s git-mirror snapshot source authority`); + return spec.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", spec.sourceBranch); +} + +export function hwlabRuntimeSourceSnapshotRef(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { + if (!/^[0-9a-f]{40}$/iu.test(sourceCommit)) throw new Error(`sourceCommit must be a 40-hex git SHA for source snapshot ref: ${sourceCommit}`); + return `${hwlabRuntimeSourceSnapshotStageRefPrefix(spec).replace(/\/+$/u, "")}/${sourceCommit.toLowerCase()}`; +} + export const HWLAB_NODE_LANE_CONFIG_PATH = "config/hwlab-node-lanes.yaml"; interface HwlabLaneConfig { @@ -507,6 +533,8 @@ interface HwlabLaneConfig { readonly minor: number; readonly version: string; readonly sourceBranch: string; + readonly sourceAuthority?: HwlabRuntimeSourceAuthoritySpec; + readonly sourceSnapshot?: HwlabRuntimeSourceSnapshotSpec; readonly workspace: string; readonly cicdRepo: string; readonly cicdRepoLock: string; @@ -735,6 +763,8 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa const minor = numberField(raw, "minor", `lanes.${id}`); const version = stringField(raw, "version", `lanes.${id}`); if (version !== `v0.${minor}`) throw new Error(`lanes.${id}.version must equal v0.${minor}`); + if (minor >= 3 && raw.sourceAuthority === undefined) throw new Error(`lanes.${id}.sourceAuthority is required for HWLAB runtime v0.3+`); + if (minor >= 3 && raw.sourceSnapshot === undefined) throw new Error(`lanes.${id}.sourceSnapshot is required for HWLAB runtime v0.3+`); return { id, node: stringField(raw, "node", `lanes.${id}`), @@ -742,6 +772,8 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa minor, version, sourceBranch: stringField(raw, "sourceBranch", `lanes.${id}`), + sourceAuthority: sourceAuthorityConfig(raw.sourceAuthority, `lanes.${id}.sourceAuthority`), + sourceSnapshot: sourceSnapshotConfig(raw.sourceSnapshot, `lanes.${id}.sourceSnapshot`), workspace: stringField(raw, "workspace", `lanes.${id}`), cicdRepo: stringField(raw, "cicdRepo", `lanes.${id}`), cicdRepoLock: stringField(raw, "cicdRepoLock", `lanes.${id}`), @@ -805,6 +837,8 @@ function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record< bootstrapAdmin: mergeOptionalRecord(baseRaw.bootstrapAdmin, targetRaw.bootstrapAdmin), codeAgentProvider: mergeOptionalRecord(baseRaw.codeAgentProvider, targetRaw.codeAgentProvider), codeAgentRuntime: mergeOptionalRecord(baseRaw.codeAgentRuntime, targetRaw.codeAgentRuntime), + sourceAuthority: mergeOptionalRecord(baseRaw.sourceAuthority, targetRaw.sourceAuthority), + sourceSnapshot: mergeOptionalRecord(baseRaw.sourceSnapshot, targetRaw.sourceSnapshot), sourceWorkspace: mergeOptionalRecord(baseRaw.sourceWorkspace, targetRaw.sourceWorkspace), externalPostgres: mergeOptionalRecord(baseRaw.externalPostgres, targetRaw.externalPostgres), runtimeStore: mergeOptionalRecord(baseRaw.runtimeStore, targetRaw.runtimeStore), @@ -817,6 +851,38 @@ function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record< return laneConfig(id, merged); } +function sourceAuthorityConfig(value: unknown, path: string): HwlabRuntimeSourceAuthoritySpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + mode: enumStringField(raw, "mode", path, ["gitMirrorSnapshot"]), + resolver: enumStringField(raw, "resolver", path, ["k8s-git-mirror"]), + allowHostGit: falseBooleanField(raw, "allowHostGit", path), + allowHostWorkspace: falseBooleanField(raw, "allowHostWorkspace", path), + allowGithubDirectInPipeline: falseBooleanField(raw, "allowGithubDirectInPipeline", path), + }; +} + +function sourceSnapshotConfig(value: unknown, path: string): HwlabRuntimeSourceSnapshotSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + const stageRefPrefix = stringField(raw, "stageRefPrefix", path); + if (!stageRefPrefix.startsWith("refs/")) throw new Error(`${path}.stageRefPrefix must start with refs/`); + if (stageRefPrefix.includes("..") || /\s/u.test(stageRefPrefix)) throw new Error(`${path}.stageRefPrefix must not contain whitespace or ..`); + if (!stageRefPrefix.includes("{branch}")) throw new Error(`${path}.stageRefPrefix must include {branch}`); + return { + stageRefPrefix, + missingObjectPolicy: enumStringField(raw, "missingObjectPolicy", path, ["fail-fast"]), + refreshPolicy: enumStringField(raw, "refreshPolicy", path, ["sync-before-snapshot"]), + }; +} + +function falseBooleanField(obj: Record, key: string, path: string): false { + const value = booleanField(obj, key, path); + if (value !== false) throw new Error(`${path}.${key} must be false`); + return false; +} + function buildkitConfig(value: unknown, path: string): HwlabRuntimeBuildkitSpec | undefined { if (value === undefined) return undefined; const raw = asRecord(value, path); @@ -1682,6 +1748,8 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { minor: config.minor, version: config.version, sourceBranch: config.sourceBranch, + ...(config.sourceAuthority === undefined ? {} : { sourceAuthority: config.sourceAuthority }), + ...(config.sourceSnapshot === undefined ? {} : { sourceSnapshot: config.sourceSnapshot }), workspace: config.workspace, cicdRepo: config.cicdRepo, cicdRepoLock: config.cicdRepoLock, diff --git a/scripts/src/hwlab-node/cleanup.ts b/scripts/src/hwlab-node/cleanup.ts index f1ec0b1a..b6981251 100644 --- a/scripts/src/hwlab-node/cleanup.ts +++ b/scripts/src/hwlab-node/cleanup.ts @@ -13,7 +13,7 @@ import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; -import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, hwlabRuntimeSourceSnapshotStageRefPrefix, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; @@ -110,25 +110,28 @@ export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { source `read_deploy=${shellQuote(mirror.serviceReadName)}`, `repo_path=${shellQuote(`/cache/${mirror.sourceRepository}.git`)}`, `source_branch=${shellQuote(mirror.sourceBranch)}`, + `source_stage_ref_prefix=${shellQuote(hwlabRuntimeSourceSnapshotStageRefPrefix(spec))}`, "read_ref() {", " ref=\"$1\"", " kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'repo_path=$1; ref=$2; git --git-dir=\"$repo_path\" rev-parse --verify \"$ref^{commit}\" 2>/dev/null' sh \"$repo_path\" \"$ref\" 2>/dev/null", "}", "mirror_stage=$(read_ref \"refs/mirror-stage/heads/$source_branch\")", "mirror_stage_rc=$?", - "local_head=$(read_ref \"refs/heads/$source_branch\")", - "local_head_rc=$?", "source_commit=\"$mirror_stage\"", "source_ref=\"refs/mirror-stage/heads/$source_branch\"", - "if ! printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then", - " source_commit=\"$local_head\"", - " source_ref=\"refs/heads/$source_branch\"", + "source_stage_ref=''", + "snapshot_commit=''", + "if printf '%s' \"$source_commit\" | grep -Eq '^[0-9a-fA-F]{40}$'; then", + " source_stage_ref=\"${source_stage_ref_prefix%/}/$source_commit\"", + " snapshot_commit=$(read_ref \"$source_stage_ref\")", "fi", - "node - \"$mirror_stage_rc\" \"$local_head_rc\" \"$mirror_stage\" \"$local_head\" \"$source_commit\" \"$source_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'", - "const [mirrorStageRc, localHeadRc, mirrorStage, localHead, sourceCommit, sourceRef, repoPath, branch] = process.argv.slice(2);", + "node - \"$mirror_stage_rc\" \"$mirror_stage\" \"$snapshot_commit\" \"$source_commit\" \"$source_ref\" \"$source_stage_ref\" \"$repo_path\" \"$source_branch\" <<'NODE'", + "const [mirrorStageRc, mirrorStage, snapshotCommit, sourceCommit, sourceRef, sourceStageRef, repoPath, branch] = process.argv.slice(2);", "const isSha = (value) => /^[0-9a-f]{40}$/i.test(value || '');", - "const ok = isSha(sourceCommit);", - "console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-cache', sourceAuthority: 'git-mirror-cache', sourceCommit: ok ? sourceCommit.toLowerCase() : null, sourceRef: ok ? sourceRef : null, mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, localHead: isSha(localHead) ? localHead.toLowerCase() : null, mirrorStageRc: Number(mirrorStageRc), localHeadRc: Number(localHeadRc), branch, repoPath, valuesRedacted: true }));", + "const normalizedSource = isSha(sourceCommit) ? sourceCommit.toLowerCase() : null;", + "const normalizedSnapshot = isSha(snapshotCommit) ? snapshotCommit.toLowerCase() : null;", + "const ok = normalizedSource !== null && normalizedSnapshot === normalizedSource;", + "console.log(JSON.stringify({ ok, mode: 'k8s-git-mirror-snapshot', sourceAuthority: 'git-mirror-snapshot', sourceCommit: ok ? normalizedSnapshot : normalizedSource, sourceRef: normalizedSource ? sourceRef : null, sourceStageRef: sourceStageRef || null, snapshotPresent: ok, degradedReason: ok ? null : normalizedSource === null ? 'source-branch-missing' : 'source-snapshot-missing', mirrorStage: isSha(mirrorStage) ? mirrorStage.toLowerCase() : null, sourceSnapshot: normalizedSnapshot, mirrorStageRc: Number(mirrorStageRc), branch, repoPath, valuesRedacted: true }));", "NODE", ].join("\n"); const result = runNodeK3sScript(spec, script, 45); diff --git a/scripts/src/hwlab-node/git-mirror.ts b/scripts/src/hwlab-node/git-mirror.ts index 83c40370..98e9ec01 100644 --- a/scripts/src/hwlab-node/git-mirror.ts +++ b/scripts/src/hwlab-node/git-mirror.ts @@ -456,15 +456,16 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record, sco if (reason === "public-probe-not-ready") { return `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`; } + if (reason === "source-snapshot-missing") { + return `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`; + } if (reason === "git-mirror-pending-flush") { return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`; } + if (reason === "git-mirror-not-in-sync") { + return `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane} --full`; + } return `${nodeRuntimeStatusCommand(scoped)} --full`; } diff --git a/scripts/src/hwlab-node/runtime-common.ts b/scripts/src/hwlab-node/runtime-common.ts index 42db1de9..a503b99f 100644 --- a/scripts/src/hwlab-node/runtime-common.ts +++ b/scripts/src/hwlab-node/runtime-common.ts @@ -146,6 +146,7 @@ export function nodeRuntimeGitMirrorRefSources(scoped: ReturnType>/tmp/hwlab-node-gitmirror-status.err || true)", + "\" sh \"$source_repository\" \"$source_branch\" \"$source_stage_ref_prefix\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)", "fi", "if [ -z \"$summary_json\" ]; then summary_json='{}'; fi", "read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")", @@ -180,7 +185,8 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType Object.keys(summary).length === 0 ? "REFS\n-" : webObserveTable( - ["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"], + ["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"], [[ shortValue(summary.localSource), shortValue(summary.githubSource), + shortValue(summary.sourceSnapshot), shortValue(summary.localGitops), shortValue(summary.githubGitops), webObserveText(summary.pendingFlush), @@ -783,6 +790,10 @@ export function compactNodeRuntimeGitMirrorStatus(status: Record