diff --git a/.agents/skills/unidesk-cicd/references/branch-follower.md b/.agents/skills/unidesk-cicd/references/branch-follower.md index 2d4afbbf..fbc69c3a 100644 --- a/.agents/skills/unidesk-cicd/references/branch-follower.md +++ b/.agents/skills/unidesk-cicd/references/branch-follower.md @@ -44,6 +44,20 @@ Use configRef summaries in plan/status; do not create a `full.md` or super Markd - `agentrun-jd01-v02`: follows `pikasTech/agentrun@v0.2`, adapter `agentrun-yaml-lane`, native trigger `build image Job -> GitOps publish Job -> git-mirror flush Job -> Tekton PipelineRun -> Argo Application closeout -> runtime Deployment sourceCommit readiness`. The same source commit must use deterministic Job names so a later controller loop can resume or reuse already completed stages. - `web-probe-sentinel-master`: follows `pikasTech/unidesk@master`, adapter `web-probe-sentinel-cicd`, native trigger `Tekton PipelineRun -> Argo Application closeout -> runtime Deployment sourceCommit readiness`. +These three followers are the initial production set. HWLAB and AgentRun both run on JD01; there is no D601 target in the automatic follower set unless YAML is explicitly changed. + +## Reuse And Mirror Contract + +The controller must preserve the runtime reuse capabilities that already exist in the runtime lanes: + +- runtime reuse: if both code identity and env identity are unchanged for a microservice, skip rebuild and rollout for that service; +- env reuse: if code changed but env identity is unchanged, reuse the previous environment image and publish only the changed service artifact; +- git mirror: source sync, immutable source snapshot creation and GitOps flush are generic branch-follower stages, not adapter-local afterthoughts. + +Adapters should expose reuse evidence through compact native state. HWLAB uses the `plan-artifacts` task event summary (`affectedServices`, `buildServices`, `reusedServices`, `artifactProvenanceAudit`). AgentRun publishes deterministic image/GitOps/git-mirror stage names and source-commit labels so a later loop can resume closeout without rebuilding completed stages. Sentinel keeps the same source/CI/Argo/runtime contract but has no GitOps branch flush gate. + +The normal convergence budget is 120 seconds per source change. A follower may report `ClosingOut` while waiting for Argo/runtime readiness, but it must not report `Noop` when the source sha matches and required native gates such as git-mirror flush are still incomplete. + ## Status Contract Default `status` output must show follower id, phase, adapter, source branch + observed sha, target sha, last triggered sha, last succeeded sha, in-flight job/PipelineRun, budget source and next drill-down commands. @@ -54,6 +68,8 @@ Status and decision inputs are Kubernetes-native: - source: k8s git-mirror cache ref and immutable snapshot ref; - CI: Tekton `PipelineRun.status.conditions`; +- CI drill-down: compact TaskRun timings and plan-artifact reuse summary when available; +- git mirror: source snapshot readiness plus GitOps `pendingFlush`/`githubInSync` when the follower owns a GitOps branch; - deployment: Argo `Application.status.sync` and `Application.status.health`; - runtime: selected Deployment/StatefulSet readiness plus source commit labels, annotations or env. diff --git a/.agents/skills/unidesk-cicd/references/git-mirror.md b/.agents/skills/unidesk-cicd/references/git-mirror.md index ea83e045..4b19f1cd 100644 --- a/.agents/skills/unidesk-cicd/references/git-mirror.md +++ b/.agents/skills/unidesk-cicd/references/git-mirror.md @@ -19,6 +19,8 @@ bun scripts/cli.ts hwlab nodes git-mirror flush --node --lane v03 --confi - `sync`: 把当前配置声明的 GitHub refs 拉入本地 mirror，并为 source branch tip 创建不可变 snapshot ref。 - `flush`: 把本地 lane GitOps ref 快进推回 GitHub。 +`cicd branch-follower` 已把 `sync` 和 `flush` 作为通用 Kubernetes-native stage 接入 HWLAB/AgentRun 自动跟随：trigger 前先 sync 并创建 immutable source snapshot，GitOps publish 后必须 flush，status/closeout 以 mirror cache ref 和 `pendingFlush=false`/`githubInSync=true` 为准。正常自动跟随不需要 operator 手动清理旧状态；旧 ConfigMap 或历史 PipelineRun 干扰时只用 `cicd branch-follower cleanup-state --follower --confirm` 做显式 cleanup。 + PipelineRun `gitops-promote` 如果报 git mirror 控制面漂移、refs 不一致或 flush/publish 未完成，优先按当前 `devops-infra/git-mirror.yaml` 收敛：先 `git-mirror apply --confirm`，再 `git-mirror sync --confirm --wait`，然后用 `control-plane cleanup-runs --pipeline-run --confirm` 受控清理失败 PipelineRun 后重试。旧 branch/path allowlist gate 已删除，不要恢复旧 hook、直接 `kubectl delete`、手工 patch pod 内 hook 或绕过 `flush`。 手动 trigger closeout 不能只看 PipelineRun `Completed`。必须继续查 `control-plane status --pipeline-run ` 和 `git-mirror status`；node-scoped `trigger-current --confirm --wait` 会自动做必要的 mirror pre/post flush，但 closeout 仍要确认最终 `pendingFlush=false`、`githubInSync=true`。如果 lower-level 手工路径或旧 job 留下 `pendingFlush=true`，执行 `git-mirror flush --confirm --wait` 到 `pendingFlush=false`。 diff --git a/.agents/skills/unidesk-gh/references/pull-requests.md b/.agents/skills/unidesk-gh/references/pull-requests.md index 77f8a21f..53681766 100644 --- a/.agents/skills/unidesk-gh/references/pull-requests.md +++ b/.agents/skills/unidesk-gh/references/pull-requests.md @@ -3,8 +3,10 @@ PR work uses guarded UniDesk GitHub commands: - `pr review-plan`, `pr diff --file`, and bounded file drill-down before review. -- `pr preflight` before merge when required by the issue or branch policy. -- `pr merge --merge --delete-branch` by default. +- `pr preflight` is optional read-only diagnosis; `pr merge` runs preflight internally. +- `pr merge --merge` by default deletes the merged same-repo head branch, cleans a matching clean local `.worktree`, and fast-forwards the local main worktree on the PR base branch. +- `pr merge --merge --sync-node JD01` additionally runs mapped node source-workspace sync when supported, currently HWLAB `v0.3`. +- Use `--keep-branch` or `--skip-local-closeout` only when intentionally preserving post-merge state. - Use squash only when ancestry and semantic absorption are explicitly safe. Closeout should mention source branch, validation evidence and any residual risk. diff --git a/AGENTS.md b/AGENTS.md index c80a78c9..8ee4b8d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,11 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台。本文件只做自动加载的顶级索引；长期规则、详细流程和判定标准必须放到 skill 或 `docs/reference/*.md`。 +## P0: 文件体积与脚本分流 + +- P0: 任何源码/CLI 文件超过 3000 行必须先按职责拆分再继续，禁止继续追加绕过。 +- P0: 禁止把 shell/Node/Python 等脚本作为大段字符串内嵌；脚本必须放入原生后缀文件（如 `.sh`/`.mjs`/`.py`）并从文件加载。 + ## P0: 主 worktree 同步提交第一原则 - P0: 发现固定主/目标 worktree 落后 remote 时，必须立刻先 `git stash push -u` 保存脏改（如有，含 untracked），再 `git pull --ff-only` 快进到最新 remote，然后 `git stash apply` 并按语义合并；主工作区恢复出的并行改动必须先直接提交，再继续后续任务；禁止用 reset、drop 或覆盖式 checkout 丢弃并行改动。 diff --git a/scripts/native/cicd/compact-git-mirror.mjs b/scripts/native/cicd/compact-git-mirror.mjs new file mode 100644 index 00000000..c6ce6689 --- /dev/null +++ b/scripts/native/cicd/compact-git-mirror.mjs @@ -0,0 +1,47 @@ +import { execFileSync } from "node:child_process"; + +const repoPath = process.env.REPO_PATH || ""; +const repository = process.env.REPOSITORY || ""; +const sourceBranch = process.env.SOURCE_BRANCH || ""; +const snapshotPrefix = (process.env.SNAPSHOT_PREFIX || "").replace(/\/+$/u, ""); +const gitopsBranch = process.env.GITOPS_BRANCH || ""; + +function rev(ref) { + if (!ref) return null; + try { + const out = execFileSync("git", [`--git-dir=${repoPath}`, "rev-parse", "--verify", `${ref}^{commit}`], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); + return /^[0-9a-f]{40}$/iu.test(out) ? out : null; + } catch { + return null; + } +} + +const localSource = rev(`refs/heads/${sourceBranch}`); +const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`); +const snapshotSource = githubSource || localSource; +const sourceStageRef = snapshotSource ? `${snapshotPrefix}/${snapshotSource}` : null; +const sourceSnapshot = sourceStageRef === null ? null : rev(sourceStageRef); +const localGitops = gitopsBranch ? rev(`refs/heads/${gitopsBranch}`) : null; +const githubGitops = gitopsBranch ? rev(`refs/mirror-stage/heads/${gitopsBranch}`) : null; +const pendingFlush = gitopsBranch ? Boolean(localGitops && localGitops !== githubGitops) : null; +const githubInSync = gitopsBranch ? Boolean(!localGitops || localGitops === githubGitops) : null; +const sourceSnapshotReady = snapshotSource ? sourceSnapshot === snapshotSource : false; + +process.stdout.write(JSON.stringify({ + ok: Boolean(localSource) && sourceSnapshotReady && pendingFlush !== true, + repository, + repoPath, + sourceBranch, + gitopsBranch: gitopsBranch || null, + localSource, + githubSource, + sourceStageRef, + sourceSnapshot, + sourceSnapshotReady, + localGitops, + githubGitops, + pendingFlush, + githubInSync, + statusAuthority: "k8s-git-mirror-cache", + valuesRedacted: true, +})); diff --git a/scripts/native/cicd/compact-native-object.mjs b/scripts/native/cicd/compact-native-object.mjs new file mode 100644 index 00000000..88aea783 --- /dev/null +++ b/scripts/native/cicd/compact-native-object.mjs @@ -0,0 +1,129 @@ +import { readFileSync } from "node:fs"; + +const key = process.argv[2] || ""; +const input = JSON.parse(readFileSync(0, "utf8")); + +function cleanMap(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + const out = {}; + for (const [k, v] of Object.entries(value)) { + if (k === "kubectl.kubernetes.io/last-applied-configuration") continue; + out[k] = v; + } + return out; +} + +function metadata(obj) { + return { + name: obj?.metadata?.name || null, + namespace: obj?.metadata?.namespace || null, + labels: cleanMap(obj?.metadata?.labels), + annotations: cleanMap(obj?.metadata?.annotations), + }; +} + +function compactContainer(container) { + return { + name: container?.name || null, + image: container?.image || null, + env: Array.isArray(container?.env) + ? container.env.filter((item) => item && typeof item.name === "string" && typeof item.value === "string").map((item) => ({ name: item.name, value: item.value })) + : [], + }; +} + +function condition(obj, type) { + const conditions = Array.isArray(obj?.status?.conditions) ? obj.status.conditions : []; + return conditions.find((item) => item?.type === type) || conditions[0] || null; +} + +function timestampMs(value) { + const parsed = Date.parse(String(value || "")); + return Number.isFinite(parsed) ? parsed : null; +} + +function durationSeconds(start, end) { + const s = timestampMs(start); + const e = timestampMs(end); + return s === null || e === null || e < s ? null : Math.round((e - s) / 1000); +} + +let output = input; +if (key === "pipelineRun") { + const succeeded = condition(input, "Succeeded"); + output = { + apiVersion: input.apiVersion, + kind: input.kind, + metadata: metadata(input), + spec: { params: Array.isArray(input?.spec?.params) ? input.spec.params : [] }, + status: { + conditions: Array.isArray(input?.status?.conditions) ? input.status.conditions : [], + startTime: input?.status?.startTime || null, + completionTime: input?.status?.completionTime || null, + durationSeconds: durationSeconds(input?.status?.startTime, input?.status?.completionTime), + succeeded: succeeded?.status || null, + reason: succeeded?.reason || null, + }, + }; +} else if (key === "taskRuns") { + const items = (Array.isArray(input?.items) ? input.items : []).map((item) => { + const succeeded = condition(item, "Succeeded"); + return { + name: item?.metadata?.name || null, + namespace: item?.metadata?.namespace || null, + pipelineTask: item?.metadata?.labels?.["tekton.dev/pipelineTask"] || item?.metadata?.labels?.["tekton.dev/task"] || null, + status: succeeded?.status || null, + reason: succeeded?.reason || null, + startTime: item?.status?.startTime || null, + completionTime: item?.status?.completionTime || null, + durationSeconds: durationSeconds(item?.status?.startTime, item?.status?.completionTime), + }; + }).sort((left, right) => String(left.startTime || "").localeCompare(String(right.startTime || ""))); + const slow = items.filter((item) => typeof item.durationSeconds === "number" && item.durationSeconds > 60); + output = { + ok: true, + count: items.length, + succeededCount: items.filter((item) => item.status === "True").length, + failedCount: items.filter((item) => item.status === "False").length, + activeCount: items.filter((item) => item.status !== "True" && item.status !== "False").length, + items, + performance: { slowCount: slow.length, slowTaskRuns: slow.slice(0, 8), warning: slow.length > 0 ? "taskrun-over-60s" : null }, + statusAuthority: "kubernetes-api-serviceaccount", + }; +} else if (key === "argoApplication") { + output = { + apiVersion: input.apiVersion, + kind: input.kind, + metadata: metadata(input), + status: { + sync: input?.status?.sync || null, + health: input?.status?.health || null, + operationState: input?.status?.operationState + ? { phase: input.status.operationState.phase || null, message: input.status.operationState.message || null, finishedAt: input.status.operationState.finishedAt || null } + : null, + }, + }; +} else if (/^workload\d+$/.test(key)) { + const template = input?.spec?.template || {}; + output = { + apiVersion: input.apiVersion, + kind: input.kind, + metadata: metadata(input), + spec: { + replicas: input?.spec?.replicas ?? null, + template: { + metadata: { labels: cleanMap(template?.metadata?.labels), annotations: cleanMap(template?.metadata?.annotations) }, + spec: { containers: Array.isArray(template?.spec?.containers) ? template.spec.containers.map(compactContainer) : [] }, + }, + }, + status: { + replicas: input?.status?.replicas ?? null, + readyReplicas: input?.status?.readyReplicas ?? null, + availableReplicas: input?.status?.availableReplicas ?? null, + updatedReplicas: input?.status?.updatedReplicas ?? null, + conditions: Array.isArray(input?.status?.conditions) ? input.status.conditions.map((item) => ({ type: item.type || null, status: item.status || null, reason: item.reason || null })) : [], + }, + }; +} + +console.log(JSON.stringify(output)); diff --git a/scripts/native/cicd/controller-loop.sh b/scripts/native/cicd/controller-loop.sh new file mode 100644 index 00000000..e56f5515 --- /dev/null +++ b/scripts/native/cicd/controller-loop.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +interval="${UNIDESK_CICD_BRANCH_FOLLOWER_INTERVAL_SECONDS}" +timeout="${UNIDESK_CICD_BRANCH_FOLLOWER_TIMEOUT_SECONDS}" + +while true; do + started_at=$(date -Iseconds) + echo "branch-follower loop started ${started_at}" + cd /work + rm -rf /work/unidesk + /etc/unidesk-cicd-branch-follower/sync-source.sh \ + "${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}" \ + "${UNIDESK_CONTROLLER_SOURCE_BRANCH}" \ + "${UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX}" \ + "/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git" + git clone --branch "${UNIDESK_CONTROLLER_SOURCE_BRANCH}" "/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git" /work/unidesk + cp /etc/unidesk-cicd-branch-follower/cicd-branch-followers.yaml /work/unidesk/config/cicd-branch-followers.yaml + cd /work/unidesk + bun scripts/cli.ts cicd branch-follower run-once --all --confirm --controller --config config/cicd-branch-followers.yaml --timeout-seconds "${timeout}" || true + echo "branch-follower loop finished $(date -Iseconds)" + cd /work + sleep "${interval}" +done diff --git a/scripts/native/cicd/controller-one-shot.sh b/scripts/native/cicd/controller-one-shot.sh new file mode 100644 index 00000000..c2b3d7e6 --- /dev/null +++ b/scripts/native/cicd/controller-one-shot.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu + +cd /work +rm -rf /work/unidesk +started_at=$(date -Iseconds) +echo "branch-follower one-shot started ${started_at}" + +/etc/unidesk-cicd-branch-follower/sync-source.sh \ + "${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}" \ + "${UNIDESK_CONTROLLER_SOURCE_BRANCH}" \ + "${UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX}" \ + "/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git" + +git clone --branch "${UNIDESK_CONTROLLER_SOURCE_BRANCH}" "/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git" /work/unidesk +cp /etc/unidesk-cicd-branch-follower/cicd-branch-followers.yaml /work/unidesk/config/cicd-branch-followers.yaml +cd /work/unidesk + +"$@" + +echo "branch-follower one-shot finished $(date -Iseconds)" diff --git a/scripts/native/cicd/git-ssh-proxy.sh b/scripts/native/cicd/git-ssh-proxy.sh new file mode 100644 index 00000000..4751f135 --- /dev/null +++ b/scripts/native/cicd/git-ssh-proxy.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +exec ssh \ + -i /root/.ssh/id_rsa \ + -o IdentitiesOnly=yes \ + -o BatchMode=yes \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile=/root/.ssh/known_hosts \ + -o ConnectTimeout=15 \ + -o ServerAliveInterval=5 \ + -o ServerAliveCountMax=1 \ + -o "ProxyCommand=node /etc/unidesk-cicd-branch-follower/github-proxy-connect.mjs ${UNIDESK_CONTROLLER_GITHUB_PROXY_HOST} ${UNIDESK_CONTROLLER_GITHUB_PROXY_PORT} %h %p" \ + "$@" diff --git a/scripts/native/cicd/github-proxy-connect.mjs b/scripts/native/cicd/github-proxy-connect.mjs new file mode 100644 index 00000000..b7f87649 --- /dev/null +++ b/scripts/native/cicd/github-proxy-connect.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +import net from "node:net"; + +const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2); +const proxyPort = Number.parseInt(proxyPortRaw || "", 10); +const targetPort = Number.parseInt(targetPortRaw || "", 10); +if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64); + +let settled = false; +let tunnel = false; +function finish(code) { + if (settled) return; + settled = true; + process.exit(code); +} + +const socket = net.createConnection({ host: proxyHost, port: proxyPort }); +let buffer = Buffer.alloc(0); +socket.setTimeout(15000, () => { + socket.destroy(); + finish(65); +}); +socket.on("connect", () => socket.write(`CONNECT ${targetHost}:${targetPort} HTTP/1.1\r\nHost: ${targetHost}:${targetPort}\r\nProxy-Connection: Keep-Alive\r\n\r\n`)); +socket.on("error", () => finish(tunnel ? 69 : 66)); +socket.on("close", () => finish(tunnel ? 0 : 68)); +socket.on("data", function onData(chunk) { + buffer = Buffer.concat([buffer, chunk]); + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd === -1 && buffer.length < 8192) return; + if (headerEnd === -1) { + socket.destroy(); + finish(68); + return; + } + const statusLine = buffer.slice(0, headerEnd).toString("latin1").split("\r\n", 1)[0] || ""; + const statusCode = Number.parseInt(statusLine.split(" ")[1] || "", 10); + if (!statusLine.startsWith("HTTP/1.") || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { + socket.destroy(); + finish(67); + return; + } + socket.off("data", onData); + socket.setTimeout(0); + tunnel = true; + const rest = buffer.slice(headerEnd + 4); + if (rest.length) process.stdout.write(rest); + process.stdin.on("error", () => {}); + process.stdout.on("error", () => {}); + process.stdin.pipe(socket); + socket.pipe(process.stdout); +}); diff --git a/scripts/native/cicd/kube-get.mjs b/scripts/native/cicd/kube-get.mjs new file mode 100644 index 00000000..bc0b9ac5 --- /dev/null +++ b/scripts/native/cicd/kube-get.mjs @@ -0,0 +1,27 @@ +import { readFileSync } from "node:fs"; +import https from "node:https"; + +const path = process.argv[2]; +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); +const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); +const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +const req = https.request({ host, port, path, method: "GET", ca, headers: { authorization: `Bearer ${token}` } }, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => { + if ((res.statusCode || 0) >= 200 && (res.statusCode || 0) < 300) { + process.stdout.write(body); + process.exit(0); + } + process.stderr.write(body || `kube api status ${res.statusCode}`); + process.exit(1); + }); +}); +req.on("error", (error) => { + process.stderr.write(error?.message || String(error)); + process.exit(1); +}); +req.end(); diff --git a/scripts/native/cicd/native-job.mjs b/scripts/native/cicd/native-job.mjs new file mode 100644 index 00000000..e050d6be --- /dev/null +++ b/scripts/native/cicd/native-job.mjs @@ -0,0 +1,127 @@ +import { readFileSync } from "node:fs"; +import https from "node:https"; + +const namespace = process.env.NAMESPACE || ""; +const jobName = process.env.JOB_NAME || ""; +const logContainer = process.env.LOG_CONTAINER || ""; +const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60"); +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); +const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); +const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); +const manifest = JSON.parse(Buffer.from(readFileSync(0, "utf8").replace(/\s+/g, ""), "base64").toString("utf8")); + +function request(method, path, body, contentType = "application/json") { + return new Promise((resolve, reject) => { + const headers = { authorization: `Bearer ${token}` }; + const payload = body === undefined ? null : typeof body === "string" ? body : JSON.stringify(body); + if (payload !== null) { + headers["content-type"] = contentType; + headers["content-length"] = Buffer.byteLength(payload); + } + const req = https.request({ host, port, path, method, ca, headers }, (res) => { + let text = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { text += chunk; }); + res.on("end", () => resolve({ status: res.statusCode || 0, text })); + }); + req.on("error", reject); + if (payload !== null) req.write(payload); + req.end(); + }); +} + +function parse(text) { + try { + return text ? JSON.parse(text) : null; + } catch { + return null; + } +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function condition(job, type) { + return (Array.isArray(job?.status?.conditions) ? job.status.conditions : []).find((item) => item?.type === type && item?.status === "True") || null; +} + +async function getJob() { + const result = await request("GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs/${encodeURIComponent(jobName)}`); + if (result.status === 404) return null; + if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api GET job status ${result.status}`); + return parse(result.text); +} + +async function podNames() { + const selector = encodeURIComponent(`job-name=${jobName}`); + const result = await request("GET", `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${selector}`); + if (result.status < 200 || result.status >= 300) return []; + const list = parse(result.text); + return (Array.isArray(list?.items) ? list.items : []).map((pod) => pod?.metadata?.name).filter(Boolean); +} + +async function logsTail() { + const names = await podNames(); + let combined = ""; + for (const pod of names.slice(-2)) { + const container = logContainer ? `&container=${encodeURIComponent(logContainer)}` : ""; + const result = await request("GET", `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods/${encodeURIComponent(pod)}/log?tailLines=120${container}`); + if (result.status >= 200 && result.status < 300) combined += `${result.text}\n`; + } + return combined.length > 6000 ? combined.slice(-6000) : combined; +} + +let created = false; +let reused = false; +const existing = await getJob(); +if (existing) { + reused = true; +} else { + const result = await request("POST", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs`, manifest); + if (result.status === 409) reused = true; + else if (result.status >= 200 && result.status < 300) created = true; + else { + process.stderr.write(result.text || `kube api POST job status ${result.status}`); + process.exit(1); + } +} + +const startedAt = Date.now(); +const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000; +let polls = 0; +let latest = await getJob(); +while (Date.now() <= deadline) { + const complete = condition(latest, "Complete"); + const failed = condition(latest, "Failed"); + if (complete || failed) break; + polls += 1; + await delay(2000); + latest = await getJob(); +} + +const complete = condition(latest, "Complete"); +const failed = condition(latest, "Failed"); +const logs = await logsTail(); +const timedOut = !complete && !failed; +const output = { + ok: Boolean(complete) && !timedOut, + completed: Boolean(complete), + failed: Boolean(failed), + timedOut, + created, + reused, + jobName, + namespace, + polls, + elapsedMs: Date.now() - startedAt, + conditionReason: complete?.reason || failed?.reason || null, + conditionMessage: complete?.message || failed?.message || null, + logsTail: logs || null, + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + valuesRedacted: true, +}; +process.stdout.write(JSON.stringify(output)); +if (!output.ok) process.exit(1); diff --git a/scripts/native/cicd/plan-artifacts.mjs b/scripts/native/cicd/plan-artifacts.mjs new file mode 100644 index 00000000..dbc99fff --- /dev/null +++ b/scripts/native/cicd/plan-artifacts.mjs @@ -0,0 +1,96 @@ +import { readFileSync } from "node:fs"; +import https from "node:https"; + +const namespace = process.argv[2] || ""; +const pipelineRun = process.argv[3] || ""; +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); +const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); +const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +function request(path) { + return new Promise((resolve) => { + const req = https.request({ host, port, path, method: "GET", ca, headers: { authorization: `Bearer ${token}` } }, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => resolve({ status: res.statusCode || 0, body })); + }); + req.on("error", (error) => resolve({ status: 599, body: error?.message || String(error) })); + req.end(); + }); +} + +function parse(text) { + try { + return text ? JSON.parse(text) : null; + } catch { + return null; + } +} + +function strings(value) { + return Array.isArray(value) ? value.filter((item) => typeof item === "string") : []; +} + +const out = { + ok: false, + pipelineRun, + pods: [], + eventFound: false, + degradedReason: "plan-artifacts-log-query-skipped", + statusAuthority: "kubernetes-api-serviceaccount", +}; + +if (!namespace || !pipelineRun) { + process.stdout.write(JSON.stringify(out)); + process.exit(0); +} + +const selector = encodeURIComponent(`tekton.dev/pipelineRun=${pipelineRun}`); +const podsResult = await request(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${selector}`); +const pods = parse(podsResult.body); +const items = Array.isArray(pods?.items) ? pods.items : []; +const planPods = items.filter((pod) => { + const labels = pod?.metadata?.labels || {}; + const name = pod?.metadata?.name || ""; + return labels["tekton.dev/pipelineTask"] === "plan-artifacts" || labels["tekton.dev/task"] === "plan-artifacts" || /plan-artifacts/u.test(name); +}); +out.pods = planPods.map((pod) => pod?.metadata?.name).filter(Boolean); + +const events = []; +for (const podName of out.pods.slice(-4)) { + const log = await request(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods/${encodeURIComponent(podName)}/log?tailLines=240`); + if (log.status < 200 || log.status >= 300) continue; + for (const raw of log.body.split(/\r?\n/u)) { + const line = raw.trim(); + if (!line.startsWith("{")) continue; + const event = parse(line); + if (event?.event === "g14-ci-plan") events.push(event); + } +} + +const latest = events.at(-1) || null; +if (latest) { + const audit = latest.artifactProvenanceAudit && typeof latest.artifactProvenanceAudit === "object" ? latest.artifactProvenanceAudit : null; + const unsafeReuseServices = strings(audit?.unsafeReuseServices); + const provenanceRebuildServices = strings(audit?.provenanceRebuildServices); + Object.assign(out, { + ok: true, + eventFound: true, + degradedReason: null, + sourceCommitId: typeof latest.sourceCommitId === "string" ? latest.sourceCommitId : null, + affectedServices: strings(latest.affectedServices), + rolloutServices: strings(latest.rolloutServices), + buildServices: strings(latest.buildServices), + reusedServices: strings(latest.reusedServices), + buildSkippedCount: typeof latest.buildSkippedCount === "number" ? latest.buildSkippedCount : null, + artifactProvenanceAudit: audit, + summary: `build=${strings(latest.buildServices).length} reuse=${strings(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`, + disclosure: "parsed from plan-artifacts g14-ci-plan log event via Kubernetes pod logs", + }); +} else { + out.degradedReason = podsResult.status >= 200 && podsResult.status < 300 ? "g14-ci-plan-event-not-found" : "plan-artifacts-pod-list-failed"; +} + +process.stdout.write(JSON.stringify(out)); diff --git a/scripts/native/cicd/read-native-bundle.sh b/scripts/native/cicd/read-native-bundle.sh new file mode 100644 index 00000000..845f2070 --- /dev/null +++ b/scripts/native/cicd/read-native-bundle.sh @@ -0,0 +1,114 @@ +#!/bin/sh +set +e + +tmpdir=$(mktemp -d) +cleanup() { rm -rf "${tmpdir}"; } +trap cleanup EXIT INT TERM + +script_dir="${NATIVE_CICD_SCRIPT_DIR}" +repo_path="${REPO_PATH}" +branch="${SOURCE_BRANCH}" +repository="${REPOSITORY}" +snapshot_prefix="${SNAPSHOT_PREFIX}" +gitops_branch="${GITOPS_BRANCH:-}" +tekton_namespace="${TEKTON_NAMESPACE:-}" +pipeline_run_prefix="${PIPELINE_RUN_PREFIX:-}" +argo_namespace="${ARGO_NAMESPACE:-}" +argo_application="${ARGO_APPLICATION:-}" + +emit_file_b64() { + key="$1" + path="$2" + printf 'UNIDESK_NATIVE_JSON\t%s\t' "${key}" + base64 "${path}" | tr -d '\n' + printf '\n' +} + +emit_error_b64() { + key="$1" + path="$2" + printf 'UNIDESK_NATIVE_ERROR\t%s\t' "${key}" + base64 "${path}" | tr -d '\n' + printf '\n' +} + +emit_kube_json() { + key="$1" + path="$2" + raw="${tmpdir}/${key}.raw" + out="${tmpdir}/${key}.out" + err="${tmpdir}/${key}.err" + if node "${script_dir}/kube-get.mjs" "${path}" >"${raw}" 2>"${err}" && node "${script_dir}/compact-native-object.mjs" "${key}" <"${raw}" >"${out}" 2>>"${err}"; then + emit_file_b64 "${key}" "${out}" + else + emit_error_b64 "${key}" "${err}" + fi +} + +emit_plan_artifacts() { + namespace="$1" + pipeline_run="$2" + out="${tmpdir}/planArtifacts.out" + err="${tmpdir}/planArtifacts.err" + if node "${script_dir}/plan-artifacts.mjs" "${namespace}" "${pipeline_run}" >"${out}" 2>"${err}"; then + emit_file_b64 planArtifacts "${out}" + else + emit_error_b64 planArtifacts "${err}" + fi +} + +source_commit= +source_err="${tmpdir}/source.err" +if [ -x /etc/unidesk-cicd-branch-follower/sync-source.sh ]; then + /etc/unidesk-cicd-branch-follower/sync-source.sh "${repository}" "${branch}" "${snapshot_prefix}" "${repo_path}" >/dev/null 2>"${source_err}" || true +fi + +if [ -d "${repo_path}/objects" ]; then + source_commit=$(git --git-dir="${repo_path}" rev-parse --verify "refs/heads/${branch}^{commit}" 2>>"${source_err}" | head -n 1 | tr -d '\r' || true) +else + printf 'formal controller/job must mount k8s git-mirror cache at %s; fallback exec is disabled\n' "${repo_path}" >>"${source_err}" +fi + +case "${source_commit}" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) + stage_ref="${snapshot_prefix%/}/${source_commit}" + source_out="${tmpdir}/source.out" + printf '{"commit":"%s","branch":"%s","stageRef":"%s","sourceAuthority":"k8s-git-mirror-snapshot","mode":"k8s-git-mirror-cache","repoPath":"%s"}' "${source_commit}" "${branch}" "${stage_ref}" "${repo_path}" >"${source_out}" + emit_file_b64 source "${source_out}" + ;; + *) + emit_error_b64 source "${source_err}" + ;; +esac + +if [ -d "${repo_path}/objects" ]; then + git_mirror_out="${tmpdir}/gitMirror.out" + git_mirror_err="${tmpdir}/gitMirror.err" + if REPO_PATH="${repo_path}" REPOSITORY="${repository}" SOURCE_BRANCH="${branch}" SNAPSHOT_PREFIX="${snapshot_prefix}" GITOPS_BRANCH="${gitops_branch}" node "${script_dir}/compact-git-mirror.mjs" >"${git_mirror_out}" 2>"${git_mirror_err}"; then + emit_file_b64 gitMirror "${git_mirror_out}" + else + emit_error_b64 gitMirror "${git_mirror_err}" + fi +fi + +if [ -n "${source_commit}" ] && [ -n "${tekton_namespace}" ] && [ -n "${pipeline_run_prefix}" ]; then + sha12=$(printf '%s' "${source_commit}" | cut -c1-12) + pipeline_run="${pipeline_run_prefix}-${sha12}" + emit_kube_json pipelineRun "/apis/tekton.dev/v1/namespaces/${tekton_namespace}/pipelineruns/${pipeline_run}" + emit_kube_json taskRuns "/apis/tekton.dev/v1/namespaces/${tekton_namespace}/taskruns?labelSelector=tekton.dev%2FpipelineRun%3D${pipeline_run}" + emit_plan_artifacts "${tekton_namespace}" "${pipeline_run}" +fi + +if [ -n "${argo_namespace}" ] && [ -n "${argo_application}" ]; then + emit_kube_json argoApplication "/apis/argoproj.io/v1alpha1/namespaces/${argo_namespace}/applications/${argo_application}" +fi + +if [ -n "${WORKLOAD_REFS_B64:-}" ]; then + printf '%s' "${WORKLOAD_REFS_B64}" | base64 -d | while IFS="$(printf '\t')" read -r key path; do + [ -n "${key}" ] || continue + [ -n "${path}" ] || continue + emit_kube_json "${key}" "${path}" + done +fi + +exit 0 diff --git a/scripts/native/cicd/submit-pipelinerun.mjs b/scripts/native/cicd/submit-pipelinerun.mjs new file mode 100644 index 00000000..00a9d552 --- /dev/null +++ b/scripts/native/cicd/submit-pipelinerun.mjs @@ -0,0 +1,125 @@ +import { readFileSync } from "node:fs"; +import https from "node:https"; + +const namespace = process.env.NAMESPACE || ""; +const pipelineRun = process.env.PIPELINERUN || ""; +const shouldWait = process.env.WAIT === "true"; +const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60"); +const host = process.env.KUBERNETES_SERVICE_HOST; +const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); +const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); +const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); +const manifest = JSON.parse(Buffer.from(readFileSync(0, "utf8").replace(/\s+/g, ""), "base64").toString("utf8")); + +function request(method, path, body, contentType = "application/json") { + return new Promise((resolve, reject) => { + const headers = { authorization: `Bearer ${token}` }; + const payload = body === undefined ? null : typeof body === "string" ? body : JSON.stringify(body); + if (payload !== null) { + headers["content-type"] = contentType; + headers["content-length"] = Buffer.byteLength(payload); + } + const req = https.request({ host, port, path, method, ca, headers }, (res) => { + let text = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { text += chunk; }); + res.on("end", () => resolve({ status: res.statusCode || 0, text })); + }); + req.on("error", reject); + if (payload !== null) req.write(payload); + req.end(); + }); +} + +function parseBody(result) { + if (!result.text) return null; + try { + return JSON.parse(result.text); + } catch { + return null; + } +} + +async function getPipelineRun() { + const result = await request("GET", `/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/pipelineruns/${encodeURIComponent(pipelineRun)}`); + if (result.status === 404) return { found: false, object: null, status: result.status, text: result.text }; + if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api GET pipelinerun status ${result.status}`); + return { found: true, object: parseBody(result), status: result.status, text: result.text }; +} + +function succeededCondition(object) { + const conditions = Array.isArray(object?.status?.conditions) ? object.status.conditions : []; + return conditions.find((item) => item && item.type === "Succeeded") || null; +} + +function compact(object) { + const condition = succeededCondition(object); + return { + name: object?.metadata?.name || pipelineRun, + namespace: object?.metadata?.namespace || namespace, + generation: object?.metadata?.generation ?? null, + startTime: object?.status?.startTime || null, + completionTime: object?.status?.completionTime || null, + conditionStatus: condition?.status || null, + conditionReason: condition?.reason || null, + conditionMessage: condition?.message || null, + }; +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +let created = false; +let reused = false; +let latest = await getPipelineRun(); +if (latest.found) { + reused = true; +} else { + const result = await request("POST", `/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/pipelineruns`, manifest); + if (result.status === 409) { + reused = true; + } else if (result.status >= 200 && result.status < 300) { + created = true; + } else { + process.stderr.write(result.text || `kube api POST pipelinerun status ${result.status}`); + process.exit(1); + } + latest = await getPipelineRun(); +} + +const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000; +let polls = 0; +while (shouldWait) { + const condition = succeededCondition(latest.object); + if (condition?.status === "True" || condition?.status === "False") break; + if (Date.now() >= deadline) break; + polls += 1; + process.stderr.write(JSON.stringify({ event: "cicd.branch-follower.native-tekton.wait", pipelineRun, namespace, polls, conditionStatus: condition?.status || null, valuesRedacted: true }) + "\n"); + await delay(2000); + latest = await getPipelineRun(); +} + +const condition = succeededCondition(latest.object); +const completed = condition?.status === "True"; +const failed = condition?.status === "False"; +const terminal = completed || failed; +const output = { + ok: !failed, + submitted: true, + created, + reused, + wait: shouldWait, + polls, + completed, + failed, + terminal, + stillRunning: !terminal, + timedOutWait: shouldWait && !terminal, + pipelineRun: compact(latest.object), + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + valuesRedacted: true, +}; +process.stdout.write(JSON.stringify(output)); +if (failed) process.exit(1); diff --git a/scripts/native/cicd/sync-source.sh b/scripts/native/cicd/sync-source.sh new file mode 100644 index 00000000..d324b760 --- /dev/null +++ b/scripts/native/cicd/sync-source.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu + +repository="$1" +branch="$2" +snapshot_prefix="$3" +repo_path="$4" + +private_key="${UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY}" +proxy_host="${UNIDESK_CONTROLLER_GITHUB_PROXY_HOST}" +proxy_port="${UNIDESK_CONTROLLER_GITHUB_PROXY_PORT}" + +mkdir -p "$(dirname "${repo_path}")" /root/.ssh +cp "${private_key}" /root/.ssh/id_rsa +chmod 0400 /root/.ssh/id_rsa +touch /root/.ssh/known_hosts + +test -n "${proxy_host}" +test -n "${proxy_port}" +export GIT_SSH=/etc/unidesk-cicd-branch-follower/git-ssh-proxy.sh +unset GIT_SSH_COMMAND + +remote="ssh://git@ssh.github.com:443/${repository}.git" +if [ -d "${repo_path}/objects" ] && [ -f "${repo_path}/HEAD" ]; then + git --git-dir="${repo_path}" remote set-url origin "${remote}" || git --git-dir="${repo_path}" remote add origin "${remote}" +else + rm -rf "${repo_path}" + git init --bare "${repo_path}" >/dev/null + git --git-dir="${repo_path}" remote add origin "${remote}" +fi + +git --git-dir="${repo_path}" config uploadpack.allowReachableSHA1InWant true +git --git-dir="${repo_path}" config uploadpack.allowAnySHA1InWant true +timeout 30 git --git-dir="${repo_path}" fetch --quiet --prune origin "+refs/heads/${branch}:refs/mirror-stage/heads/${branch}" +source_sha=$(git --git-dir="${repo_path}" rev-parse --verify "refs/mirror-stage/heads/${branch}^{commit}") +git --git-dir="${repo_path}" update-ref "refs/heads/${branch}" "${source_sha}" +if [ -n "${snapshot_prefix}" ]; then + git --git-dir="${repo_path}" update-ref "${snapshot_prefix%/}/${source_sha}" "${source_sha}" +fi +git --git-dir="${repo_path}" update-server-info + +printf '{"event":"unidesk-cicd-git-mirror-sync","repository":"%s","branch":"%s","commit":"%s","sourceAuthority":"k8s-git-mirror-cache"}\n' "${repository}" "${branch}" "${source_sha}" diff --git a/scripts/native/cicd/wait-job.sh b/scripts/native/cicd/wait-job.sh new file mode 100644 index 00000000..02fc2b4d --- /dev/null +++ b/scripts/native/cicd/wait-job.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +deadline=$(( $(date +%s) + ${TIMEOUT_SECONDS} )) + +while true; do + job_json=$(kubectl -n "${NAMESPACE}" get job "${JOB_NAME}" -o json) + phase=$(printf '%s' "${job_json}" | node -e "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>{const j=JSON.parse(s);const c=j.status?.conditions||[];const done=c.find(x=>x.type==='Complete'&&x.status==='True');const failed=c.find(x=>x.type==='Failed'&&x.status==='True');process.stdout.write(done?'complete':failed?'failed':'running');})") + if [ "${phase}" = complete ]; then exit 0; fi + if [ "${phase}" = failed ]; then exit 1; fi + if [ "$(date +%s)" -ge "${deadline}" ]; then exit 124; fi + sleep 2 +done diff --git a/scripts/src/cicd-controller-render.ts b/scripts/src/cicd-controller-render.ts new file mode 100644 index 00000000..8527bdd4 --- /dev/null +++ b/scripts/src/cicd-controller-render.ts @@ -0,0 +1,227 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower controller render helpers. +// Responsibility: Kubernetes controller/reconcile Job manifests and controller bootstrap scripts. +import { readFileSync } from "node:fs"; +import { rootPath } from "./config"; +import { shQuote } from "./platform-infra-ops-library"; +import type { BranchFollowerRegistry, ParsedOptions } from "./cicd-types"; + +const SPEC_REF = "PJ2026-01060703"; + +export function renderControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, jobName: string, mode: { dryRun: boolean; recordState: boolean }, timeoutSeconds: number): Record { + const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-reconcile-job" }; + const commandArgs = [ + "bun", + "scripts/cli.ts", + "cicd", + "branch-follower", + "run-once", + ...(options.followerId === null ? ["--all"] : ["--follower", options.followerId]), + mode.dryRun ? "--dry-run" : "--confirm", + "--wait", + "--controller", + "--config", + "config/cicd-branch-followers.yaml", + "--timeout-seconds", + String(timeoutSeconds), + ...(mode.recordState ? ["--record-state"] : []), + ]; + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { name: jobName, namespace: registry.controller.namespace, labels }, + spec: { + backoffLimit: 0, + ttlSecondsAfterFinished: 600, + activeDeadlineSeconds: timeoutSeconds + 30, + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + serviceAccountName: registry.controller.serviceAccountName, + volumes: [ + { name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } }, + { name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } }, + { name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } }, + { name: "work", emptyDir: {} }, + ], + containers: [ + { + name: "reconcile", + image: registry.controller.image, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "/etc/unidesk-cicd-branch-follower/controller-one-shot.sh"], + args: commandArgs, + env: [ + { name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch }, + { name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository }, + { name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) }, + { name: "UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY", value: `/git-ssh/${registry.controller.source.githubSsh.privateKeySecretKey}` }, + { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_HOST", value: registry.controller.source.githubSsh.proxyHost }, + { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_PORT", value: String(registry.controller.source.githubSsh.proxyPort) }, + ], + volumeMounts: [ + { name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true }, + { name: "git-mirror-cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + { name: "work", mountPath: "/work" }, + ], + }, + ], + }, + }, + }, + }; +} + +export function waitForJobShell(namespace: string, jobName: string, timeoutSeconds: number): string { + return [ + `NAMESPACE=${shQuote(namespace)}`, + `JOB_NAME=${shQuote(jobName)}`, + `TIMEOUT_SECONDS=${shQuote(String(timeoutSeconds))}`, + "export NAMESPACE JOB_NAME TIMEOUT_SECONDS", + nativeCicdScript("wait-job.sh"), + ].join("\n"); +} + +export function renderControllerManifests(registry: BranchFollowerRegistry): Record[] { + const labels = registry.controller.labels; + const selector = labels; + return [ + { + apiVersion: "v1", + kind: "Namespace", + metadata: { name: registry.controller.namespace, labels }, + }, + { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels }, + rules: [ + { apiGroups: [""], resources: ["configmaps", "pods", "events"], verbs: ["get", "list", "watch", "create", "update", "patch"] }, + { apiGroups: ["apps"], resources: ["deployments"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] }, + { apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["get", "list", "watch", "create", "update", "patch"] }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels }, + subjects: [{ kind: "ServiceAccount", name: registry.controller.serviceAccountName, namespace: registry.controller.namespace }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: registry.controller.serviceAccountName }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRole", + metadata: { name: registry.controller.serviceAccountName, labels }, + rules: [ + { apiGroups: [""], resources: ["pods", "pods/log", "configmaps", "events"], verbs: ["get", "list", "watch"] }, + { apiGroups: [""], resources: ["pods/exec"], verbs: ["create"] }, + { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] }, + { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch", "create", "patch", "delete"] }, + { apiGroups: ["tekton.dev"], resources: ["taskruns"], verbs: ["get", "list", "watch"] }, + { apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch", "patch"] }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRoleBinding", + metadata: { name: registry.controller.serviceAccountName, labels }, + subjects: [{ kind: "ServiceAccount", name: registry.controller.serviceAccountName, namespace: registry.controller.namespace }], + roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: registry.controller.serviceAccountName }, + }, + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: registry.controller.configMapName, namespace: registry.controller.namespace, labels }, + data: { + "cicd-branch-followers.yaml": registry.rawText, + "sync-source.sh": nativeCicdScript("sync-source.sh"), + "controller-one-shot.sh": nativeCicdScript("controller-one-shot.sh"), + "controller-loop.sh": nativeCicdScript("controller-loop.sh"), + "github-proxy-connect.mjs": nativeCicdScript("github-proxy-connect.mjs"), + "git-ssh-proxy.sh": nativeCicdScript("git-ssh-proxy.sh"), + }, + }, + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: registry.controller.stateConfigMapName, namespace: registry.controller.namespace, labels }, + data: { + _createdAt: new Date().toISOString(), + _specRef: SPEC_REF, + _registrySha256: registry.rawSha256, + }, + }, + { + apiVersion: "coordination.k8s.io/v1", + kind: "Lease", + metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels }, + spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) }, + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: registry.controller.deploymentName, namespace: registry.controller.namespace, labels }, + spec: { + replicas: 1, + selector: { matchLabels: selector }, + template: { + metadata: { + labels: selector, + annotations: { + "unidesk.pikapython.com/spec-ref": SPEC_REF, + "unidesk.pikapython.com/registry-sha256": registry.rawSha256, + "unidesk.pikapython.com/host-worktree-authority": "false", + }, + }, + spec: { + serviceAccountName: registry.controller.serviceAccountName, + terminationGracePeriodSeconds: 30, + volumes: [ + { name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } }, + { name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } }, + { name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } }, + { name: "work", emptyDir: {} }, + ], + containers: [ + { + name: "controller", + image: registry.controller.image, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "/etc/unidesk-cicd-branch-follower/controller-loop.sh"], + env: [ + { name: "UNIDESK_CICD_BRANCH_FOLLOWER_INTERVAL_SECONDS", value: String(registry.controller.loop.intervalSeconds) }, + { name: "UNIDESK_CICD_BRANCH_FOLLOWER_TIMEOUT_SECONDS", value: String(registry.controller.loop.reconcileTimeoutSeconds) }, + { name: "UNIDESK_CONTROLLER_GIT_MIRROR_READ_URL", value: registry.controller.source.gitMirrorReadUrl }, + { name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch }, + { name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository }, + { name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) }, + { name: "UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY", value: `/git-ssh/${registry.controller.source.githubSsh.privateKeySecretKey}` }, + { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_HOST", value: registry.controller.source.githubSsh.proxyHost }, + { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_PORT", value: String(registry.controller.source.githubSsh.proxyPort) }, + ], + volumeMounts: [ + { name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true }, + { name: "git-mirror-cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + { name: "work", mountPath: "/work" }, + ], + }, + ], + }, + }, + }, + }, + ]; +} + +function nativeCicdScript(name: string): string { + return readFileSync(rootPath("scripts/native/cicd", name), "utf8"); +} diff --git a/scripts/src/cicd-native.ts b/scripts/src/cicd-native.ts new file mode 100644 index 00000000..dbc58bc8 --- /dev/null +++ b/scripts/src/cicd-native.ts @@ -0,0 +1,76 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower native Kubernetes helpers. +// Responsibility: submit/probe Kubernetes Jobs and Tekton PipelineRuns via file-backed native scripts. +import { repoRoot, rootPath } from "./config"; +import { runCommand, type CommandResult } from "./command"; +import type { NativeK8sJobResult } from "./cicd-types"; + +const NATIVE_SCRIPT_DIR = "scripts/native/cicd"; + +export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record, wait: boolean, timeoutSeconds: number): CommandResult { + return runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "submit-pipelinerun.mjs")], repoRoot, { + input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"), + timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000, + env: { + ...process.env, + NAMESPACE: namespace, + PIPELINERUN: pipelineRun, + WAIT: wait ? "true" : "false", + TIMEOUT_SECONDS: String(timeoutSeconds), + }, + }); +} + +export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record, timeoutSeconds: number, logContainer: string): NativeK8sJobResult { + const result = runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "native-job.mjs")], repoRoot, { + input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"), + timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000, + env: { + ...process.env, + NAMESPACE: namespace, + JOB_NAME: jobName, + LOG_CONTAINER: logContainer, + TIMEOUT_SECONDS: String(timeoutSeconds), + }, + }); + const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + return { + ok: result.exitCode === 0 && parsed?.ok === true, + completed: parsed?.completed === true, + failed: parsed?.failed === true || result.exitCode !== 0, + timedOut: parsed?.timedOut === true || result.timedOut, + created: parsed?.created === true, + reused: parsed?.reused === true, + jobName, + namespace, + polls: numberOrNull(parsed?.polls) ?? 0, + elapsedMs: numberOrNull(parsed?.elapsedMs) ?? 0, + logsTail: stringOrNull(parsed?.logsTail), + conditionReason: stringOrNull(parsed?.conditionReason), + conditionMessage: stringOrNull(parsed?.conditionMessage) ?? (result.exitCode === 0 ? null : tailText(result.stderr || result.stdout, 500)), + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + }; +} + +function parseJsonObject(text: string): Record | null { + const trimmed = text.trim(); + if (trimmed.length === 0) return null; + try { + const parsed = JSON.parse(trimmed) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; + } catch { + return null; + } +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function tailText(text: string, maxChars: number): string { + return text.length <= maxChars ? text : text.slice(text.length - maxChars); +} diff --git a/scripts/src/cicd-types.ts b/scripts/src/cicd-types.ts new file mode 100644 index 00000000..3209bca7 --- /dev/null +++ b/scripts/src/cicd-types.ts @@ -0,0 +1,299 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower shared types. +// Responsibility: type contracts shared by branch follower entry, controller render, and native K8s helpers. + +export type OutputMode = "human" | "json" | "yaml"; +export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "cleanup-state" | "events" | "logs"; +export type BranchFollowerPhase = + | "Observed" + | "Noop" + | "PendingTrigger" + | "Triggering" + | "ClosingOut" + | "Succeeded" + | "Failed" + | "Superseded" + | "Blocked" + | "Skipped"; + +export interface ParsedOptions { + action: BranchFollowerAction; + configPath: string; + followerId: string | null; + all: boolean; + confirm: boolean; + dryRun: boolean; + wait: boolean; + controller: boolean; + live: boolean; + noLive: boolean; + full: boolean; + raw: boolean; + recordState: boolean; + output: OutputMode; + limit: number; + tailBytes: number; + timeoutSeconds: number | null; +} + +export interface CommandSpec { + argv: string[]; + timeoutSeconds: number; +} + +export interface FollowerSpec { + id: string; + enabled: boolean; + adapter: string; + description: string; + source: { + repository: string; + branch: string; + branchRef: string; + authorityRef: string; + snapshotPrefix: string; + snapshotRef: string; + }; + target: { + node: string; + lane: string; + namespace: string; + sentinel: string | null; + configRefs: Record; + }; + budgets: { + endToEndSeconds: number; + statusSeconds: number; + triggerSeconds: number; + sourceSyncSeconds: number; + }; + commands: { + plan: CommandSpec; + status: CommandSpec; + trigger: CommandSpec; + events: CommandSpec; + logs: CommandSpec; + }; + nativeStatus: NativeStatusSpec; + closeoutChecks: string[]; +} + +export interface NativeStatusSpec { + source: { + gitMirrorReadUrl: string; + gitMirrorNamespace: string; + gitMirrorDeployment: string; + repoPath: string; + }; + tekton: { + namespace: string; + pipelineRunPrefix: string; + } | null; + argo: { + namespace: string; + application: string; + } | null; + runtime: { + namespace: string; + workloads: NativeWorkloadSpec[]; + } | null; +} + +export interface NativeWorkloadSpec { + kind: "Deployment" | "StatefulSet"; + name: string; + sourceCommit: { + labels: string[]; + annotations: string[]; + podLabels: string[]; + podAnnotations: string[]; + env: string[]; + }; +} + +export interface ControllerSpec { + namespace: string; + kubeRoute: string; + fieldManager: string; + serviceAccountName: string; + deploymentName: string; + configMapName: string; + stateConfigMapName: string; + leaseName: string; + image: string; + labels: Record; + source: { + repository: string; + branch: string; + gitMirrorReadUrl: string; + gitMirrorCachePvcName: string; + githubSsh: { + secretName: string; + privateKeySecretKey: string; + proxyHost: string; + proxyPort: number; + }; + sourceAuthority: { + mode: string; + resolver: string; + allowHostGit: boolean; + allowHostWorkspace: boolean; + allowGithubDirectInPipeline: boolean; + }; + sourceSnapshot: { + stageRefPrefix: string; + missingObjectPolicy: string; + refreshPolicy: string; + }; + }; + loop: { + intervalSeconds: number; + reconcileTimeoutSeconds: number; + }; + budgets: { + applyWaitSeconds: number; + statusSeconds: number; + runOnceSeconds: number; + }; +} + +export interface BranchFollowerRegistry { + path: string; + rawText: string; + rawSha256: string; + metadata: { + id: string; + owner: string; + specRef: string; + version: string; + }; + controller: ControllerSpec; + followers: FollowerSpec[]; +} + +export interface AdapterSummary { + ok: boolean; + command: string; + exitCode: number | null; + timedOut: boolean; + observedSha: string | null; + targetSha: string | null; + lastTriggeredSha: string | null; + lastSucceededSha: string | null; + pipelineRun: string | null; + pipelineRunPresent: boolean | null; + inFlightJob: string | null; + aligned: boolean | null; + phase: BranchFollowerPhase; + message: string; + payload: Record | null; + stderrTail: string; + stdoutTail: string; +} + +export interface NativeObjectBundle { + ok: boolean; + source: Record | null; + gitMirror: Record | null; + pipelineRun: Record | null; + taskRuns: Record | null; + planArtifacts: Record | null; + argoApplication: Record | null; + workloads: Record[]; + errors: string[]; + exitCode: number | null; + timedOut: boolean; + stdoutTail: string; + stderrTail: string; +} + +export interface TriggerResult { + ok: boolean; + completed: boolean; + message: string; + jobId: string | null; + command: Record; +} + +export interface NativeCloseoutWaitResult { + ok: boolean; + completed: boolean; + timedOut: boolean; + polls: number; + elapsedMs: number; + refresh: Record | null; + summary: Record | null; + statusAuthority: "k8s-native"; + parsedDownstreamCliOutput: false; +} + +export interface NativeK8sJobResult { + ok: boolean; + completed: boolean; + failed: boolean; + timedOut: boolean; + created: boolean; + reused: boolean; + jobName: string; + namespace: string; + polls: number; + elapsedMs: number; + logsTail: string | null; + conditionReason: string | null; + conditionMessage: string | null; + statusAuthority: "kubernetes-api-serviceaccount"; + parsedDownstreamCliOutput: false; +} + +export interface FollowerState { + id: string; + adapter: string; + enabled: boolean; + phase: BranchFollowerPhase; + source: { + repository: string; + branch: string; + branchRef: string; + snapshotPrefix: string; + observedSha: string | null; + }; + target: { + node: string; + lane: string; + namespace: string; + sentinel: string | null; + targetSha: string | null; + }; + lastTriggeredSha: string | null; + lastSucceededSha: string | null; + pipelineRun: string | null; + inFlightJob: string | null; + budgetSource: Record; + controller: { + mode: "local-cli" | "k8s-controller"; + stateConfigMap: string; + leaseName: string; + }; + decision: string; + dryRun: boolean; + updatedAt: string; + warnings: string[]; + next: Record; + command?: Record; +} + +export interface K8sStateRead { + ok: boolean; + stateByFollower: Record>; + stateConfigMapPresent: boolean; + deployment: Record | null; + lease: Record | null; + pods: Record | null; + errors: string[]; +} + +export interface K8sFollowerStateRead { + ok: boolean; + stateByFollower: Record>; + present: boolean; + error: string; +} diff --git a/scripts/src/cicd.ts b/scripts/src/cicd.ts index 91c29409..c919465a 100644 --- a/scripts/src/cicd.ts +++ b/scripts/src/cicd.ts @@ -13,12 +13,16 @@ import { agentRunPipelineRunName, resolveAgentRunLaneTarget } from "./agentrun-l import { yamlLaneGitMirrorJobManifest, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLanePipelineRunManifest } from "./agentrun/secrets"; import { yamlLaneK3sBuildImageJobManifest } from "./agentrun/yaml-lane"; import { nodeRuntimePipelineRunName } from "./hwlab-node/cleanup"; -import { nodeRuntimePipelineRunManifest } from "./hwlab-node/web-probe"; +import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render"; +import { nodeRuntimeGitMirrorTarget, nodeRuntimePipelineRunManifest } from "./hwlab-node/web-probe"; import { loadSentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; import { sentinelPublishPipelineRunManifest } from "./hwlab-node-web-sentinel-cicd-jobs"; import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared"; import { transPath } from "./hwlab-node/runtime-common"; import { configRefGraph, resolveConfigRefString } from "./ops/config-refs"; +import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render"; +import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; +import type { AdapterSummary, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeObjectBundle, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, TriggerResult } from "./cicd-types"; import { arrayField, asRecord, @@ -36,300 +40,6 @@ const DEFAULT_CONFIG_PATH = "config/cicd-branch-followers.yaml"; const SPEC_REF = "PJ2026-01060703"; const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; -type OutputMode = "human" | "json" | "yaml"; -type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "cleanup-state" | "events" | "logs"; -type BranchFollowerPhase = - | "Observed" - | "Noop" - | "PendingTrigger" - | "Triggering" - | "ClosingOut" - | "Succeeded" - | "Failed" - | "Superseded" - | "Blocked" - | "Skipped"; - -interface ParsedOptions { - action: BranchFollowerAction; - configPath: string; - followerId: string | null; - all: boolean; - confirm: boolean; - dryRun: boolean; - wait: boolean; - controller: boolean; - live: boolean; - noLive: boolean; - full: boolean; - raw: boolean; - recordState: boolean; - output: OutputMode; - limit: number; - tailBytes: number; - timeoutSeconds: number | null; -} - -interface CommandSpec { - argv: string[]; - timeoutSeconds: number; -} - -interface FollowerSpec { - id: string; - enabled: boolean; - adapter: string; - description: string; - source: { - repository: string; - branch: string; - branchRef: string; - authorityRef: string; - snapshotPrefix: string; - snapshotRef: string; - }; - target: { - node: string; - lane: string; - namespace: string; - sentinel: string | null; - configRefs: Record; - }; - budgets: { - endToEndSeconds: number; - statusSeconds: number; - triggerSeconds: number; - sourceSyncSeconds: number; - }; - commands: { - plan: CommandSpec; - status: CommandSpec; - trigger: CommandSpec; - events: CommandSpec; - logs: CommandSpec; - }; - nativeStatus: NativeStatusSpec; - closeoutChecks: string[]; -} - -interface NativeStatusSpec { - source: { - gitMirrorReadUrl: string; - gitMirrorNamespace: string; - gitMirrorDeployment: string; - repoPath: string; - }; - tekton: { - namespace: string; - pipelineRunPrefix: string; - } | null; - argo: { - namespace: string; - application: string; - } | null; - runtime: { - namespace: string; - workloads: NativeWorkloadSpec[]; - } | null; -} - -interface NativeWorkloadSpec { - kind: "Deployment" | "StatefulSet"; - name: string; - sourceCommit: { - labels: string[]; - annotations: string[]; - podLabels: string[]; - podAnnotations: string[]; - env: string[]; - }; -} - -interface ControllerSpec { - namespace: string; - kubeRoute: string; - fieldManager: string; - serviceAccountName: string; - deploymentName: string; - configMapName: string; - stateConfigMapName: string; - leaseName: string; - image: string; - labels: Record; - source: { - repository: string; - branch: string; - gitMirrorReadUrl: string; - gitMirrorCachePvcName: string; - githubSsh: { - secretName: string; - privateKeySecretKey: string; - proxyHost: string; - proxyPort: number; - }; - sourceAuthority: { - mode: string; - resolver: string; - allowHostGit: boolean; - allowHostWorkspace: boolean; - allowGithubDirectInPipeline: boolean; - }; - sourceSnapshot: { - stageRefPrefix: string; - missingObjectPolicy: string; - refreshPolicy: string; - }; - }; - loop: { - intervalSeconds: number; - reconcileTimeoutSeconds: number; - }; - budgets: { - applyWaitSeconds: number; - statusSeconds: number; - runOnceSeconds: number; - }; -} - -interface BranchFollowerRegistry { - path: string; - rawText: string; - rawSha256: string; - metadata: { - id: string; - owner: string; - specRef: string; - version: string; - }; - controller: ControllerSpec; - followers: FollowerSpec[]; -} - -interface AdapterSummary { - ok: boolean; - command: string; - exitCode: number | null; - timedOut: boolean; - observedSha: string | null; - targetSha: string | null; - lastTriggeredSha: string | null; - lastSucceededSha: string | null; - pipelineRun: string | null; - pipelineRunPresent: boolean | null; - inFlightJob: string | null; - aligned: boolean | null; - phase: BranchFollowerPhase; - message: string; - payload: Record | null; - stderrTail: string; - stdoutTail: string; -} - -interface NativeObjectBundle { - ok: boolean; - source: Record | null; - pipelineRun: Record | null; - argoApplication: Record | null; - workloads: Record[]; - errors: string[]; - exitCode: number | null; - timedOut: boolean; - stdoutTail: string; - stderrTail: string; -} - -interface TriggerResult { - ok: boolean; - completed: boolean; - message: string; - jobId: string | null; - command: Record; -} - -interface NativeCloseoutWaitResult { - ok: boolean; - completed: boolean; - timedOut: boolean; - polls: number; - elapsedMs: number; - refresh: Record | null; - summary: Record | null; - statusAuthority: "k8s-native"; - parsedDownstreamCliOutput: false; -} - -interface NativeK8sJobResult { - ok: boolean; - completed: boolean; - failed: boolean; - timedOut: boolean; - created: boolean; - reused: boolean; - jobName: string; - namespace: string; - polls: number; - elapsedMs: number; - logsTail: string | null; - conditionReason: string | null; - conditionMessage: string | null; - statusAuthority: "kubernetes-api-serviceaccount"; - parsedDownstreamCliOutput: false; -} - -interface FollowerState { - id: string; - adapter: string; - enabled: boolean; - phase: BranchFollowerPhase; - source: { - repository: string; - branch: string; - branchRef: string; - snapshotPrefix: string; - observedSha: string | null; - }; - target: { - node: string; - lane: string; - namespace: string; - sentinel: string | null; - targetSha: string | null; - }; - lastTriggeredSha: string | null; - lastSucceededSha: string | null; - pipelineRun: string | null; - inFlightJob: string | null; - budgetSource: Record; - controller: { - mode: "local-cli" | "k8s-controller"; - stateConfigMap: string; - leaseName: string; - }; - decision: string; - dryRun: boolean; - updatedAt: string; - warnings: string[]; - next: Record; - command?: Record; -} - -interface K8sStateRead { - ok: boolean; - stateByFollower: Record>; - stateConfigMapPresent: boolean; - deployment: Record | null; - lease: Record | null; - pods: Record | null; - errors: string[]; -} - -interface K8sFollowerStateRead { - ok: boolean; - stateByFollower: Record>; - present: boolean; - error: string; -} - export function cicdHelp(): unknown { return { command: "cicd branch-follower plan|apply|status|run-once|cleanup-state|events|logs", @@ -1049,10 +759,13 @@ async function decideAndMaybeTrigger( } else if (superseded) { phase = "Superseded"; decision = `previous in-flight sha ${shortSha(previousObserved)} was superseded by ${shortSha(observedSha)}`; - } else if (targetSha !== null && targetSha === observedSha) { + } else if (targetSha !== null && targetSha === observedSha && live.aligned === true) { phase = "Noop"; decision = "target already matches observed source sha"; lastSucceededSha = observedSha; + } else if (targetSha !== null && targetSha === observedSha) { + phase = "PendingTrigger"; + decision = "target sha matches observed source sha but native closeout gates are not complete; resume deterministic closeout"; } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha) { phase = "ClosingOut"; decision = "same sha was already triggered; use status/events/logs for closeout"; @@ -1175,10 +888,20 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f const namespace = follower.nativeStatus.tekton.namespace; const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun); const startedAt = Date.now(); - const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds); + const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds))); + if (sync !== null && !sync.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed"); + } + const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds)); const payload = parseJsonObject(result.stdout) ?? {}; const pipelineRunCompleted = payload.completed === true; const failed = payload.failed === true || result.exitCode !== 0; + const flush = !failed && options.wait && pipelineRunCompleted + ? runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)) + : null; + if (flush !== null && !flush.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed"); + } const closeout = !failed && options.wait && pipelineRunCompleted ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) : null; @@ -1190,7 +913,13 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f stageRef: `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`, wait: options.wait, result, - payload, + payload: { + ...payload, + nativeCapabilities: { + gitMirrorSync: sync === null ? null : sync.result, + gitMirrorFlush: flush === null ? null : flush.result, + }, + }, closeout, successMessage: `native HWLAB PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, }); @@ -1207,13 +936,17 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo const startedAt = Date.now(); const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; + const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds))); + if (sync !== null && !sync.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed"); + } const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63); const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds)), "buildkit"); const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" }); const digest = stringOrNull(buildPayload.digest); const envIdentity = stringOrNull(buildPayload.envIdentity); if (!build.ok || digest === null || envIdentity === null) { - return nativeAgentRunStageFailure(follower, observedSha, "image-build", buildJob, build, buildPayload, "native AgentRun image build failed"); + return nativeK8sStageFailure(follower, observedSha, "image-build", buildJob, build, buildPayload, "native AgentRun image build failed"); } const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image }); @@ -1221,12 +954,11 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish"); const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" }); if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) { - return nativeAgentRunStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed"); + return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed"); } - const flushJob = `${jobPrefix}-flush-${observedSha.slice(0, 12)}`.slice(0, 63); - const flush = runNativeK8sJob(spec.gitMirror.namespace, flushJob, yamlLaneGitMirrorJobManifest(spec, "flush", flushJob), remainingSeconds(startedAt, timeoutSeconds), "flush"); - if (!flush.ok) { - return nativeAgentRunStageFailure(follower, observedSha, "git-mirror-flush", flushJob, flush, {}, "native AgentRun git-mirror flush failed"); + const flush = runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)); + if (flush !== null && !flush.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed"); } const pipelineRun = agentRunPipelineRunName(spec, observedSha); const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds)); @@ -1248,9 +980,10 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo ...tektonPayload, agentrun: { configPath, + gitMirrorSync: sync === null ? null : { jobName: sync.jobName, payload: sync.result }, imageBuild: { jobName: buildJob, payload: buildPayload }, gitopsPublish: { jobName: publishJob, payload: publishPayload }, - gitMirrorFlush: { jobName: flushJob, payload: flush }, + gitMirrorFlush: flush === null ? null : { jobName: flush.jobName, payload: flush.result }, }, }, closeout, @@ -1269,7 +1002,7 @@ function nativeTriggerError(follower: FollowerSpec, message: string, reason: str }; } -function nativeAgentRunStageFailure( +function nativeK8sStageFailure( follower: FollowerSpec, observedSha: string, phase: string, @@ -1305,6 +1038,44 @@ function nativeAgentRunStageFailure( }; } +function runNativeGitMirrorStage( + follower: FollowerSpec, + observedSha: string, + action: "sync" | "flush", + timeoutSeconds: number, +): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { + const job = nativeGitMirrorJobForFollower(follower, observedSha, action); + if (job === null) return null; + const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action); + return { jobName: job.jobName, namespace: job.namespace, result }; +} + +function nativeGitMirrorJobForFollower( + follower: FollowerSpec, + observedSha: string, + action: "sync" | "flush", +): { namespace: string; jobName: string; manifest: Record } | null { + const jobName = nativeCapabilityJobName(follower.id, action, observedSha); + if (follower.adapter === "hwlab-node-runtime") { + const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); + const mirror = nodeRuntimeGitMirrorTarget(spec); + return { + namespace: mirror.namespace, + jobName, + manifest: nodeRuntimeGitMirrorJobManifest(mirror, action, jobName), + }; + } + if (follower.adapter === "agentrun-yaml-lane") { + const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); + return { + namespace: spec.gitMirror.namespace, + jobName, + manifest: yamlLaneGitMirrorJobManifest(spec, action, jobName), + }; + } + return null; +} + function nativeTektonTriggerResult(input: { follower: FollowerSpec; observedSha: string; @@ -1506,7 +1277,10 @@ function nativeCloseoutSummary(live: AdapterSummary): Record { pipelineRun: live.pipelineRun, pipelineRunPresent: live.pipelineRunPresent, message: live.message, + gitMirror: asOptionalRecord(payload?.gitMirror), tekton: asOptionalRecord(payload?.tekton), + taskRuns: compactTaskRunsPayload(asOptionalRecord(payload?.taskRuns)), + planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload?.planArtifacts)), argo: asOptionalRecord(payload?.argo), runtime: asOptionalRecord(payload?.runtime), errors: Array.isArray(payload?.errors) ? payload.errors.slice(0, 5) : [], @@ -1555,290 +1329,6 @@ function runNativeArgoRefresh(argo: NonNullable): Comm return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: 10_000 }); } -function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record, wait: boolean, timeoutSeconds: number): CommandResult { - const manifestJson = JSON.stringify(manifest); - const manifestBase64 = Buffer.from(manifestJson, "utf8").toString("base64"); - const nodeScript = nativeTektonPipelineRunNodeScript(); - const script = [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - "cat >\"$tmpdir/manifest.b64\" <<'UNIDESK_SENTINEL_PIPELINERUN_B64'", - manifestBase64, - "UNIDESK_SENTINEL_PIPELINERUN_B64", - `NAMESPACE=${shQuote(namespace)}`, - `PIPELINERUN=${shQuote(pipelineRun)}`, - `WAIT=${wait ? "true" : "false"}`, - `TIMEOUT_SECONDS=${shQuote(String(timeoutSeconds))}`, - "export NAMESPACE PIPELINERUN WAIT TIMEOUT_SECONDS", - "cat >\"$tmpdir/submit-pipelinerun.mjs\" <<'NODE_NATIVE_TEKTON'", - nodeScript, - "NODE_NATIVE_TEKTON", - "node \"$tmpdir/submit-pipelinerun.mjs\" \"$tmpdir/manifest.b64\"", - ].join("\n"); - return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000 }); -} - -function runNativeK8sJob(namespace: string, jobName: string, manifest: Record, timeoutSeconds: number, logContainer: string): NativeK8sJobResult { - const manifestBase64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); - const script = [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - "cat >\"$tmpdir/manifest.b64\" <<'UNIDESK_NATIVE_JOB_B64'", - manifestBase64, - "UNIDESK_NATIVE_JOB_B64", - `NAMESPACE=${shQuote(namespace)}`, - `JOB_NAME=${shQuote(jobName)}`, - `LOG_CONTAINER=${shQuote(logContainer)}`, - `TIMEOUT_SECONDS=${shQuote(String(timeoutSeconds))}`, - "export NAMESPACE JOB_NAME LOG_CONTAINER TIMEOUT_SECONDS", - "cat >\"$tmpdir/native-job.mjs\" <<'NODE_NATIVE_JOB'", - nativeK8sJobNodeScript(), - "NODE_NATIVE_JOB", - "node \"$tmpdir/native-job.mjs\" \"$tmpdir/manifest.b64\"", - ].join("\n"); - const result = runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000 }); - const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; - return { - ok: result.exitCode === 0 && parsed?.ok === true, - completed: parsed?.completed === true, - failed: parsed?.failed === true || result.exitCode !== 0, - timedOut: parsed?.timedOut === true || result.timedOut, - created: parsed?.created === true, - reused: parsed?.reused === true, - jobName, - namespace, - polls: numberOrNull(parsed?.polls) ?? 0, - elapsedMs: numberOrNull(parsed?.elapsedMs) ?? 0, - logsTail: stringOrNull(parsed?.logsTail), - conditionReason: stringOrNull(parsed?.conditionReason), - conditionMessage: stringOrNull(parsed?.conditionMessage) ?? (result.exitCode === 0 ? null : tailText(result.stderr || result.stdout, 500)), - statusAuthority: "kubernetes-api-serviceaccount", - parsedDownstreamCliOutput: false, - }; -} - -function nativeK8sJobNodeScript(): string { - return [ - "import { readFileSync } from 'node:fs';", - "import https from 'node:https';", - "const manifestPath = process.argv[2];", - "const namespace = process.env.NAMESPACE || '';", - "const jobName = process.env.JOB_NAME || '';", - "const logContainer = process.env.LOG_CONTAINER || '';", - "const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || '60');", - "const host = process.env.KUBERNETES_SERVICE_HOST;", - "const port = Number(process.env.KUBERNETES_SERVICE_PORT || '443');", - "const token = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/token', 'utf8').trim();", - "const ca = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt');", - "const manifest = JSON.parse(Buffer.from(readFileSync(manifestPath, 'utf8').replace(/\\s+/g, ''), 'base64').toString('utf8'));", - "function request(method, path, body, contentType = 'application/json') {", - " return new Promise((resolve, reject) => {", - " const headers = { authorization: `Bearer ${token}` };", - " const payload = body === undefined ? null : typeof body === 'string' ? body : JSON.stringify(body);", - " if (payload !== null) { headers['content-type'] = contentType; headers['content-length'] = Buffer.byteLength(payload); }", - " const req = https.request({ host, port, path, method, ca, headers }, (res) => {", - " let text = '';", - " res.setEncoding('utf8');", - " res.on('data', (chunk) => { text += chunk; });", - " res.on('end', () => resolve({ status: res.statusCode || 0, text }));", - " });", - " req.on('error', reject);", - " if (payload !== null) req.write(payload);", - " req.end();", - " });", - "}", - "function parse(text) { try { return text ? JSON.parse(text) : null; } catch { return null; } }", - "function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }", - "function condition(job, type) { return (Array.isArray(job?.status?.conditions) ? job.status.conditions : []).find((item) => item?.type === type && item?.status === 'True') || null; }", - "async function getJob() {", - " const result = await request('GET', `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs/${encodeURIComponent(jobName)}`);", - " if (result.status === 404) return null;", - " if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api GET job status ${result.status}`);", - " return parse(result.text);", - "}", - "async function podNames() {", - " const selector = encodeURIComponent(`job-name=${jobName}`);", - " const result = await request('GET', `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${selector}`);", - " if (result.status < 200 || result.status >= 300) return [];", - " const list = parse(result.text);", - " return (Array.isArray(list?.items) ? list.items : []).map((pod) => pod?.metadata?.name).filter(Boolean);", - "}", - "async function logsTail() {", - " const names = await podNames();", - " let combined = '';", - " for (const pod of names.slice(-2)) {", - " const container = logContainer ? `&container=${encodeURIComponent(logContainer)}` : '';", - " const result = await request('GET', `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods/${encodeURIComponent(pod)}/log?tailLines=120${container}`);", - " if (result.status >= 200 && result.status < 300) combined += `${result.text}\\n`;", - " }", - " return combined.length > 6000 ? combined.slice(-6000) : combined;", - "}", - "let created = false;", - "let reused = false;", - "let existing = await getJob();", - "if (existing) {", - " reused = true;", - "} else {", - " const result = await request('POST', `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs`, manifest);", - " if (result.status === 409) reused = true;", - " else if (result.status >= 200 && result.status < 300) created = true;", - " else { process.stderr.write(result.text || `kube api POST job status ${result.status}`); process.exit(1); }", - "}", - "const startedAt = Date.now();", - "const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000;", - "let polls = 0;", - "let latest = await getJob();", - "while (Date.now() <= deadline) {", - " const complete = condition(latest, 'Complete');", - " const failed = condition(latest, 'Failed');", - " if (complete || failed) break;", - " polls += 1;", - " await delay(2000);", - " latest = await getJob();", - "}", - "const complete = condition(latest, 'Complete');", - "const failed = condition(latest, 'Failed');", - "const logs = await logsTail();", - "const timedOut = !complete && !failed;", - "const output = {", - " ok: Boolean(complete) && !timedOut,", - " completed: Boolean(complete),", - " failed: Boolean(failed),", - " timedOut,", - " created,", - " reused,", - " jobName,", - " namespace,", - " polls,", - " elapsedMs: Date.now() - startedAt,", - " conditionReason: complete?.reason || failed?.reason || null,", - " conditionMessage: complete?.message || failed?.message || null,", - " logsTail: logs || null,", - " statusAuthority: 'kubernetes-api-serviceaccount',", - " parsedDownstreamCliOutput: false,", - " valuesRedacted: true,", - "};", - "process.stdout.write(JSON.stringify(output));", - "if (!output.ok) process.exit(1);", - ].join("\n"); -} - -function nativeTektonPipelineRunNodeScript(): string { - return [ - "import { readFileSync } from 'node:fs';", - "import https from 'node:https';", - "const manifestPath = process.argv[2];", - "const namespace = process.env.NAMESPACE || '';", - "const pipelineRun = process.env.PIPELINERUN || '';", - "const shouldWait = process.env.WAIT === 'true';", - "const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || '60');", - "const host = process.env.KUBERNETES_SERVICE_HOST;", - "const port = Number(process.env.KUBERNETES_SERVICE_PORT || '443');", - "const token = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/token', 'utf8').trim();", - "const ca = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt');", - "const manifest = JSON.parse(Buffer.from(readFileSync(manifestPath, 'utf8').replace(/\\s+/g, ''), 'base64').toString('utf8'));", - "function request(method, path, body, contentType = 'application/json') {", - " return new Promise((resolve, reject) => {", - " const headers = { authorization: `Bearer ${token}` };", - " const payload = body === undefined ? null : typeof body === 'string' ? body : JSON.stringify(body);", - " if (payload !== null) { headers['content-type'] = contentType; headers['content-length'] = Buffer.byteLength(payload); }", - " const req = https.request({ host, port, path, method, ca, headers }, (res) => {", - " let text = '';", - " res.setEncoding('utf8');", - " res.on('data', (chunk) => { text += chunk; });", - " res.on('end', () => resolve({ status: res.statusCode || 0, text }));", - " });", - " req.on('error', reject);", - " if (payload !== null) req.write(payload);", - " req.end();", - " });", - "}", - "function parseBody(result) {", - " if (!result.text) return null;", - " try { return JSON.parse(result.text); } catch { return null; }", - "}", - "async function getPipelineRun() {", - " const result = await request('GET', `/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/pipelineruns/${encodeURIComponent(pipelineRun)}`);", - " if (result.status === 404) return { found: false, object: null, status: result.status, text: result.text };", - " if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api GET pipelinerun status ${result.status}`);", - " return { found: true, object: parseBody(result), status: result.status, text: result.text };", - "}", - "function succeededCondition(object) {", - " const conditions = Array.isArray(object?.status?.conditions) ? object.status.conditions : [];", - " return conditions.find((item) => item && item.type === 'Succeeded') || null;", - "}", - "function compact(object) {", - " const condition = succeededCondition(object);", - " return {", - " name: object?.metadata?.name || pipelineRun,", - " namespace: object?.metadata?.namespace || namespace,", - " generation: object?.metadata?.generation ?? null,", - " startTime: object?.status?.startTime || null,", - " completionTime: object?.status?.completionTime || null,", - " conditionStatus: condition?.status || null,", - " conditionReason: condition?.reason || null,", - " conditionMessage: condition?.message || null,", - " };", - "}", - "function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }", - "let created = false;", - "let reused = false;", - "let latest = await getPipelineRun();", - "if (latest.found) {", - " reused = true;", - "} else {", - " const result = await request('POST', `/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/pipelineruns`, manifest);", - " if (result.status === 409) {", - " reused = true;", - " } else if (result.status >= 200 && result.status < 300) {", - " created = true;", - " } else {", - " process.stderr.write(result.text || `kube api POST pipelinerun status ${result.status}`);", - " process.exit(1);", - " }", - " latest = await getPipelineRun();", - "}", - "const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000;", - "let polls = 0;", - "while (shouldWait) {", - " const condition = succeededCondition(latest.object);", - " if (condition?.status === 'True' || condition?.status === 'False') break;", - " if (Date.now() >= deadline) break;", - " polls += 1;", - " process.stderr.write(JSON.stringify({ event: 'cicd.branch-follower.native-tekton.wait', pipelineRun, namespace, polls, conditionStatus: condition?.status || null, valuesRedacted: true }) + '\\n');", - " await delay(2000);", - " latest = await getPipelineRun();", - "}", - "const condition = succeededCondition(latest.object);", - "const completed = condition?.status === 'True';", - "const failed = condition?.status === 'False';", - "const terminal = completed || failed;", - "const output = {", - " ok: !failed,", - " submitted: true,", - " created,", - " reused,", - " wait: shouldWait,", - " polls,", - " completed,", - " failed,", - " terminal,", - " stillRunning: !terminal,", - " timedOutWait: shouldWait && !terminal,", - " pipelineRun: compact(latest.object),", - " statusAuthority: 'kubernetes-api-serviceaccount',", - " parsedDownstreamCliOutput: false,", - " valuesRedacted: true,", - "};", - "process.stdout.write(JSON.stringify(output));", - "if (failed) process.exit(1);", - ].join("\n"); -} - async function readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise { const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds; const bundle = readNativeObjectBundle(registry, follower, options, timeoutSeconds); @@ -1849,6 +1339,8 @@ async function readAdapterStatus(registry: BranchFollowerRegistry, follower: Fol const pipelineSucceeded = pipelineRunSucceeded(bundle.pipelineRun); const argoReady = follower.nativeStatus.argo === null ? null : argoApplicationReady(bundle.argoApplication); const runtimeReady = follower.nativeStatus.runtime === null ? null : runtimeWorkloadsReady(follower.nativeStatus.runtime, bundle.workloads); + const gitMirrorRequired = nativeGitMirrorRequired(follower); + const gitMirrorReady = gitMirrorRequired ? nativeGitMirrorReady(bundle.gitMirror) : null; const hasTektonGate = follower.nativeStatus.tekton !== null; const hasRuntimeTarget = runtimeTargetSha !== null; const requiresRuntimeTarget = follower.nativeStatus.runtime !== null; @@ -1860,6 +1352,7 @@ async function readAdapterStatus(registry: BranchFollowerRegistry, follower: Fol && hasTargetEvidence && (requiresRuntimeTarget ? hasRuntimeTarget && runtimeTargetSha === observedSha : true) && pipelineGateOk + && gitMirrorReady !== false && argoReady !== false && runtimeReady !== false; const targetSha = hasRuntimeTarget && runtimeTargetSha === observedSha && aligned ? runtimeTargetSha : runtimeTargetSha; @@ -1881,13 +1374,17 @@ async function readAdapterStatus(registry: BranchFollowerRegistry, follower: Fol phase, message: nativeStatusMessage(ok, phase, observedSha, targetSha, { pipelineSucceeded, + gitMirrorReady, argoReady, runtimeReady, errors: bundle.errors, }), payload: { source: bundle.source, + gitMirror: nativeGitMirrorSummary(bundle.gitMirror), tekton: nativePipelineRunSummary(bundle.pipelineRun), + taskRuns: bundle.taskRuns, + planArtifacts: bundle.planArtifacts, argo: nativeArgoSummary(bundle.argoApplication), runtime: nativeRuntimeSummary(follower.nativeStatus.runtime, bundle.workloads), errors: bundle.errors, @@ -1902,21 +1399,22 @@ async function readAdapterStatus(registry: BranchFollowerRegistry, follower: Fol function inferPhase(ok: boolean, aligned: boolean | null, observedSha: string | null, targetSha: string | null, timedOut: boolean): BranchFollowerPhase { if (!ok || timedOut) return "Blocked"; if (aligned === true) return "Succeeded"; - if (observedSha !== null && targetSha !== null && observedSha === targetSha) return "Noop"; + if (observedSha !== null && targetSha !== null && observedSha === targetSha) return "ClosingOut"; if (observedSha !== null) return "PendingTrigger"; return "Observed"; } -function nativeStatusMessage(ok: boolean, phase: BranchFollowerPhase, observedSha: string | null, targetSha: string | null, gates: { pipelineSucceeded: boolean | null; argoReady: boolean | null; runtimeReady: boolean | null; errors: string[] }): string { +function nativeStatusMessage(ok: boolean, phase: BranchFollowerPhase, observedSha: string | null, targetSha: string | null, gates: { pipelineSucceeded: boolean | null; gitMirrorReady: boolean | null; argoReady: boolean | null; runtimeReady: boolean | null; errors: string[] }): string { if (!ok) return gates.errors[0] ?? "native status read failed"; if (phase === "Noop" || phase === "Succeeded") return `target matches ${shortSha(observedSha)}`; - if (observedSha !== null && targetSha !== null) return `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}`; if (observedSha !== null) { const gatesText = [ gates.pipelineSucceeded === false ? "pipelineRun not successful" : null, + gates.gitMirrorReady === false ? "git-mirror not flushed" : null, gates.argoReady === false ? "argo not healthy/synced" : null, gates.runtimeReady === false ? "runtime not ready" : null, ].filter((item): item is string => item !== null).join("; "); + if (targetSha !== null) return gatesText.length > 0 ? `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}`; return gatesText.length > 0 ? `observed ${shortSha(observedSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target unknown`; } return "k8s-native status did not expose observed source sha"; @@ -1928,147 +1426,31 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll const tekton = native.tekton; const argo = native.argo; const runtime = native.runtime; - const workloadCommands = (runtime?.workloads ?? []).map((workload, index) => { + const gitopsBranch = nativeGitMirrorGitopsBranch(follower); + const workloadRefs = (runtime?.workloads ?? []).map((workload, index) => { const resource = workload.kind === "Deployment" ? "deployments" : "statefulsets"; - return `emit_kube_json ${shQuote(`workload${index}`)} ${shQuote(`/apis/apps/v1/namespaces/${runtime?.namespace ?? follower.target.namespace}/${resource}/${workload.name}`)}`; + return `workload${index}\t/apis/apps/v1/namespaces/${runtime?.namespace ?? follower.target.namespace}/${resource}/${workload.name}`; }); + const workloadRefsText = workloadRefs.length === 0 ? "" : `${workloadRefs.join("\n")}\n`; const script = [ "set +e", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", - "cat >\"$tmpdir/compact-native-object.mjs\" <<'NODE_COMPACT'", - "import { readFileSync } from 'node:fs';", - "const key = process.argv[2] || '';", - "const input = JSON.parse(readFileSync(0, 'utf8'));", - "const cleanMap = (value) => {", - " if (!value || typeof value !== 'object' || Array.isArray(value)) return {};", - " const out = {};", - " for (const [k, v] of Object.entries(value)) {", - " if (k === 'kubectl.kubernetes.io/last-applied-configuration') continue;", - " out[k] = v;", - " }", - " return out;", - "};", - "const metadata = (obj) => ({", - " name: obj?.metadata?.name || null,", - " namespace: obj?.metadata?.namespace || null,", - " labels: cleanMap(obj?.metadata?.labels),", - " annotations: cleanMap(obj?.metadata?.annotations),", - "});", - "const compactContainer = (container) => ({", - " name: container?.name || null,", - " image: container?.image || null,", - " env: Array.isArray(container?.env) ? container.env.filter((item) => item && typeof item.name === 'string' && typeof item.value === 'string').map((item) => ({ name: item.name, value: item.value })) : [],", - "});", - "let output = input;", - "if (key === 'pipelineRun') {", - " output = {", - " apiVersion: input.apiVersion,", - " kind: input.kind,", - " metadata: metadata(input),", - " spec: { params: Array.isArray(input?.spec?.params) ? input.spec.params : [] },", - " status: { conditions: Array.isArray(input?.status?.conditions) ? input.status.conditions : [], startTime: input?.status?.startTime || null, completionTime: input?.status?.completionTime || null },", - " };", - "} else if (key === 'argoApplication') {", - " output = {", - " apiVersion: input.apiVersion,", - " kind: input.kind,", - " metadata: metadata(input),", - " status: { sync: input?.status?.sync || null, health: input?.status?.health || null, operationState: input?.status?.operationState ? { phase: input.status.operationState.phase || null, message: input.status.operationState.message || null, finishedAt: input.status.operationState.finishedAt || null } : null },", - " };", - "} else if (/^workload\\d+$/.test(key)) {", - " const template = input?.spec?.template || {};", - " output = {", - " apiVersion: input.apiVersion,", - " kind: input.kind,", - " metadata: metadata(input),", - " spec: { replicas: input?.spec?.replicas ?? null, template: { metadata: { labels: cleanMap(template?.metadata?.labels), annotations: cleanMap(template?.metadata?.annotations) }, spec: { containers: Array.isArray(template?.spec?.containers) ? template.spec.containers.map(compactContainer) : [] } } },", - " status: { replicas: input?.status?.replicas ?? null, readyReplicas: input?.status?.readyReplicas ?? null, availableReplicas: input?.status?.availableReplicas ?? null, updatedReplicas: input?.status?.updatedReplicas ?? null, conditions: Array.isArray(input?.status?.conditions) ? input.status.conditions.map((item) => ({ type: item.type || null, status: item.status || null, reason: item.reason || null })) : [] },", - " };", - "}", - "console.log(JSON.stringify(output));", - "NODE_COMPACT", - "cat >\"$tmpdir/kube-get.mjs\" <<'NODE_KUBE_GET'", - "import { readFileSync } from 'node:fs';", - "import https from 'node:https';", - "const path = process.argv[2];", - "const host = process.env.KUBERNETES_SERVICE_HOST;", - "const port = Number(process.env.KUBERNETES_SERVICE_PORT || '443');", - "const token = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/token', 'utf8').trim();", - "const ca = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt');", - "const req = https.request({ host, port, path, method: 'GET', ca, headers: { authorization: `Bearer ${token}` } }, (res) => {", - " let body = '';", - " res.setEncoding('utf8');", - " res.on('data', (chunk) => { body += chunk; });", - " res.on('end', () => {", - " if ((res.statusCode || 0) >= 200 && (res.statusCode || 0) < 300) { process.stdout.write(body); process.exit(0); }", - " process.stderr.write(body || `kube api status ${res.statusCode}`);", - " process.exit(1);", - " });", - "});", - "req.on('error', (error) => { process.stderr.write(error?.message || String(error)); process.exit(1); });", - "req.end();", - "NODE_KUBE_GET", - "emit_kube_json() {", - " key=\"$1\"", - " path=\"$2\"", - " raw=\"$tmpdir/$key.raw\"", - " out=\"$tmpdir/$key.out\"", - " err=\"$tmpdir/$key.err\"", - " if node \"$tmpdir/kube-get.mjs\" \"$path\" >\"$raw\" 2>\"$err\" && node \"$tmpdir/compact-native-object.mjs\" \"$key\" <\"$raw\" >\"$out\" 2>>\"$err\"; then", - " printf 'UNIDESK_NATIVE_JSON\\t%s\\t' \"$key\"", - " base64 \"$out\" | tr -d '\\n'", - " printf '\\n'", - " else", - " printf 'UNIDESK_NATIVE_ERROR\\t%s\\t' \"$key\"", - " base64 \"$err\" | tr -d '\\n'", - " printf '\\n'", - " fi", - "}", - `repo_path=${shQuote(source.repoPath)}`, - `branch=${shQuote(follower.source.branch)}`, - `repository=${shQuote(follower.source.repository)}`, - `snapshot_prefix=${shQuote(follower.source.snapshotPrefix)}`, - `read_url=${shQuote(source.gitMirrorReadUrl)}`, - `mirror_ns=${shQuote(source.gitMirrorNamespace)}`, - `mirror_deploy=${shQuote(source.gitMirrorDeployment)}`, - "source_commit=", - "source_err=\"$tmpdir/source.err\"", - "if [ -x /etc/unidesk-cicd-branch-follower/sync-source.sh ]; then", - " /etc/unidesk-cicd-branch-follower/sync-source.sh \"$repository\" \"$branch\" \"$snapshot_prefix\" \"$repo_path\" >/dev/null 2>\"$source_err\" || true", - "fi", - "if [ -d \"$repo_path/objects\" ]; then", - " source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$branch^{commit}\" 2>>\"$source_err\" | head -n 1 | tr -d '\\r' || true)", - "else", - " printf 'formal controller/job must mount k8s git-mirror cache at %s; fallback exec is disabled\\n' \"$repo_path\" >>\"$source_err\"", - "fi", - "case \"$source_commit\" in", - " [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f])", - " stage_ref=\"${snapshot_prefix%/}/$source_commit\"", - " printf 'UNIDESK_NATIVE_JSON\\tsource\\t'", - " printf '{\"commit\":\"%s\",\"branch\":\"%s\",\"stageRef\":\"%s\",\"sourceAuthority\":\"k8s-git-mirror-snapshot\",\"mode\":\"k8s-git-mirror-cache\",\"repoPath\":\"%s\"}' \"$source_commit\" \"$branch\" \"$stage_ref\" \"$repo_path\" | base64 | tr -d '\\n'", - " printf '\\n'", - " ;;", - " *)", - " printf 'UNIDESK_NATIVE_ERROR\\tsource\\t'", - " base64 \"$source_err\" | tr -d '\\n'", - " printf '\\n'", - " ;;", - "esac", - tekton === null - ? "true" - : [ - "if [ -n \"$source_commit\" ]; then", - " sha12=$(printf '%s' \"$source_commit\" | cut -c1-12)", - ` emit_kube_json pipelineRun ${shQuote(`/apis/tekton.dev/v1/namespaces/${tekton.namespace}/pipelineruns/${tekton.pipelineRunPrefix}-`)}"$sha12"`, - "fi", - ].join("\n"), - argo === null - ? "true" - : `emit_kube_json argoApplication ${shQuote(`/apis/argoproj.io/v1alpha1/namespaces/${argo.namespace}/applications/${argo.application}`)}`, - ...workloadCommands, - "exit 0", + nativeBundleScriptLoadShell(), + `REPO_PATH=${shQuote(source.repoPath)}`, + `SOURCE_BRANCH=${shQuote(follower.source.branch)}`, + `REPOSITORY=${shQuote(follower.source.repository)}`, + `SNAPSHOT_PREFIX=${shQuote(follower.source.snapshotPrefix)}`, + `GITOPS_BRANCH=${shQuote(gitopsBranch ?? "")}`, + `TEKTON_NAMESPACE=${shQuote(tekton?.namespace ?? "")}`, + `PIPELINE_RUN_PREFIX=${shQuote(tekton?.pipelineRunPrefix ?? "")}`, + `ARGO_NAMESPACE=${shQuote(argo?.namespace ?? "")}`, + `ARGO_APPLICATION=${shQuote(argo?.application ?? "")}`, + `WORKLOAD_REFS_B64=${shQuote(Buffer.from(workloadRefsText, "utf8").toString("base64"))}`, + "NATIVE_CICD_SCRIPT_DIR=\"$tmpdir\"", + "export NATIVE_CICD_SCRIPT_DIR REPO_PATH SOURCE_BRANCH REPOSITORY SNAPSHOT_PREFIX GITOPS_BRANCH TEKTON_NAMESPACE PIPELINE_RUN_PREFIX ARGO_NAMESPACE ARGO_APPLICATION WORKLOAD_REFS_B64", + "\"$tmpdir/read-native-bundle.sh\"", ].join("\n"); const result = runKubeScript(registry, options, script, "", Math.max(5, timeoutSeconds) * 1000); const parsed = parseNativeBundleLines(result.stdout); @@ -2076,7 +1458,10 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll return { ok: result.exitCode === 0 && sourceRecord !== null && parsed.fatalErrors.length === 0, source: sourceRecord, + gitMirror: asOptionalRecord(parsed.objects.gitMirror), pipelineRun: asOptionalRecord(parsed.objects.pipelineRun), + taskRuns: asOptionalRecord(parsed.objects.taskRuns), + planArtifacts: asOptionalRecord(parsed.objects.planArtifacts), argoApplication: asOptionalRecord(parsed.objects.argoApplication), workloads: Object.entries(parsed.objects) .filter(([key]) => /^workload\d+$/u.test(key)) @@ -2095,6 +1480,24 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll }; } +const NATIVE_BUNDLE_SCRIPT_NAMES = [ + "read-native-bundle.sh", + "kube-get.mjs", + "compact-native-object.mjs", + "compact-git-mirror.mjs", + "plan-artifacts.mjs", +] as const; + +function nativeBundleScriptLoadShell(): string { + return NATIVE_BUNDLE_SCRIPT_NAMES.map((name) => { + const encoded = Buffer.from(readFileSync(rootPath("scripts/native/cicd", name), "utf8"), "utf8").toString("base64"); + return [ + `printf '%s' ${shQuote(encoded)} | base64 -d > "$tmpdir/${name}"`, + `chmod +x "$tmpdir/${name}"`, + ].join("\n"); + }).join("\n"); +} + function parseNativeBundleLines(stdout: string): { objects: Record; errors: string[]; fatalErrors: string[] } { const objects: Record = {}; const errors: string[] = []; @@ -2266,6 +1669,49 @@ function nativeRuntimeSummary(runtime: NativeStatusSpec["runtime"], workloads: R }; } +function nativeGitMirrorGitopsBranch(follower: FollowerSpec): string | null { + if (follower.adapter === "hwlab-node-runtime") { + return hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node).gitopsBranch; + } + if (follower.adapter === "agentrun-yaml-lane") { + return resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec.gitops.branch; + } + return null; +} + +function nativeGitMirrorRequired(follower: FollowerSpec): boolean { + return nativeGitMirrorGitopsBranch(follower) !== null + && (follower.closeoutChecks.includes("gitMirrorPostFlush") || follower.closeoutChecks.includes("gitops")); +} + +function nativeGitMirrorReady(gitMirror: Record | null): boolean { + if (gitMirror === null) return false; + if (gitMirror.ok === false) return false; + if (gitMirror.sourceSnapshotReady === false) return false; + if (gitMirror.pendingFlush === true) return false; + if (gitMirror.githubInSync === false) return false; + return true; +} + +function nativeGitMirrorSummary(gitMirror: Record | null): Record | null { + if (gitMirror === null) return null; + return { + ok: gitMirror.ok === true, + repository: stringOrNull(gitMirror.repository), + sourceBranch: stringOrNull(gitMirror.sourceBranch), + gitopsBranch: stringOrNull(gitMirror.gitopsBranch), + localSource: stringOrNull(gitMirror.localSource), + githubSource: stringOrNull(gitMirror.githubSource), + sourceStageRef: stringOrNull(gitMirror.sourceStageRef), + sourceSnapshotReady: gitMirror.sourceSnapshotReady === true, + localGitops: stringOrNull(gitMirror.localGitops), + githubGitops: stringOrNull(gitMirror.githubGitops), + pendingFlush: gitMirror.pendingFlush === true, + githubInSync: gitMirror.githubInSync === true, + statusAuthority: stringOrNull(gitMirror.statusAuthority) ?? "k8s-git-mirror-cache", + }; +} + function latestCondition(status: Record | null, type: string): Record | null { const conditions = Array.isArray(status?.conditions) ? status.conditions : []; for (const condition of conditions as unknown[]) { @@ -2552,7 +1998,10 @@ function compactNativePayload(payload: Record | null): Record | null): Record | null): Record | null { + if (taskRuns === null) return null; + return { + ok: taskRuns.ok === true, + count: numberOrNull(taskRuns.count), + succeededCount: numberOrNull(taskRuns.succeededCount), + failedCount: numberOrNull(taskRuns.failedCount), + activeCount: numberOrNull(taskRuns.activeCount), + performance: asOptionalRecord(taskRuns.performance), + items: Array.isArray(taskRuns.items) ? taskRuns.items.slice(0, 16) : [], + }; +} + +function compactPlanArtifactsPayload(planArtifacts: Record | null): Record | null { + if (planArtifacts === null) return null; + return { + ok: planArtifacts.ok === true, + pipelineRun: stringOrNull(planArtifacts.pipelineRun), + eventFound: planArtifacts.eventFound === true, + degradedReason: stringOrNull(planArtifacts.degradedReason), + sourceCommitId: stringOrNull(planArtifacts.sourceCommitId), + affectedServices: Array.isArray(planArtifacts.affectedServices) ? planArtifacts.affectedServices.slice(0, 40) : [], + rolloutServices: Array.isArray(planArtifacts.rolloutServices) ? planArtifacts.rolloutServices.slice(0, 40) : [], + buildServices: Array.isArray(planArtifacts.buildServices) ? planArtifacts.buildServices.slice(0, 40) : [], + reusedServices: Array.isArray(planArtifacts.reusedServices) ? planArtifacts.reusedServices.slice(0, 40) : [], + buildSkippedCount: numberOrNull(planArtifacts.buildSkippedCount), + summary: stringOrNull(planArtifacts.summary), + disclosure: stringOrNull(planArtifacts.disclosure), + }; +} + function compactSourcePayload(source: Record | null): Record | null { if (source === null) return null; return { @@ -2668,335 +2148,6 @@ function runControllerReconcileJob(registry: BranchFollowerRegistry, options: Pa }; } -function renderControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, jobName: string, mode: { dryRun: boolean; recordState: boolean }, timeoutSeconds: number): Record { - const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-reconcile-job" }; - const commandArgs = [ - "bun", - "scripts/cli.ts", - "cicd", - "branch-follower", - "run-once", - ...(options.followerId === null ? ["--all"] : ["--follower", options.followerId]), - mode.dryRun ? "--dry-run" : "--confirm", - "--wait", - "--controller", - "--config", - "config/cicd-branch-followers.yaml", - "--timeout-seconds", - String(timeoutSeconds), - ...(mode.recordState ? ["--record-state"] : []), - ]; - return { - apiVersion: "batch/v1", - kind: "Job", - metadata: { name: jobName, namespace: registry.controller.namespace, labels }, - spec: { - backoffLimit: 0, - ttlSecondsAfterFinished: 600, - activeDeadlineSeconds: timeoutSeconds + 30, - template: { - metadata: { labels }, - spec: { - restartPolicy: "Never", - serviceAccountName: registry.controller.serviceAccountName, - volumes: [ - { name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } }, - { name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } }, - { name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } }, - { name: "work", emptyDir: {} }, - ], - containers: [ - { - name: "reconcile", - image: registry.controller.image, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-lc"], - args: [controllerOneShotScript(commandArgs)], - env: [ - { name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch }, - { name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository }, - { name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) }, - ], - volumeMounts: [ - { name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true }, - { name: "git-mirror-cache", mountPath: "/cache" }, - { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - { name: "work", mountPath: "/work" }, - ], - }, - ], - }, - }, - }, - }; -} - -function controllerOneShotScript(commandArgs: string[]): string { - return [ - "set -eu", - "cd /work", - "rm -rf /work/unidesk", - "started_at=$(date -Iseconds)", - "echo \"branch-follower one-shot started ${started_at}\"", - "/etc/unidesk-cicd-branch-follower/sync-source.sh \"${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}\" \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"${UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\"", - "git clone --branch \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\" /work/unidesk", - "cp /etc/unidesk-cicd-branch-follower/cicd-branch-followers.yaml /work/unidesk/config/cicd-branch-followers.yaml", - "cd /work/unidesk", - `${commandArgs.map(shQuote).join(" ")}`, - "echo \"branch-follower one-shot finished $(date -Iseconds)\"", - ].join("\n"); -} - -function waitForJobShell(namespace: string, jobName: string, timeoutSeconds: number): string { - return [ - `deadline=$(( $(date +%s) + ${timeoutSeconds} ))`, - "while true; do", - ` job_json=$(kubectl -n ${shQuote(namespace)} get job ${shQuote(jobName)} -o json)`, - " phase=$(printf '%s' \"$job_json\" | node -e \"let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>{const j=JSON.parse(s);const c=j.status?.conditions||[];const done=c.find(x=>x.type==='Complete'&&x.status==='True');const failed=c.find(x=>x.type==='Failed'&&x.status==='True');process.stdout.write(done?'complete':failed?'failed':'running');})\")", - " if [ \"$phase\" = complete ]; then exit 0; fi", - " if [ \"$phase\" = failed ]; then exit 1; fi", - " if [ \"$(date +%s)\" -ge \"$deadline\" ]; then exit 124; fi", - " sleep 2", - "done", - ].join("\n"); -} - -function renderControllerManifests(registry: BranchFollowerRegistry): Record[] { - const labels = registry.controller.labels; - const selector = labels; - return [ - { - apiVersion: "v1", - kind: "Namespace", - metadata: { name: registry.controller.namespace, labels }, - }, - { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels }, - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "Role", - metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels }, - rules: [ - { apiGroups: [""], resources: ["configmaps", "pods", "events"], verbs: ["get", "list", "watch", "create", "update", "patch"] }, - { apiGroups: ["apps"], resources: ["deployments"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] }, - { apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["get", "list", "watch", "create", "update", "patch"] }, - ], - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "RoleBinding", - metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels }, - subjects: [{ kind: "ServiceAccount", name: registry.controller.serviceAccountName, namespace: registry.controller.namespace }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: registry.controller.serviceAccountName }, - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "ClusterRole", - metadata: { name: registry.controller.serviceAccountName, labels }, - rules: [ - { apiGroups: [""], resources: ["pods", "pods/log", "configmaps", "events"], verbs: ["get", "list", "watch"] }, - { apiGroups: [""], resources: ["pods/exec"], verbs: ["create"] }, - { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] }, - { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch", "create", "patch", "delete"] }, - { apiGroups: ["tekton.dev"], resources: ["taskruns"], verbs: ["get", "list", "watch"] }, - { apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch", "patch"] }, - ], - }, - { - apiVersion: "rbac.authorization.k8s.io/v1", - kind: "ClusterRoleBinding", - metadata: { name: registry.controller.serviceAccountName, labels }, - subjects: [{ kind: "ServiceAccount", name: registry.controller.serviceAccountName, namespace: registry.controller.namespace }], - roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: registry.controller.serviceAccountName }, - }, - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: registry.controller.configMapName, namespace: registry.controller.namespace, labels }, - data: { - "cicd-branch-followers.yaml": registry.rawText, - "sync-source.sh": nativeGitMirrorSyncShell(registry), - }, - }, - { - apiVersion: "v1", - kind: "ConfigMap", - metadata: { name: registry.controller.stateConfigMapName, namespace: registry.controller.namespace, labels }, - data: { - _createdAt: new Date().toISOString(), - _specRef: SPEC_REF, - _registrySha256: registry.rawSha256, - }, - }, - { - apiVersion: "coordination.k8s.io/v1", - kind: "Lease", - metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels }, - spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) }, - }, - { - apiVersion: "apps/v1", - kind: "Deployment", - metadata: { name: registry.controller.deploymentName, namespace: registry.controller.namespace, labels }, - spec: { - replicas: 1, - selector: { matchLabels: selector }, - template: { - metadata: { - labels: selector, - annotations: { - "unidesk.pikapython.com/spec-ref": SPEC_REF, - "unidesk.pikapython.com/registry-sha256": registry.rawSha256, - "unidesk.pikapython.com/host-worktree-authority": "false", - }, - }, - spec: { - serviceAccountName: registry.controller.serviceAccountName, - terminationGracePeriodSeconds: 30, - volumes: [ - { name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } }, - { name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } }, - { name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } }, - { name: "work", emptyDir: {} }, - ], - containers: [ - { - name: "controller", - image: registry.controller.image, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-lc"], - args: [controllerLoopScript()], - env: [ - { name: "UNIDESK_CICD_BRANCH_FOLLOWER_INTERVAL_SECONDS", value: String(registry.controller.loop.intervalSeconds) }, - { name: "UNIDESK_CICD_BRANCH_FOLLOWER_TIMEOUT_SECONDS", value: String(registry.controller.loop.reconcileTimeoutSeconds) }, - { name: "UNIDESK_CONTROLLER_GIT_MIRROR_READ_URL", value: registry.controller.source.gitMirrorReadUrl }, - { name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch }, - { name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository }, - { name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) }, - ], - volumeMounts: [ - { name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true }, - { name: "git-mirror-cache", mountPath: "/cache" }, - { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - { name: "work", mountPath: "/work" }, - ], - }, - ], - }, - }, - }, - }, - ]; -} - -function controllerLoopScript(): string { - return [ - "set -eu", - "interval=\"${UNIDESK_CICD_BRANCH_FOLLOWER_INTERVAL_SECONDS}\"", - "timeout=\"${UNIDESK_CICD_BRANCH_FOLLOWER_TIMEOUT_SECONDS}\"", - "while true; do", - " started_at=$(date -Iseconds)", - " echo \"branch-follower loop started ${started_at}\"", - " cd /work", - " rm -rf /work/unidesk", - " /etc/unidesk-cicd-branch-follower/sync-source.sh \"${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}\" \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"${UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\"", - " git clone --branch \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\" /work/unidesk", - " cp /etc/unidesk-cicd-branch-follower/cicd-branch-followers.yaml /work/unidesk/config/cicd-branch-followers.yaml", - " cd /work/unidesk", - " bun scripts/cli.ts cicd branch-follower run-once --all --confirm --controller --config config/cicd-branch-followers.yaml --timeout-seconds \"${timeout}\" || true", - " echo \"branch-follower loop finished $(date -Iseconds)\"", - " cd /work", - " sleep \"${interval}\"", - "done", - ].join("\n"); -} - -function nativeGitMirrorSyncShell(registry: BranchFollowerRegistry): string { - const ssh = registry.controller.source.githubSsh; - return [ - "#!/bin/sh", - "set -eu", - "repository=\"$1\"", - "branch=\"$2\"", - "snapshot_prefix=\"$3\"", - "repo_path=\"$4\"", - `private_key=${shQuote(`/git-ssh/${ssh.privateKeySecretKey}`)}`, - `proxy_host=${shQuote(ssh.proxyHost)}`, - `proxy_port=${shQuote(String(ssh.proxyPort))}`, - "mkdir -p \"$(dirname \"$repo_path\")\" /root/.ssh", - "cp \"$private_key\" /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - "touch /root/.ssh/known_hosts", - "cat >/tmp/unidesk-cicd-github-proxy-connect.mjs <<'NODE_PROXY'", - "#!/usr/bin/env node", - "import net from 'node:net';", - "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", - "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", - "const targetPort = Number.parseInt(targetPortRaw || '', 10);", - "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);", - "let settled = false;", - "let tunnel = false;", - "function finish(code) { if (settled) return; settled = true; process.exit(code); }", - "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", - "let buffer = Buffer.alloc(0);", - "socket.setTimeout(15000, () => { socket.destroy(); finish(65); });", - "socket.on('connect', () => socket.write(`CONNECT ${targetHost}:${targetPort} HTTP/1.1\\r\\nHost: ${targetHost}:${targetPort}\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n`));", - "socket.on('error', () => finish(tunnel ? 69 : 66));", - "socket.on('close', () => finish(tunnel ? 0 : 68));", - "socket.on('data', function onData(chunk) {", - " buffer = Buffer.concat([buffer, chunk]);", - " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", - " if (headerEnd === -1 && buffer.length < 8192) return;", - " if (headerEnd === -1) { socket.destroy(); finish(68); return; }", - " const statusLine = buffer.slice(0, headerEnd).toString('latin1').split('\\r\\n', 1)[0] || '';", - " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", - " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); finish(67); return; }", - " socket.off('data', onData);", - " socket.setTimeout(0);", - " tunnel = true;", - " const rest = buffer.slice(headerEnd + 4);", - " if (rest.length) process.stdout.write(rest);", - " process.stdin.on('error', () => {});", - " process.stdout.on('error', () => {});", - " process.stdin.pipe(socket);", - " socket.pipe(process.stdout);", - "});", - "NODE_PROXY", - "chmod 0700 /tmp/unidesk-cicd-github-proxy-connect.mjs", - "cat >/tmp/unidesk-cicd-git-ssh.sh </dev/null", - " git --git-dir=\"$repo_path\" remote add origin \"$remote\"", - "fi", - "git --git-dir=\"$repo_path\" config uploadpack.allowReachableSHA1InWant true", - "git --git-dir=\"$repo_path\" config uploadpack.allowAnySHA1InWant true", - "timeout 30 git --git-dir=\"$repo_path\" fetch --quiet --prune origin \"+refs/heads/${branch}:refs/mirror-stage/heads/${branch}\"", - "source_sha=$(git --git-dir=\"$repo_path\" rev-parse --verify \"refs/mirror-stage/heads/${branch}^{commit}\")", - "git --git-dir=\"$repo_path\" update-ref \"refs/heads/${branch}\" \"$source_sha\"", - "if [ -n \"$snapshot_prefix\" ]; then", - " git --git-dir=\"$repo_path\" update-ref \"${snapshot_prefix%/}/$source_sha\" \"$source_sha\"", - "fi", - "git --git-dir=\"$repo_path\" update-server-info", - "printf '{\"event\":\"unidesk-cicd-git-mirror-sync\",\"repository\":\"%s\",\"branch\":\"%s\",\"commit\":\"%s\",\"sourceAuthority\":\"k8s-git-mirror-cache\"}\\n' \"$repository\" \"$branch\" \"$source_sha\"", - "", - ].join("\n"); -} - function selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[] { let selected = registry.followers; if (options.followerId !== null) selected = selected.filter((item) => item.id === options.followerId); @@ -3175,6 +2326,16 @@ function safeJobSegment(value: string): string { return value.replace(/[^A-Za-z0-9_.-]/gu, "_").slice(0, 60); } +function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { + const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`; + return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); +} + +function safeK8sNameSegment(value: string): string { + const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, ""); + return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, ""); +} + function tailText(text: string, maxChars: number): string { if (text.length <= maxChars) return text; return text.slice(text.length - maxChars); diff --git a/scripts/src/gh/help.ts b/scripts/src/gh/help.ts index b3850d0e..3f27cd44 100644 --- a/scripts/src/gh/help.ts +++ b/scripts/src/gh/help.ts @@ -60,7 +60,7 @@ export function ghHelp(): unknown { "bun scripts/cli.ts gh pr comment edit (--body-stdin|--body-file |--body ) [--repo owner/name] [--number N compat] [--dry-run] [compatibility alias for pr comment update]", "bun scripts/cli.ts gh pr comment delete [--repo owner/name] [--number N compat] [--dry-run]", "bun scripts/cli.ts gh pr close|reopen [--repo owner/name] [--number N compat] [--dry-run]", - "bun scripts/cli.ts gh pr merge [--repo owner/name] [--number N compat] [--merge|--squash|--rebase] [--delete-branch] [--dry-run]", + "bun scripts/cli.ts gh pr merge [--repo owner/name] [--number N compat] [--merge|--squash|--rebase] [--delete-branch|--keep-branch] [--sync-node NODE]... [--skip-local-closeout] [--dry-run]", "bun scripts/cli.ts gh pr delete [unsupported: use close]", ], defaults: { repo: DEFAULT_REPO }, @@ -105,7 +105,7 @@ export function ghHelp(): unknown { "PR preflight/closeout accept the same owner/repo#number shorthand as PR view/read so merge readiness checks do not require repeating --repo after a PR URL has already been normalized.", "PR list does not fetch mergeability or statusCheckRollup; request those closeout fields with gh pr view --json headRefName,baseRefName,mergeable,mergeStateStatus,statusCheckRollup.", "PR preflight is a low-noise read-only closeout helper for explicit diagnosis only. It combines redacted auth capability, PR branch/state metadata, mergeability, mergeStateStatus, compact status check counts, and an explicit read-only policy. It is not a required step before gh pr merge. Use --full or --raw to include all fetched status contexts.", - "PR merge is the one-command guarded write path: it reads closeout metadata itself, retries GitHub UNKNOWN/null mergeability with YAML-configured exponential backoff, refuses non-open/draft/conflicting/non-clean/failed/pending PRs, then uses GitHub REST merge. Use --dry-run to see the exact merge plan without writing.", + "PR merge is the one-command guarded write path: it reads closeout metadata itself, retries GitHub UNKNOWN/null mergeability with YAML-configured exponential backoff, refuses non-open/draft/conflicting/non-clean/failed/pending PRs, then uses GitHub REST merge. It defaults to deleting the merged same-repo head branch, cleaning the matching local .worktree when clean, and fast-forwarding the local main worktree on the PR base branch; --sync-node NODE additionally runs mapped node source-workspace sync. Use --dry-run to see the exact merge and closeout plan without writing.", ], }; } @@ -168,6 +168,7 @@ export function ghScopedHelpNotes(tokens: string[]): string[] { } else if (key === "pr merge") { notes.push("PR merge is one-command guarded: it performs the readiness check itself; `gh pr preflight` is optional read-only diagnosis, not a required first step."); notes.push("When GitHub reports mergeability as UNKNOWN/null, merge automatically retries with YAML-configured exponential backoff and shows retry attempts as N/M."); + notes.push("After merge it deletes the same-repo head branch by default, removes a clean matching local `.worktree`, and fast-forwards the local main worktree; use `--keep-branch` or `--skip-local-closeout` only when intentionally preserving state."); } else if (key === "pr review-plan" || key === "pr diff") { notes.push("Use `pr review-plan` first for a bounded changed-file index with per-file drill-down commands."); notes.push("Use `pr diff --file [--hunk N]` for bounded patch review; full patch disclosure requires explicit --full or --raw."); diff --git a/scripts/src/gh/options.ts b/scripts/src/gh/options.ts index 8d2a4b61..8367777e 100644 --- a/scripts/src/gh/options.ts +++ b/scripts/src/gh/options.ts @@ -321,6 +321,9 @@ export function stdinAliasFileOption(args: string[], fileOption: string, stdinFl export function parseOptions(args: string[]): GitHubOptions { validateKnownOptions(args); const [top, sub] = args; + if (hasFlag(args, "--delete-branch") && hasFlag(args, "--keep-branch")) { + throw new Error("gh pr merge accepts either --delete-branch or --keep-branch, not both"); + } const requestedJsonFields = commaListOption(args, "--json"); const limitMax = top === "pr" && (sub === "files" || sub === "diff") ? MAX_PR_FILES_LIMIT @@ -372,7 +375,9 @@ export function parseOptions(args: string[]): GitHubOptions { boardGithubStatus: parseBoardGithubStatus(args), boardRowUpsertValues: parseBoardRowUpsertValues(args), mergeMethod: parsePullRequestMergeMethod(args), - deleteBranch: hasFlag(args, "--delete-branch"), + deleteBranch: top === "pr" && sub === "merge" ? !hasFlag(args, "--keep-branch") : hasFlag(args, "--delete-branch"), + localCloseout: !hasFlag(args, "--skip-local-closeout"), + syncNodes: optionValues(args, "--sync-node").map((value) => value.trim()).filter((value) => value.length > 0), attachmentSelector: optionValue(args, "--attachment"), outputPath: optionValue(args, "--output"), filePath: optionValue(args, "--file"), diff --git a/scripts/src/gh/pr-merge-closeout.ts b/scripts/src/gh/pr-merge-closeout.ts new file mode 100644 index 00000000..9420c029 --- /dev/null +++ b/scripts/src/gh/pr-merge-closeout.ts @@ -0,0 +1,170 @@ +// SPEC: PJ2026-01060703 GitHub PR merge closeout. +// Responsibility: guarded local/node worktree closeout after a successful PR merge. + +import { relative, resolve, sep } from "node:path"; +import { repoRoot } from "../config"; +import { runCommand, type CommandResult } from "../command"; +import type { GitHubOptions, GitHubPullRequest } from "./types"; + +interface WorktreeEntry { + path: string; + branch: string | null; + head: string | null; +} + +export function runPrMergeCloseout(repo: string, pr: GitHubPullRequest, options: GitHubOptions): Record { + const headRef = pr.head?.ref ?? null; + const baseRef = pr.base?.ref ?? null; + const localEnabled = options.localCloseout; + return { + ok: true, + valuesPrinted: false, + headRef, + baseRef, + localWorktree: localEnabled ? cleanupHeadWorktree(headRef, options.dryRun) : skipped("local-closeout-disabled"), + mainWorktree: localEnabled ? syncLocalMainWorktree(repo, baseRef, options.dryRun) : skipped("local-closeout-disabled"), + nodeSyncs: syncRequestedNodes(repo, baseRef, options.syncNodes, options.dryRun), + }; +} + +function cleanupHeadWorktree(headRef: string | null, dryRun: boolean): Record { + if (headRef === null || headRef.length === 0) return skipped("head-ref-missing"); + const list = git(["worktree", "list", "--porcelain"], 10_000); + if (list.exitCode !== 0) return failed("worktree-list-failed", list); + const entries = parseWorktreeList(list.stdout); + const matches = entries.filter((entry) => entry.branch === `refs/heads/${headRef}` && isManagedTaskWorktree(entry.path)); + if (matches.length === 0) return skipped("local-head-worktree-not-found", { headRef }); + const removals = matches.map((entry) => removeWorktree(entry.path, dryRun)); + const branchDelete = deleteLocalBranch(headRef, dryRun); + return { + ok: removals.every((item) => item.ok === true || item.planned === true) && (branchDelete.ok === true || branchDelete.planned === true || branchDelete.skipped === true), + headRef, + worktrees: removals, + branchDelete, + }; +} + +function removeWorktree(path: string, dryRun: boolean): Record { + const status = git(["-C", path, "status", "--porcelain"], 10_000); + if (status.exitCode !== 0) return failed("worktree-status-failed", status, { path }); + if (status.stdout.trim().length > 0) return skipped("worktree-dirty", { path, porcelainLines: status.stdout.trim().split(/\r?\n/u).slice(0, 20) }); + if (dryRun) return { planned: true, action: "git-worktree-remove", path }; + const removed = git(["worktree", "remove", path], 30_000); + return removed.exitCode === 0 ? { ok: true, action: "git-worktree-remove", path } : failed("worktree-remove-failed", removed, { path }); +} + +function deleteLocalBranch(headRef: string, dryRun: boolean): Record { + const list = git(["branch", "--list", headRef], 10_000); + if (list.exitCode !== 0) return failed("branch-list-failed", list, { headRef }); + if (list.stdout.trim().length === 0) return skipped("local-branch-not-found", { headRef }); + if (dryRun) return { planned: true, action: "git-branch-delete", branch: headRef }; + const deleted = git(["branch", "-d", headRef], 20_000); + return deleted.exitCode === 0 ? { ok: true, action: "git-branch-delete", branch: headRef } : failed("branch-delete-failed", deleted, { branch: headRef }); +} + +function syncLocalMainWorktree(repo: string, baseRef: string | null, dryRun: boolean): Record { + if (baseRef === null || baseRef.length === 0) return skipped("base-ref-missing"); + const remote = git(["remote", "get-url", "origin"], 10_000); + if (remote.exitCode !== 0) return failed("remote-read-failed", remote); + if (!remoteMatchesRepo(remote.stdout.trim(), repo)) return skipped("local-repo-mismatch", { repo, origin: redactRemote(remote.stdout.trim()) }); + const current = git(["rev-parse", "--abbrev-ref", "HEAD"], 10_000); + if (current.exitCode !== 0) return failed("current-branch-read-failed", current); + const currentBranch = current.stdout.trim(); + if (currentBranch !== baseRef) return skipped("main-worktree-on-different-branch", { currentBranch, baseRef }); + if (dryRun) return { planned: true, action: "git-fetch-merge-ff-only", path: repoRoot, branch: baseRef }; + + const status = git(["status", "--porcelain"], 10_000); + if (status.exitCode !== 0) return failed("main-worktree-status-failed", status); + const hadDirty = status.stdout.trim().length > 0; + const stash = hadDirty ? git(["stash", "push", "-u", "-m", `unidesk-gh-pr-merge-closeout ${repo} ${baseRef}`], 30_000) : null; + if (stash !== null && stash.exitCode !== 0) return failed("main-worktree-stash-failed", stash, { branch: baseRef }); + + const fetched = git(["fetch", "origin", baseRef], 60_000); + const merged = fetched.exitCode === 0 ? git(["merge", "--ff-only", `origin/${baseRef}`], 60_000) : null; + const applied = stash !== null ? git(["stash", "apply"], 60_000) : null; + const ok = fetched.exitCode === 0 && merged?.exitCode === 0 && (applied === null || applied.exitCode === 0); + return { + ok, + action: "git-fetch-merge-ff-only", + path: repoRoot, + branch: baseRef, + dirtyPreservedByStash: hadDirty, + fetch: commandSummary(fetched), + merge: merged === null ? null : commandSummary(merged), + stashApply: applied === null ? null : commandSummary(applied), + }; +} + +function syncRequestedNodes(repo: string, baseRef: string | null, nodes: string[], dryRun: boolean): Record[] { + return nodes.map((node) => syncRequestedNode(repo, baseRef, node, dryRun)); +} + +function syncRequestedNode(repo: string, baseRef: string | null, node: string, dryRun: boolean): Record { + const command = nodeSyncCommand(repo, baseRef, node); + if (command === null) return skipped("node-sync-unsupported", { repo, baseRef, node }); + if (dryRun) return { planned: true, node, command: command.join(" ") }; + const result = runCommand(command, repoRoot, { timeoutMs: 120_000 }); + return result.exitCode === 0 ? { ok: true, node, command: command.join(" ") } : failed("node-sync-failed", result, { node, command: command.join(" ") }); +} + +function nodeSyncCommand(repo: string, baseRef: string | null, node: string): string[] | null { + if (repo.toLowerCase() === "pikastech/hwlab" && baseRef === "v0.3") { + return ["bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "source-workspace", "sync", "--node", node.toUpperCase(), "--lane", "v03", "--confirm"]; + } + return null; +} + +function parseWorktreeList(text: string): WorktreeEntry[] { + const entries: WorktreeEntry[] = []; + let current: WorktreeEntry | null = null; + for (const line of text.split(/\r?\n/u)) { + if (line.startsWith("worktree ")) { + if (current !== null) entries.push(current); + current = { path: line.slice("worktree ".length), branch: null, head: null }; + } else if (line.startsWith("branch ") && current !== null) { + current.branch = line.slice("branch ".length); + } else if (line.startsWith("HEAD ") && current !== null) { + current.head = line.slice("HEAD ".length); + } + } + if (current !== null) entries.push(current); + return entries; +} + +function isManagedTaskWorktree(path: string): boolean { + const rel = relative(repoRoot, resolve(path)); + return rel === ".worktree" || rel.startsWith(`.worktree${sep}`); +} + +function git(args: string[], timeoutMs: number): CommandResult { + return runCommand(["git", ...args], repoRoot, { timeoutMs }); +} + +function remoteMatchesRepo(remote: string, repo: string): boolean { + const normalized = remote + .replace(/^git@github\.com:/u, "") + .replace(/^https:\/\/github\.com\//u, "") + .replace(/\.git$/u, "") + .toLowerCase(); + return normalized === repo.toLowerCase(); +} + +function redactRemote(remote: string): string { + return remote.replace(/:\/\/[^/@]+@/u, "://@"); +} + +function skipped(reason: string, details: Record = {}): Record { + return { ok: true, skipped: true, skippedReason: reason, ...details }; +} + +function failed(reason: string, result: CommandResult, details: Record = {}): Record { + return { ok: false, degradedReason: reason, ...details, command: result.command.join(" "), exitCode: result.exitCode, timedOut: result.timedOut, stderrTail: tail(result.stderr), stdoutTail: tail(result.stdout) }; +} + +function commandSummary(result: CommandResult): Record { + return { exitCode: result.exitCode, timedOut: result.timedOut, stderrTail: tail(result.stderr), stdoutTail: tail(result.stdout) }; +} + +function tail(text: string): string { + return text.trim().split(/\r?\n/u).slice(-8).join("\n").slice(-1000); +} diff --git a/scripts/src/gh/pr-merge.ts b/scripts/src/gh/pr-merge.ts index 25423bef..9bb36501 100644 --- a/scripts/src/gh/pr-merge.ts +++ b/scripts/src/gh/pr-merge.ts @@ -5,6 +5,7 @@ import { repoParts, resolveToken } from "./auth-and-safety"; import { authStatus } from "./auth-pr-read"; import { authRequired, commandError, errorPayload, githubRequest, isGitHubError, runnerDisposition, validationError } from "./client"; import { isRecord } from "./notify-claudeqq"; +import { runPrMergeCloseout } from "./pr-merge-closeout"; import { compactAuthCapability, deleteHeadBranchAfterMerge, loadPrMergeUnknownRetryConfig, mergeabilityHasUnknownPending, nextPrMergeRetryDelayMs, prCloseoutMetadata, prCloseoutSummary, preflightPullRequestSummary, prGraphqlMetadata, prMergeRetryCommand, prMetadataSummary, prPreflightPolicy, prSummary, sleepMs, statusRollupSummary } from "./pr-summary"; import { ghShort, ghTable, ghText } from "./render"; import { UNIDESK_CLI_CONFIG_PATH } from "./types"; @@ -22,15 +23,19 @@ export async function prMerge(repo: string, token: string, number: number, optio if (isGitHubError(pr)) return commandError("pr merge", repo, pr, { number, phase: "fetch-pr" }); const summary = prSummary(pr); if (summary.merged === true) { + const branchDeletion = options.deleteBranch && !options.dryRun ? await deleteHeadBranchAfterMerge(repo, token, pr) : { attempted: false, skippedReason: options.deleteBranch ? "dry-run" : "keep-branch-requested" }; + const closeout = runPrMergeCloseout(repo, pr, options); return withPrMergeRendered({ ok: true, command: "pr merge", repo, number, method: options.mergeMethod, + deleteBranch: options.deleteBranch, alreadyMerged: true, pullRequest: summary, - branchDeletion: { attempted: false, skippedReason: "already-merged" }, + branchDeletion, + closeout, rest: true, }); } @@ -97,6 +102,7 @@ export async function prMerge(repo: string, token: string, number: number, optio mergeability, statusChecks, retry, + closeout: runPrMergeCloseout(repo, pr, options), }); } const merged = await githubRequest>(token, "PUT", `/repos/${owner}/${name}/pulls/${number}/merge`, { @@ -105,19 +111,22 @@ export async function prMerge(repo: string, token: string, number: number, optio if (isGitHubError(merged)) return commandError("pr merge", repo, merged, { number, phase: "merge", method: options.mergeMethod, pullRequest: summary }); const after = await githubRequest(token, "GET", `/repos/${owner}/${name}/pulls/${number}`); if (isGitHubError(after)) return commandError("pr merge", repo, after, { number, phase: "fetch-after-merge", mergeResult: merged }); - const branchDeletion = options.deleteBranch ? await deleteHeadBranchAfterMerge(repo, token, after) : { attempted: false, skippedReason: "delete-branch-not-requested" }; + const branchDeletion = options.deleteBranch ? await deleteHeadBranchAfterMerge(repo, token, after) : { attempted: false, skippedReason: "keep-branch-requested" }; + const closeout = runPrMergeCloseout(repo, after, options); return withPrMergeRendered({ ok: true, command: "pr merge", repo, number, method: options.mergeMethod, + deleteBranch: options.deleteBranch, mergeResult: merged, pullRequest: prSummary(after), mergeability, statusChecks, retry, branchDeletion, + closeout, rest: true, }); } @@ -149,6 +158,10 @@ export function renderPrMergeTable(result: GitHubCommandResult, fallbackDetails? : {}; const counts = isRecord(statusChecks.counts) ? statusChecks.counts : {}; const branchDeletion = isRecord(result.branchDeletion) ? result.branchDeletion : {}; + const closeout = isRecord(result.closeout) ? result.closeout : {}; + const localWorktree = isRecord(closeout.localWorktree) ? closeout.localWorktree : {}; + const mainWorktree = isRecord(closeout.mainWorktree) ? closeout.mainWorktree : {}; + const nodeSyncs = Array.isArray(closeout.nodeSyncs) ? closeout.nodeSyncs.filter(isRecord) : []; const retry = isRecord(result.retry) ? result.retry : isRecord(details.retry) @@ -186,6 +199,7 @@ export function renderPrMergeTable(result: GitHubCommandResult, fallbackDetails? ` blockers=${blockers.length === 0 ? "-" : blockers.join(",")} pending=${pending.length === 0 ? "-" : pending.join(",")}`, ` retry=${retry.attempts !== undefined && retry.maxAttempts !== undefined ? `${ghText(retry.attempts)}/${ghText(retry.maxAttempts)} exhausted=${ghText(retry.exhausted)}` : "-"}`, ` branchDeletion=${ghText(branchDeletion.ok ?? branchDeletion.skippedReason ?? branchDeletion.attempted)}`, + ` closeout localWorktree=${closeoutCell(localWorktree)} mainWorktree=${closeoutCell(mainWorktree)} nodeSyncs=${nodeSyncs.length === 0 ? "-" : nodeSyncs.map(closeoutCell).join(",")}`, "", "Next:", ]; @@ -200,6 +214,14 @@ export function renderPrMergeTable(result: GitHubCommandResult, fallbackDetails? return lines.join("\n"); } +function closeoutCell(value: Record): string { + if (value.planned === true) return "planned"; + if (value.skipped === true) return `skipped:${ghText(value.skippedReason)}`; + if (value.ok === true) return "ok"; + if (value.ok === false) return `failed:${ghText(value.degradedReason)}`; + return "-"; +} + export async function prPreflight(repo: string, number: number, commandName: "preflight" | "pr preflight" | "pr closeout", includeRaw: boolean): Promise { const auth = await authStatus(repo); const authCapability = compactAuthCapability(auth); diff --git a/scripts/src/gh/pr-summary.ts b/scripts/src/gh/pr-summary.ts index 03533f1a..4f2a8b03 100644 --- a/scripts/src/gh/pr-summary.ts +++ b/scripts/src/gh/pr-summary.ts @@ -480,7 +480,7 @@ export function prMergeRetryCommand(repo: string, number: unknown, method: unkno "--repo", repo, prMergeMethodFlag(method), - deleteBranch === true ? "--delete-branch" : "", + deleteBranch === false ? "--keep-branch" : "--delete-branch", ].filter((item) => item.length > 0).join(" "); } diff --git a/scripts/src/gh/types.ts b/scripts/src/gh/types.ts index 588a04de..8f628068 100644 --- a/scripts/src/gh/types.ts +++ b/scripts/src/gh/types.ts @@ -94,10 +94,10 @@ export const GH_VALUE_OPTIONS = new Set([ "--value", "--section", "--to", "--status", "--row-file", "--category", "--branch", "--tasks", "--summary", "--focus", "--validation", "--progress", "--number", "--pr", "--search", "--title-prefix", "--inactive-hours", "--comment", "--comment-file", "--description", - "--attachment", "--output", "--file", "--hunk", + "--attachment", "--output", "--file", "--hunk", "--sync-node", ]); -export const GH_FLAG_OPTIONS = new Set(["--dry-run", "--draft", "--notify-claudeqq-brief-diff", "--allow-short-body", "--body-stdin", "--body-patch-stdin", "--comment-stdin", "--raw", "--full", "--stat", "--merge", "--squash", "--rebase", "--delete-branch", "--private", "--public", "--auto-init"]); +export const GH_FLAG_OPTIONS = new Set(["--dry-run", "--draft", "--notify-claudeqq-brief-diff", "--allow-short-body", "--body-stdin", "--body-patch-stdin", "--comment-stdin", "--raw", "--full", "--stat", "--merge", "--squash", "--rebase", "--delete-branch", "--keep-branch", "--skip-local-closeout", "--private", "--public", "--auto-init"]); export const MIN_SAFE_BODY_SCAN_CHARS = MIN_SAFE_ISSUE_BODY_CHARS; @@ -460,6 +460,8 @@ export interface GitHubOptions { boardRowUpsertValues: BoardRowUpsertValues; mergeMethod: PullRequestMergeMethod; deleteBranch: boolean; + localCloseout: boolean; + syncNodes: string[]; attachmentSelector?: string; outputPath?: string; filePath?: string;