diff --git a/config/unidesk-host-k8s.yaml b/config/unidesk-host-k8s.yaml index f7d28024..2d9a28ea 100644 --- a/config/unidesk-host-k8s.yaml +++ b/config/unidesk-host-k8s.yaml @@ -23,12 +23,20 @@ delivery: changeDetection: runtimePaths: - .tekton/unidesk-host-pac.yaml + - config/hwlab-node-lanes.yaml - config/platform-infra/pipelines-as-code.yaml - config/unidesk-host-k8s.yaml - scripts/native/cicd/build-unidesk-host-image.sh - scripts/native/cicd/prepare-unidesk-host-release.mjs - scripts/native/cicd/publish-unidesk-host-gitops.mjs - scripts/native/deploy/render-unidesk-host-service.mjs + - scripts/native/hwlab/runtime-gitops-observability.mjs + - scripts/native/hwlab/runtime-gitops-postprocess.mjs + - scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs + - scripts/native/hwlab/runtime-gitops-verify.mjs + - scripts/src/config.ts + - scripts/src/hwlab-node-lanes.ts + - scripts/src/yaml-composition.ts - src/components/microservices/todo-note/Dockerfile - src/components/microservices/todo-note/bun.lock - src/components/microservices/todo-note/package.json @@ -58,6 +66,12 @@ delivery: branch: unidesk-host-gitops manifestPath: deploy/gitops/unidesk-host/todo-note.yaml releaseStatePath: deploy/gitops-state/unidesk-host/todo-note.json + resources: + - id: hwlab-nc01-v03-runtime-gitops-scripts + renderer: hwlab-runtime-gitops-scripts + configRef: config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01 + manifestPath: deploy/gitops/unidesk-host/hwlab-nc01-v03-runtime-gitops-scripts.yaml + namespace: hwlab-ci author: name: UniDesk Host CI email: unidesk-host-ci@unidesk.local diff --git a/scripts/native/cicd/publish-unidesk-host-gitops.mjs b/scripts/native/cicd/publish-unidesk-host-gitops.mjs index 018a83d3..acb100b7 100755 --- a/scripts/native/cicd/publish-unidesk-host-gitops.mjs +++ b/scripts/native/cicd/publish-unidesk-host-gitops.mjs @@ -2,6 +2,8 @@ import { spawnSync } from "node:child_process"; import { mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; +import { runtimeGitopsScriptsConfigMap } from "../hwlab/runtime-gitops-scripts-configmap.mjs"; +import { hwlabRuntimeLaneSpecForNode } from "../../src/hwlab-node-lanes.ts"; function option(name) { const index = process.argv.indexOf(name); @@ -43,6 +45,47 @@ function safeReleaseStatePath(value) { return path; } +function safeResourceManifestPath(value, pathLabel) { + const path = required(value, pathLabel); + if (path.startsWith("/") || path.split("/").includes("..") || !path.endsWith(".yaml")) throw new Error(`${pathLabel} must be a safe relative .yaml path`); + return path; +} + +export function renderGitOpsResources(gitops, sourceRoot) { + if (gitops.resources === undefined) return []; + if (!Array.isArray(gitops.resources)) throw new Error("delivery.gitops.resources must be an array"); + return gitops.resources.map((value, index) => { + const pathLabel = `delivery.gitops.resources[${index}]`; + const resource = record(value, pathLabel); + const renderer = required(resource.renderer, `${pathLabel}.renderer`); + if (renderer !== "hwlab-runtime-gitops-scripts") throw new Error(`${pathLabel}.renderer is unsupported: ${renderer}`); + const configRef = required(resource.configRef, `${pathLabel}.configRef`); + const match = /^config\/hwlab-node-lanes\.yaml#lanes\.([^.]+)\.targets\.([^.]+)$/u.exec(configRef); + if (match === null) throw new Error(`${pathLabel}.configRef must select config/hwlab-node-lanes.yaml#lanes..targets.`); + const lane = match[1]; + const node = match[2]; + const spec = hwlabRuntimeLaneSpecForNode(lane, node); + const overlay = { + nodeId: spec.nodeId, + lane: spec.lane, + runtimePath: spec.runtimePath, + observability: spec.observability, + }; + const configMapName = `${spec.pipeline}-runtime-gitops-scripts`; + const manifest = Bun.YAML.stringify(runtimeGitopsScriptsConfigMap({ + overlay, + scriptsDir: resolve(sourceRoot, "scripts/native/hwlab"), + namespace: required(resource.namespace, `${pathLabel}.namespace`), + configMapName, + })); + return { + id: required(resource.id, `${pathLabel}.id`), + path: safeResourceManifestPath(resource.manifestPath, `${pathLabel}.manifestPath`), + content: `${manifest.trimEnd()}\n`, + }; + }); +} + function releaseValue(releaseDir, name) { return required(readFileSync(resolve(releaseDir, name), "utf8").trim(), `${releaseDir}/${name}`); } @@ -100,6 +143,7 @@ const writeUrl = required(gitops.writeUrl, "delivery.gitops.writeUrl"); const branch = required(gitops.branch, "delivery.gitops.branch"); const manifestPath = safeManifestPath(gitops.manifestPath); const releaseStatePath = safeReleaseStatePath(gitops.releaseStatePath); +const gitopsResources = action === "skip" ? [] : renderGitOpsResources(gitops, sourceRoot); let digest = null; let digestRef = null; let manifest = null; @@ -146,6 +190,26 @@ if (action === "skip") { digest = existingState.digest; digestRef = existingState.digestRef; runtimeSourceCommit = existingState.sourceCommit; + const gitopsCommit = run("git", ["rev-parse", "HEAD"], worktree).stdout.trim(); + process.stdout.write(`${JSON.stringify({ + ok: true, + phase: "gitops-publish", + action, + reason, + status: "skipped", + imageStatus: "skipped", + sourceCommit, + baselineSourceCommit, + runtimeSourceCommit, + digest, + digestRef, + gitopsCommit, + resources: [], + changed: false, + pushAttempts: 0, + valuesPrinted: false, + })}\n`); + return; } else if (action === "build" && manifest !== null) { mkdirSync(dirname(targetPath), { recursive: true }); writeFileSync(targetPath, manifest, "utf8"); @@ -163,12 +227,23 @@ if (action === "skip") { removeFile(statePath); } +for (const resource of gitopsResources) { + const resourcePath = resolve(worktree, resource.path); + if (!resourcePath.startsWith(`${worktree}/`)) throw new Error(`resolved resource path escaped the GitOps worktree: ${resource.id}`); + if (enabled) { + mkdirSync(dirname(resourcePath), { recursive: true }); + writeFileSync(resourcePath, resource.content, "utf8"); + } else { + removeFile(resourcePath); + } +} + run("git", ["config", "user.name", required(author.name, "delivery.gitops.author.name")], worktree); run("git", ["config", "user.email", required(author.email, "delivery.gitops.author.email")], worktree); run("git", ["add", "-A"], worktree); const changed = run("git", ["diff", "--cached", "--quiet"], worktree, true).status !== 0; if (changed) { - const commitAction = action === "build" ? "deploy" : "remove"; + const commitAction = action === "disabled" ? "remove" : "deploy"; run("git", ["commit", "-m", `unidesk-host: ${commitAction} ${serviceRef} ${sourceCommit.slice(0, 12)}`], worktree); } @@ -208,10 +283,11 @@ process.stdout.write(`${JSON.stringify({ digest, digestRef, gitopsCommit, + resources: gitopsResources.map((resource) => ({ id: resource.id, path: resource.path })), changed, pushAttempts, valuesPrinted: false, })}\n`); } -if (!process.execArgv.includes("--check")) main(); +if (import.meta.main && !process.execArgv.includes("--check")) main(); diff --git a/scripts/native/cicd/publish-unidesk-host-gitops.test.ts b/scripts/native/cicd/publish-unidesk-host-gitops.test.ts new file mode 100644 index 00000000..98c66a9d --- /dev/null +++ b/scripts/native/cicd/publish-unidesk-host-gitops.test.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { rootPath } from "../../src/config"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("UniDesk Host GitOps publisher", () => { + test("skip preserves the GitOps branch and every managed resource", () => { + const root = mkdtempSync(join(tmpdir(), "unidesk-host-gitops-skip-")); + temporaryRoots.push(root); + const seed = join(root, "seed"); + const bare = join(root, "gitops.git"); + const releaseDir = join(root, "release"); + const publisherWorktree = join(root, "publisher"); + const configPath = join(root, "config.yaml"); + const statePath = "deploy/gitops-state/unidesk-host/todo-note.json"; + const resourcePath = "deploy/gitops/unidesk-host/hwlab-runtime-gitops-scripts.yaml"; + const runtimeSourceCommit = "a".repeat(40); + const sourceCommit = "b".repeat(40); + const digest = `sha256:${"c".repeat(64)}`; + + mkdirSync(seed); + git(["init", "-b", "unidesk-host-gitops"], seed); + identity(seed); + write(seed, "deploy/gitops/unidesk-host/todo-note.yaml", "existing service manifest\n"); + write(seed, resourcePath, "stale configmap must remain unchanged\n"); + write(seed, statePath, `${JSON.stringify({ + version: 1, + kind: "UniDeskHostReleaseState", + serviceRef: "services.todoNote", + sourceCommit: runtimeSourceCommit, + digest, + digestRef: `registry.example/todo-note@${digest}`, + }, null, 2)}\n`); + git(["add", "."], seed); + git(["commit", "-m", "seed gitops"], seed); + const originalHead = git(["rev-parse", "HEAD"], seed).stdout.trim(); + git(["init", "--bare", bare], root); + git(["remote", "add", "origin", `file://${bare}`], seed); + git(["push", "origin", "unidesk-host-gitops"], seed); + + mkdirSync(releaseDir); + writeFileSync(join(releaseDir, "action"), "skip\n"); + writeFileSync(join(releaseDir, "reason"), "runtime-inputs-unchanged\n"); + writeFileSync(join(releaseDir, "baseline-source-commit"), `${runtimeSourceCommit}\n`); + writeFileSync(configPath, Bun.YAML.stringify({ + delivery: { + enabled: true, + serviceRef: "services.todoNote", + image: { repository: "registry.example/todo-note" }, + gitops: { + readUrl: `file://${bare}`, + writeUrl: `file://${bare}`, + branch: "unidesk-host-gitops", + manifestPath: "deploy/gitops/unidesk-host/todo-note.yaml", + releaseStatePath: statePath, + resources: [{ + id: "hwlab-runtime-gitops-scripts", + renderer: "hwlab-runtime-gitops-scripts", + configRef: "config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01", + manifestPath: resourcePath, + namespace: "hwlab-ci", + }], + author: { name: "UniDesk Test", email: "unidesk-test@example.invalid" }, + }, + }, + })); + + const result = spawnSync("bun", [ + "scripts/native/cicd/publish-unidesk-host-gitops.mjs", + "--config", configPath, + "--source-root", rootPath(), + "--metadata", join(root, "unused-metadata.json"), + "--release-dir", releaseDir, + "--source-commit", sourceCommit, + "--worktree", publisherWorktree, + ], { cwd: rootPath(), encoding: "utf8", timeout: 30_000 }); + expect(result.status, result.stderr).toBe(0); + const output = JSON.parse(result.stdout) as Record; + expect(output).toMatchObject({ action: "skip", status: "skipped", changed: false, pushAttempts: 0, resources: [] }); + expect(output.gitopsCommit).toBe(originalHead); + expect(git(["rev-parse", "refs/heads/unidesk-host-gitops"], bare).stdout.trim()).toBe(originalHead); + expect(show(bare, resourcePath)).toBe("stale configmap must remain unchanged\n"); + expect(JSON.parse(show(bare, statePath)).sourceCommit).toBe(runtimeSourceCommit); + }); +}); + +function write(root: string, relativePath: string, content: string) { + const path = join(root, relativePath); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); +} + +function identity(root: string) { + git(["config", "user.name", "UniDesk Test"], root); + git(["config", "user.email", "unidesk-test@example.invalid"], root); +} + +function show(bare: string, relativePath: string): string { + return git(["show", `refs/heads/unidesk-host-gitops:${relativePath}`], bare).stdout; +} + +function git(args: string[], cwd: string) { + const result = spawnSync("git", args, { cwd, encoding: "utf8" }); + if (result.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr || result.stdout}`); + return result; +} diff --git a/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs b/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs index 9260d699..3771da77 100644 --- a/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs +++ b/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs @@ -4,6 +4,7 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { createRequire } from "node:module"; +import { runtimeGitopsScriptsConfigMap } from "./runtime-gitops-scripts-configmap.mjs"; const requireFromScript = createRequire(import.meta.url); const requireFromCwd = createRequire(path.join(process.cwd(), "package.json")); @@ -125,31 +126,9 @@ function ensureRuntimeGitopsScriptsMount(taskSpec, step) { function writeScriptsConfigMap(targetPath, namespace) { if (scriptsDir === null) throw new Error("--scripts-dir is required with --scripts-configmap"); - const data = { - "runtime-gitops-overlay.json": `${JSON.stringify({ - runtimePath: requiredOverlayString("runtimePath"), - observability: objectOr(overlay.observability), - })}\n`, - }; - for (const name of ["runtime-gitops-observability.mjs", "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs"]) { - data[name] = readFileSync(path.join(scriptsDir, name), "utf8"); - } - writeFileSync(targetPath, `${YAML.stringify({ - apiVersion: "v1", - kind: "ConfigMap", - metadata: { - name: configMapName, - namespace, - labels: { - "app.kubernetes.io/name": "hwlab-runtime-gitops-scripts", - "app.kubernetes.io/part-of": "unidesk-hwlab-control-plane", - "hwlab.pikastech.local/node": overlay.nodeId ?? null, - "hwlab.pikastech.local/lane": overlay.lane ?? null, - }, - }, - data, - }).trimEnd()}\n`, "utf8"); - return { name: configMapName, namespace, keyCount: Object.keys(data).length, path: targetPath }; + const configMap = runtimeGitopsScriptsConfigMap({ overlay, scriptsDir, namespace, configMapName }); + writeFileSync(targetPath, `${YAML.stringify(configMap).trimEnd()}\n`, "utf8"); + return { name: configMapName, namespace, keyCount: Object.keys(configMap.data).length, path: targetPath }; } function pipelineNamespace(docs) { diff --git a/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs b/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs new file mode 100644 index 00000000..36c67d91 --- /dev/null +++ b/scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs @@ -0,0 +1,39 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; + +export function runtimeGitopsScriptsConfigMap({ overlay, scriptsDir, namespace, configMapName }) { + const data = { + "runtime-gitops-overlay.json": `${JSON.stringify({ + runtimePath: requiredString(overlay.runtimePath, "overlay.runtimePath"), + observability: objectOr(overlay.observability), + })}\n`, + }; + for (const name of ["runtime-gitops-observability.mjs", "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs"]) { + data[name] = readFileSync(path.join(scriptsDir, name), "utf8"); + } + return { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: configMapName, + namespace, + labels: { + "app.kubernetes.io/name": "hwlab-runtime-gitops-scripts", + "app.kubernetes.io/part-of": "unidesk-hwlab-control-plane", + "app.kubernetes.io/managed-by": "unidesk-host-gitops", + "hwlab.pikastech.local/node": overlay.nodeId ?? null, + "hwlab.pikastech.local/lane": overlay.lane ?? null, + }, + }, + data, + }; +} + +function requiredString(value, pathLabel) { + if (typeof value !== "string" || value.length === 0) throw new Error(`${pathLabel} must be a non-empty string`); + return value; +} + +function objectOr(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} diff --git a/scripts/src/hwlab-node/render.ts b/scripts/src/hwlab-node/render.ts index 22450fef..1dd78a32 100644 --- a/scripts/src/hwlab-node/render.ts +++ b/scripts/src/hwlab-node/render.ts @@ -47,6 +47,7 @@ import type { CicdDeliveryAuthority } from "../cicd-delivery-authority"; const runtimeGitopsObservabilityNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-observability.mjs"), "utf8").trimEnd(); const runtimeGitopsPipelineGuardNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs"), "utf8").trimEnd(); +const runtimeGitopsScriptsConfigMapNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs"), "utf8").trimEnd(); const runtimeGitopsPostprocessNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-postprocess.mjs"), "utf8").trimEnd(); const runtimeGitopsVerifyNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-verify.mjs"), "utf8").trimEnd(); const runtimePipelineProvenanceNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-pipeline-provenance.mjs"), "utf8").trimEnd(); @@ -2927,6 +2928,7 @@ function runtimeGitopsPipelineGuardScript(): string[] { "runtime_gitops_guard_dir=\"$render_dir/.unidesk-runtime-gitops\"", "mkdir -p \"$runtime_gitops_guard_dir\"", ...writeRuntimeGitopsNativeScript("runtime-gitops-pipeline-guard.mjs", runtimeGitopsPipelineGuardNativeScript, "UNIDESK_RUNTIME_GITOPS_PIPELINE_GUARD_MJS"), + ...writeRuntimeGitopsNativeScript("runtime-gitops-scripts-configmap.mjs", runtimeGitopsScriptsConfigMapNativeScript, "UNIDESK_RUNTIME_GITOPS_SCRIPTS_CONFIGMAP_MJS"), ...writeRuntimeGitopsNativeScript("runtime-gitops-observability.mjs", runtimeGitopsObservabilityNativeScript, "UNIDESK_RUNTIME_GITOPS_OBSERVABILITY_MJS"), ...writeRuntimeGitopsNativeScript("runtime-gitops-postprocess.mjs", runtimeGitopsPostprocessNativeScript, "UNIDESK_RUNTIME_GITOPS_POSTPROCESS_MJS"), ...writeRuntimeGitopsNativeScript("runtime-gitops-verify.mjs", runtimeGitopsVerifyNativeScript, "UNIDESK_RUNTIME_GITOPS_VERIFY_MJS"), diff --git a/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts b/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts index 6301578d..e4cc2814 100644 --- a/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts +++ b/scripts/src/platform-infra-pipelines-as-code-source-artifact.test.ts @@ -35,6 +35,7 @@ import { observePacSourceArtifactRuntime, selectPipelineRunBySourceCommit, } from "../native/cicd/pac-source-artifact-runtime.mjs"; +import { renderGitOpsResources } from "../native/cicd/publish-unidesk-host-gitops.mjs"; const sourceCommit = "a".repeat(40); const provenance = { @@ -306,6 +307,7 @@ describe("HWLAB YAML-owned Pipeline provenance", () => { expect(scripts.join("\n")).not.toContain("UNIDESK_RUNTIME_GITOPS_OVERLAY_B64="); expect(scripts.join("\n")).not.toContain(Buffer.from(JSON.stringify(overlay.observability), "utf8").toString("base64")); const configMap = Bun.YAML.parse(readFileSync(configMapPath, "utf8")) as Record; + expect(configMap.metadata.labels["app.kubernetes.io/managed-by"]).toBe("unidesk-host-gitops"); expect(JSON.parse(configMap.data["runtime-gitops-overlay.json"])).toEqual({ runtimePath: overlay.runtimePath, observability: overlay.observability }); expect(Object.keys(configMap.data).sort()).toEqual([ "runtime-gitops-observability.mjs", @@ -313,6 +315,34 @@ describe("HWLAB YAML-owned Pipeline provenance", () => { "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs", ]); + const hostConfig = Bun.YAML.parse(readFileSync(rootPath("config", "unidesk-host-k8s.yaml"), "utf8")) as Record; + const runtimePaths = new Set(hostConfig.delivery.changeDetection.runtimePaths as string[]); + for (const inputPath of [ + "config/hwlab-node-lanes.yaml", + "scripts/native/hwlab/runtime-gitops-observability.mjs", + "scripts/native/hwlab/runtime-gitops-postprocess.mjs", + "scripts/native/hwlab/runtime-gitops-scripts-configmap.mjs", + "scripts/native/hwlab/runtime-gitops-verify.mjs", + "scripts/src/config.ts", + "scripts/src/hwlab-node-lanes.ts", + "scripts/src/yaml-composition.ts", + ]) expect(runtimePaths.has(inputPath), inputPath).toBe(true); + expect(hostConfig.delivery.gitops.resources).toContainEqual({ + id: "hwlab-nc01-v03-runtime-gitops-scripts", + renderer: "hwlab-runtime-gitops-scripts", + configRef: "config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01", + manifestPath: "deploy/gitops/unidesk-host/hwlab-nc01-v03-runtime-gitops-scripts.yaml", + namespace: "hwlab-ci", + }); + const gitopsResources = renderGitOpsResources(hostConfig.delivery.gitops, rootPath()); + expect(gitopsResources).toHaveLength(1); + expect(gitopsResources[0]?.path).toBe("deploy/gitops/unidesk-host/hwlab-nc01-v03-runtime-gitops-scripts.yaml"); + const gitopsConfigMap = Bun.YAML.parse(gitopsResources[0]?.content ?? "") as Record; + expect(gitopsConfigMap.metadata.name).toBe("hwlab-nc01-v03-ci-image-publish-runtime-gitops-scripts"); + expect(gitopsConfigMap.metadata.namespace).toBe("hwlab-ci"); + expect(gitopsConfigMap.data["runtime-gitops-postprocess.mjs"]).toContain("UNIDESK_RUNTIME_GITOPS_OVERLAY_FILE"); + expect(gitopsConfigMap.data["runtime-gitops-postprocess.mjs"].indexOf("UNIDESK_RUNTIME_GITOPS_OVERLAY_FILE")) + .toBeLessThan(gitopsConfigMap.data["runtime-gitops-postprocess.mjs"].indexOf("UNIDESK_RUNTIME_GITOPS_OVERLAY_B64")); const legacyPayload = Buffer.from(JSON.stringify({ runtimePath: overlay.runtimePath, observability: overlay.observability }), "utf8").toString("base64"); const legacyPipeline = Bun.YAML.parse(readFileSync(pipelinePath, "utf8")) as Record; const legacyStep = legacyPipeline.spec.tasks[0].taskSpec.steps[0];