ci: add PaC-driven AgentRun JD01 v02 pipeline
This commit is contained in:
Codex
@@ -173,6 +173,8 @@ export function renderedObjectsDigest(objects: readonly Record<string, unknown>[
|
||||
}
|
||||
|
||||
function agentRunPipelineManifest(spec: AgentRunLaneSpec): Record<string, unknown> {
|
||||
const build = spec.deployment.manager.imageBuild;
|
||||
if (spec.ci.buildkitImage === null) throw new Error(`config/agentrun.yaml controlPlane.lanes.${spec.lane}.ci.buildkitImage is required for AgentRun Tekton image build`);
|
||||
return {
|
||||
apiVersion: "tekton.dev/v1",
|
||||
kind: "Pipeline",
|
||||
@@ -189,44 +191,280 @@ function agentRunPipelineManifest(spec: AgentRunLaneSpec): Record<string, unknow
|
||||
{ name: "source-branch", type: "string", default: spec.source.branch },
|
||||
{ name: "gitops-branch", type: "string", default: spec.gitops.branch },
|
||||
{ name: "revision", type: "string" },
|
||||
{ name: "source-stage-ref", type: "string" },
|
||||
{ name: "registry-prefix", type: "string", default: spec.ci.registryPrefix },
|
||||
{ name: "tools-image", type: "string", default: spec.ci.toolsImage },
|
||||
{ name: "buildkit-image", type: "string", default: spec.ci.buildkitImage },
|
||||
{ name: "containerfile", type: "string", default: build.containerfile },
|
||||
{ name: "context-dir", type: "string", default: build.context },
|
||||
{ name: "image-repository", type: "string", default: `${spec.ci.registryPrefix}/${build.repository}` },
|
||||
{ name: "build-network", type: "string", default: build.network },
|
||||
{ name: "build-args-json", type: "string", default: JSON.stringify(Object.entries(build.buildArgs).sort(([left], [right]) => left.localeCompare(right)).map(([key, value]) => `${key}=${value}`)) },
|
||||
{ name: "build-http-proxy", type: "string", default: build.httpProxy ?? "" },
|
||||
{ name: "build-https-proxy", type: "string", default: build.httpsProxy ?? "" },
|
||||
{ name: "build-no-proxy", type: "string", default: build.noProxy.join(",") },
|
||||
{ name: "container-http-proxy", type: "string", default: build.buildContainerProxy.httpProxy ?? "" },
|
||||
{ name: "container-https-proxy", type: "string", default: build.buildContainerProxy.httpsProxy ?? "" },
|
||||
{ name: "container-no-proxy", type: "string", default: build.buildContainerProxy.noProxy.join(",") },
|
||||
{ name: "env-identity-files-json", type: "string", default: JSON.stringify(build.envIdentityFiles) },
|
||||
{ name: "gitops-root", type: "string", default: spec.deployment.gitopsRoot },
|
||||
{ name: "artifact-catalog", type: "string", default: spec.deployment.artifactCatalogPath },
|
||||
],
|
||||
workspaces: [{ name: "source" }, { name: "git-ssh" }],
|
||||
workspaces: [{ name: "source" }],
|
||||
tasks: [
|
||||
gitopsSmokeTask(spec),
|
||||
agentRunBuildPublishTask(spec),
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function gitopsSmokeTask(spec: AgentRunLaneSpec): Record<string, unknown> {
|
||||
function agentRunBuildPublishTask(spec: AgentRunLaneSpec): Record<string, unknown> {
|
||||
return {
|
||||
name: "render-smoke",
|
||||
name: "build-publish",
|
||||
workspaces: [{ name: "source", workspace: "source" }],
|
||||
taskSpec: {
|
||||
params: [{ name: "revision" }, { name: "tools-image" }],
|
||||
params: [
|
||||
{ name: "git-read-url" },
|
||||
{ name: "git-write-url" },
|
||||
{ name: "source-branch" },
|
||||
{ name: "gitops-branch" },
|
||||
{ name: "revision" },
|
||||
{ name: "source-stage-ref" },
|
||||
{ name: "registry-prefix" },
|
||||
{ name: "tools-image" },
|
||||
{ name: "buildkit-image" },
|
||||
{ name: "containerfile" },
|
||||
{ name: "context-dir" },
|
||||
{ name: "image-repository" },
|
||||
{ name: "build-network" },
|
||||
{ name: "build-args-json" },
|
||||
{ name: "build-http-proxy" },
|
||||
{ name: "build-https-proxy" },
|
||||
{ name: "build-no-proxy" },
|
||||
{ name: "container-http-proxy" },
|
||||
{ name: "container-https-proxy" },
|
||||
{ name: "container-no-proxy" },
|
||||
{ name: "env-identity-files-json" },
|
||||
{ name: "gitops-root" },
|
||||
{ name: "artifact-catalog" },
|
||||
],
|
||||
workspaces: [{ name: "source" }],
|
||||
steps: [
|
||||
{
|
||||
name: "render-smoke",
|
||||
name: "source-checkout",
|
||||
image: "$(params.tools-image)",
|
||||
script: [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"echo '{\"event\":\"agentrun-ci-render-smoke\",\"status\":\"placeholder\",\"reason\":\"unidesk-yaml-only-control-plane\",\"valuesPrinted\":false}'",
|
||||
].join("\n"),
|
||||
script: agentRunTektonSourceCheckoutScript(),
|
||||
},
|
||||
{
|
||||
name: "probe-env-image",
|
||||
image: "$(params.tools-image)",
|
||||
script: agentRunTektonProbeImageScript(),
|
||||
},
|
||||
{
|
||||
name: "build-env-image",
|
||||
image: "$(params.buildkit-image)",
|
||||
env: [
|
||||
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
||||
],
|
||||
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
||||
script: agentRunTektonBuildImageScript(),
|
||||
},
|
||||
{
|
||||
name: "publish-gitops",
|
||||
image: "$(params.tools-image)",
|
||||
script: agentRunTektonGitopsPublishScript(spec),
|
||||
},
|
||||
],
|
||||
},
|
||||
params: [
|
||||
{ name: "git-read-url", value: "$(params.git-read-url)" },
|
||||
{ name: "git-write-url", value: "$(params.git-write-url)" },
|
||||
{ name: "source-branch", value: "$(params.source-branch)" },
|
||||
{ name: "gitops-branch", value: "$(params.gitops-branch)" },
|
||||
{ name: "revision", value: "$(params.revision)" },
|
||||
{ name: "source-stage-ref", value: "$(params.source-stage-ref)" },
|
||||
{ name: "registry-prefix", value: "$(params.registry-prefix)" },
|
||||
{ name: "tools-image", value: "$(params.tools-image)" },
|
||||
{ name: "buildkit-image", value: "$(params.buildkit-image)" },
|
||||
{ name: "containerfile", value: "$(params.containerfile)" },
|
||||
{ name: "context-dir", value: "$(params.context-dir)" },
|
||||
{ name: "image-repository", value: "$(params.image-repository)" },
|
||||
{ name: "build-network", value: "$(params.build-network)" },
|
||||
{ name: "build-args-json", value: "$(params.build-args-json)" },
|
||||
{ name: "build-http-proxy", value: "$(params.build-http-proxy)" },
|
||||
{ name: "build-https-proxy", value: "$(params.build-https-proxy)" },
|
||||
{ name: "build-no-proxy", value: "$(params.build-no-proxy)" },
|
||||
{ name: "container-http-proxy", value: "$(params.container-http-proxy)" },
|
||||
{ name: "container-https-proxy", value: "$(params.container-https-proxy)" },
|
||||
{ name: "container-no-proxy", value: "$(params.container-no-proxy)" },
|
||||
{ name: "env-identity-files-json", value: "$(params.env-identity-files-json)" },
|
||||
{ name: "gitops-root", value: "$(params.gitops-root)" },
|
||||
{ name: "artifact-catalog", value: "$(params.artifact-catalog)" },
|
||||
],
|
||||
when: [{ input: spec.deployment.format, operator: "in", values: ["unidesk-yaml-only"] }],
|
||||
};
|
||||
}
|
||||
|
||||
function agentRunTektonSourceCheckoutScript(): string {
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"root=\"$(workspaces.source.path)\"",
|
||||
"rm -rf \"$root/repo\"",
|
||||
"git clone --no-checkout \"$(params.git-read-url)\" \"$root/repo\"",
|
||||
"cd \"$root/repo\"",
|
||||
"git fetch origin \"+$(params.source-stage-ref):refs/remotes/origin/unidesk-source-snapshot\"",
|
||||
"git checkout --detach \"$(params.revision)\"",
|
||||
"actual=$(git rev-parse HEAD)",
|
||||
"test \"$actual\" = \"$(params.revision)\"",
|
||||
"ENV_IDENTITY_FILES='$(params.env-identity-files-json)' BUILD_ARGS_JSON='$(params.build-args-json)' CONTAINER_HTTP_PROXY='$(params.container-http-proxy)' CONTAINER_HTTPS_PROXY='$(params.container-https-proxy)' CONTAINER_NO_PROXY='$(params.container-no-proxy)' node <<'NODE' > \"$root/env-identity\"",
|
||||
"const { createHash } = require('node:crypto');",
|
||||
"const { existsSync, lstatSync, readdirSync, readFileSync } = require('node:fs');",
|
||||
"const { join } = require('node:path');",
|
||||
"const files = JSON.parse(process.env.ENV_IDENTITY_FILES || '[]');",
|
||||
"const buildArgs = JSON.parse(process.env.BUILD_ARGS_JSON || '[]');",
|
||||
"const proxy = [`HTTP_PROXY=${process.env.CONTAINER_HTTP_PROXY || ''}`, `HTTPS_PROXY=${process.env.CONTAINER_HTTPS_PROXY || ''}`, `NO_PROXY=${process.env.CONTAINER_NO_PROXY || ''}`];",
|
||||
"const skip = new Set(['.git', '.worktree', '.state', 'node_modules', 'coverage', 'tmp', '.tmp']);",
|
||||
"const hash = createHash('sha256');",
|
||||
"function collect(input) {",
|
||||
" if (!existsSync(input)) return [{ path: input, missing: true }];",
|
||||
" const stat = lstatSync(input);",
|
||||
" if (stat.isFile()) return [{ path: input, missing: false }];",
|
||||
" if (!stat.isDirectory()) return [{ path: input, missing: true }];",
|
||||
" const out = [];",
|
||||
" const stack = [input];",
|
||||
" while (stack.length) {",
|
||||
" const dir = stack.pop();",
|
||||
" for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {",
|
||||
" if (entry.isDirectory() && skip.has(entry.name)) continue;",
|
||||
" const child = join(dir, entry.name);",
|
||||
" if (entry.isDirectory()) stack.push(child);",
|
||||
" else if (entry.isFile()) out.push({ path: child, missing: false });",
|
||||
" }",
|
||||
" }",
|
||||
" return out.sort((a, b) => a.path.localeCompare(b.path));",
|
||||
"}",
|
||||
"for (const item of buildArgs) { hash.update('build-arg\\0'); hash.update(item); hash.update('\\0'); }",
|
||||
"for (const item of proxy) { hash.update('build-container-proxy\\0'); hash.update(item); hash.update('\\0'); }",
|
||||
"for (const file of files) for (const entry of collect(file)) { hash.update(entry.path); hash.update('\\0'); if (!entry.missing) hash.update(readFileSync(entry.path)); hash.update('\\0'); }",
|
||||
"process.stdout.write(hash.digest('hex').slice(0, 24));",
|
||||
"NODE",
|
||||
"BUILD_ARGS='$(params.build-args-json)' node <<'NODE' > \"$root/build-args.txt\"",
|
||||
"const values = JSON.parse(process.env.BUILD_ARGS || '[]');",
|
||||
"for (const value of values) console.log(`build-arg:${value}`);",
|
||||
"NODE",
|
||||
"chmod -R a+rwX \"$root\"",
|
||||
"env_identity=$(cat \"$root/env-identity\")",
|
||||
"printf '{\"ok\":true,\"phase\":\"source-checkout\",\"sourceCommit\":\"%s\",\"sourceStageRef\":\"%s\",\"envIdentity\":\"%s\",\"valuesPrinted\":false}\\n' \"$(params.revision)\" \"$(params.source-stage-ref)\" \"$env_identity\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function agentRunTektonProbeImageScript(): string {
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"root=\"$(workspaces.source.path)\"",
|
||||
"env_identity=$(cat \"$root/env-identity\")",
|
||||
"image_repository='$(params.image-repository)'",
|
||||
"manifest_accept='application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'",
|
||||
"repo_path=${image_repository#127.0.0.1:5000/}",
|
||||
"headers=$(mktemp)",
|
||||
"digest=''",
|
||||
"if curl -fsSI -H \"Accept: $manifest_accept\" \"http://127.0.0.1:5000/v2/$repo_path/manifests/$env_identity\" >\"$headers\"; then",
|
||||
" digest=$(awk -F': ' 'tolower($1)==\"docker-content-digest\"{print $2}' \"$headers\" | tr -d '\\r' | head -n 1)",
|
||||
"fi",
|
||||
"rm -f \"$headers\"",
|
||||
"if [ -n \"$digest\" ]; then",
|
||||
" image=\"$image_repository:$env_identity\"",
|
||||
" printf '{\"ok\":true,\"status\":\"reused\",\"sourceCommit\":\"%s\",\"envIdentity\":\"%s\",\"image\":\"%s\",\"digest\":\"%s\",\"repositoryDigest\":\"%s@%s\",\"valuesPrinted\":false}\\n' \"$(params.revision)\" \"$env_identity\" \"$image\" \"$digest\" \"$image_repository\" \"$digest\" > \"$root/build-result.json\"",
|
||||
" touch \"$root/skip-build\"",
|
||||
"else",
|
||||
" printf '{\"ok\":false,\"status\":\"cache-miss\",\"sourceCommit\":\"%s\",\"envIdentity\":\"%s\",\"valuesPrinted\":false}\\n' \"$(params.revision)\" \"$env_identity\" > \"$root/build-result.json\"",
|
||||
"fi",
|
||||
"cat \"$root/build-result.json\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function agentRunTektonBuildImageScript(): string {
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"root=\"$(workspaces.source.path)\"",
|
||||
"if [ -f \"$root/skip-build\" ]; then cat \"$root/build-result.json\"; exit 0; fi",
|
||||
"env_identity=$(cat \"$root/env-identity\")",
|
||||
"image_repository='$(params.image-repository)'",
|
||||
"image=\"$image_repository:$env_identity\"",
|
||||
"context_dir='$(params.context-dir)'",
|
||||
"if [ \"$context_dir\" = \".\" ]; then context_path=\"$root/repo\"; else context_path=\"$root/repo/${context_dir#./}\"; fi",
|
||||
"args=\"build --allow network.host --frontend dockerfile.v0 --local context=$context_path --local dockerfile=$root/repo --opt filename=$(params.containerfile) --opt network=$(params.build-network)\"",
|
||||
"add_opt() { args=\"$args --opt $1\"; }",
|
||||
"if [ -n '$(params.container-http-proxy)' ]; then add_opt 'build-arg:HTTP_PROXY=$(params.container-http-proxy)'; add_opt 'build-arg:http_proxy=$(params.container-http-proxy)'; fi",
|
||||
"if [ -n '$(params.container-https-proxy)' ]; then add_opt 'build-arg:HTTPS_PROXY=$(params.container-https-proxy)'; add_opt 'build-arg:https_proxy=$(params.container-https-proxy)'; fi",
|
||||
"if [ -n '$(params.container-https-proxy)' ]; then add_opt 'build-arg:ALL_PROXY=$(params.container-https-proxy)'; add_opt 'build-arg:all_proxy=$(params.container-https-proxy)'; elif [ -n '$(params.container-http-proxy)' ]; then add_opt 'build-arg:ALL_PROXY=$(params.container-http-proxy)'; add_opt 'build-arg:all_proxy=$(params.container-http-proxy)'; fi",
|
||||
"if [ -n '$(params.container-no-proxy)' ]; then add_opt 'build-arg:NO_PROXY=$(params.container-no-proxy)'; add_opt 'build-arg:no_proxy=$(params.container-no-proxy)'; fi",
|
||||
"while IFS= read -r opt; do [ -n \"$opt\" ] && add_opt \"$opt\"; done < \"$root/build-args.txt\"",
|
||||
"args=\"$args --metadata-file $root/build-metadata.json --output type=image,name=$image,push=true,registry.insecure=true\"",
|
||||
"buildctl-daemonless.sh $args",
|
||||
"metadata_compact=$(tr -d '\\n' < \"$root/build-metadata.json\")",
|
||||
"digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)",
|
||||
"test -n \"$digest\"",
|
||||
"printf '{\"ok\":true,\"status\":\"built\",\"sourceCommit\":\"%s\",\"envIdentity\":\"%s\",\"image\":\"%s\",\"digest\":\"%s\",\"repositoryDigest\":\"%s@%s\",\"valuesPrinted\":false}\\n' \"$(params.revision)\" \"$env_identity\" \"$image\" \"$digest\" \"$image_repository\" \"$digest\" > \"$root/build-result.json\"",
|
||||
"cat \"$root/build-result.json\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function agentRunTektonGitopsPublishScript(spec: AgentRunLaneSpec): string {
|
||||
const sourcePlaceholder = "__AGENTRUN_SOURCE_COMMIT__";
|
||||
const envPlaceholder = "__AGENTRUN_ENV_IDENTITY__";
|
||||
const digestPlaceholder = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
const statusPlaceholder = "__AGENTRUN_IMAGE_STATUS__";
|
||||
const templateImage = agentRunImageArtifact(spec, { sourceCommit: sourcePlaceholder, envIdentity: envPlaceholder, digest: digestPlaceholder, status: statusPlaceholder });
|
||||
const templateFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: sourcePlaceholder, image: templateImage });
|
||||
const templateB64 = Buffer.from(JSON.stringify(templateFiles), "utf8").toString("base64");
|
||||
return [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
"root=\"$(workspaces.source.path)\"",
|
||||
"build_result=\"$root/build-result.json\"",
|
||||
"test -s \"$build_result\"",
|
||||
`templates_b64=${JSON.stringify(templateB64)}`,
|
||||
"rm -rf \"$root/gitops\"",
|
||||
"git clone \"$(params.git-write-url)\" \"$root/gitops\"",
|
||||
"cd \"$root/gitops\"",
|
||||
"git fetch origin \"$(params.gitops-branch)\" || true",
|
||||
"if git rev-parse --verify \"refs/remotes/origin/$(params.gitops-branch)^{commit}\" >/dev/null 2>&1; then git checkout -B \"$(params.gitops-branch)\" \"refs/remotes/origin/$(params.gitops-branch)\"; else git checkout --orphan \"$(params.gitops-branch)\"; git rm -rf . >/dev/null 2>&1 || true; fi",
|
||||
"git rm -rf --ignore-unmatch \"$(params.gitops-root)\" \"$(params.artifact-catalog)\" source.json >/dev/null 2>&1 || true",
|
||||
"rm -rf \"$(params.gitops-root)\" \"$(params.artifact-catalog)\" source.json",
|
||||
"TEMPLATES_B64=\"$templates_b64\" BUILD_RESULT=\"$build_result\" node <<'NODE'",
|
||||
"const { mkdirSync, readFileSync, writeFileSync } = require('node:fs');",
|
||||
"const { dirname } = require('node:path');",
|
||||
"const templates = JSON.parse(Buffer.from(process.env.TEMPLATES_B64, 'base64').toString('utf8'));",
|
||||
"const build = JSON.parse(readFileSync(process.env.BUILD_RESULT, 'utf8'));",
|
||||
"const replacements = new Map([",
|
||||
" ['__AGENTRUN_SOURCE_COMMIT__', build.sourceCommit],",
|
||||
" ['__AGENTRUN_ENV_IDENTITY__', build.envIdentity],",
|
||||
" ['sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', build.digest],",
|
||||
" ['__AGENTRUN_IMAGE_STATUS__', build.status],",
|
||||
"]);",
|
||||
"for (const file of templates) {",
|
||||
" let content = file.content;",
|
||||
" for (const [from, to] of replacements) content = content.split(from).join(to);",
|
||||
" mkdirSync(dirname(file.path), { recursive: true });",
|
||||
" writeFileSync(file.path, content);",
|
||||
"}",
|
||||
"NODE",
|
||||
"git add source.json \"$(params.artifact-catalog)\" \"$(params.gitops-root)\"",
|
||||
"if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun PaC' commit -m \"deploy: render AgentRun $(params.gitops-branch) from PaC\"; fi",
|
||||
"git push -u origin \"$(params.gitops-branch)\"",
|
||||
"gitops_commit=$(git rev-parse HEAD)",
|
||||
"BUILD_RESULT=\"$build_result\" CHANGED=\"$changed\" GITOPS_COMMIT=\"$gitops_commit\" node <<'NODE'",
|
||||
"const { readFileSync } = require('node:fs');",
|
||||
"const build = JSON.parse(readFileSync(process.env.BUILD_RESULT, 'utf8'));",
|
||||
"console.log(JSON.stringify({ ok: true, status: 'succeeded', phase: 'gitops-publish', changed: process.env.CHANGED === 'true', gitopsCommit: process.env.GITOPS_COMMIT, sourceCommit: build.sourceCommit, envIdentity: build.envIdentity, imageStatus: build.status, digest: build.digest, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function agentRunArgoProjectManifest(spec: AgentRunLaneSpec): Record<string, unknown> {
|
||||
return {
|
||||
apiVersion: "argoproj.io/v1alpha1",
|
||||
@@ -498,7 +736,11 @@ function agentRunArtifactCatalog(spec: AgentRunLaneSpec, sourceCommit: string, i
|
||||
sourceBranch: spec.source.branch,
|
||||
gitopsBranch: spec.gitops.branch,
|
||||
sourceCommitId: sourceCommit,
|
||||
summary: image.status === "placeholder" ? "build=0 reuse=0 placeholder=1" : "build=1 reuse=0 placeholder=0",
|
||||
summary: image.status === "placeholder"
|
||||
? "build=0 reuse=0 placeholder=1"
|
||||
: image.status === "reused"
|
||||
? "build=0 reuse=1 placeholder=0"
|
||||
: "build=1 reuse=0 placeholder=0",
|
||||
services: [image],
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user