diff --git a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml index 3f15279a..29362818 100644 --- a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml @@ -27,7 +27,6 @@ sentinel: sourceMode: sparse-git-checkout jobPrefix: web-probe-sentinel-auth-switch-publish gitSshSecretName: git-mirror-github-ssh - dockerSocketPath: /var/run/docker.sock activeDeadlineSeconds: 900 ttlSecondsAfterFinished: 3600 gitopsPath: deploy/gitops/node/d601/web-probe-sentinel-auth-switch @@ -50,14 +49,14 @@ sentinel: runtimeMode: runner-served-bridge assetRoot: scripts/assets/web-probe-sentinel-monitor-web envReuse: - mode: docker-layer-and-ci-node-deps + mode: k8s-buildkit-and-ci-node-deps nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules - dockerBuild: - packageMode: copy-only-dockerfile + imageBuild: + packageMode: copy-only-containerfile networkMode: host - proxySource: node.networkProfile.dockerBuildProxy + proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated - verifyPhase: pre-docker-build + verifyPhase: pre-image-build gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml index e8e943b6..da35fb45 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml @@ -27,7 +27,6 @@ sentinel: sourceMode: sparse-git-checkout jobPrefix: web-probe-sentinel-publish gitSshSecretName: git-mirror-github-ssh - dockerSocketPath: /var/run/docker.sock activeDeadlineSeconds: 900 ttlSecondsAfterFinished: 3600 gitopsPath: deploy/gitops/node/d518/web-probe-sentinel-dsflash @@ -50,14 +49,14 @@ sentinel: runtimeMode: runner-served-bridge assetRoot: scripts/assets/web-probe-sentinel-monitor-web envReuse: - mode: docker-layer-and-ci-node-deps + mode: k8s-buildkit-and-ci-node-deps nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules - dockerBuild: - packageMode: copy-only-dockerfile + imageBuild: + packageMode: copy-only-containerfile networkMode: host - proxySource: node.networkProfile.dockerBuildProxy + proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated - verifyPhase: pre-docker-build + verifyPhase: pre-image-build gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml index d88d9a04..71d5e7df 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml @@ -27,7 +27,6 @@ sentinel: sourceMode: sparse-git-checkout jobPrefix: web-probe-sentinel-publish gitSshSecretName: git-mirror-github-ssh - dockerSocketPath: /var/run/docker.sock activeDeadlineSeconds: 900 ttlSecondsAfterFinished: 3600 gitopsPath: deploy/gitops/node/d601/web-probe-sentinel @@ -50,14 +49,14 @@ sentinel: runtimeMode: runner-served-bridge assetRoot: scripts/assets/web-probe-sentinel-monitor-web envReuse: - mode: docker-layer-and-ci-node-deps + mode: k8s-buildkit-and-ci-node-deps nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules - dockerBuild: - packageMode: copy-only-dockerfile + imageBuild: + packageMode: copy-only-containerfile networkMode: host - proxySource: node.networkProfile.dockerBuildProxy + proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated - verifyPhase: pre-docker-build + verifyPhase: pre-image-build gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml index a9bb6ad9..2b18041f 100644 --- a/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.fake-echo.d518-v03.yaml @@ -28,7 +28,6 @@ sentinel: sourceMode: sparse-git-checkout jobPrefix: web-probe-sentinel-publish gitSshSecretName: git-mirror-github-ssh - dockerSocketPath: /var/run/docker.sock activeDeadlineSeconds: 900 ttlSecondsAfterFinished: 3600 gitopsPath: deploy/gitops/node/d518/web-probe-sentinel-fake-echo @@ -51,14 +50,14 @@ sentinel: runtimeMode: runner-served-bridge assetRoot: scripts/assets/web-probe-sentinel-monitor-web envReuse: - mode: docker-layer-and-ci-node-deps + mode: k8s-buildkit-and-ci-node-deps nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules - dockerBuild: - packageMode: copy-only-dockerfile + imageBuild: + packageMode: copy-only-containerfile networkMode: host - proxySource: node.networkProfile.dockerBuildProxy + proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated - verifyPhase: pre-docker-build + verifyPhase: pre-image-build gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml index c1638340..9167c03b 100644 --- a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml @@ -27,7 +27,6 @@ sentinel: sourceMode: sparse-git-checkout jobPrefix: web-probe-sentinel-mdtodo-publish gitSshSecretName: git-mirror-github-ssh - dockerSocketPath: /var/run/docker.sock activeDeadlineSeconds: 900 ttlSecondsAfterFinished: 3600 gitopsPath: deploy/gitops/node/d601/web-probe-sentinel-mdtodo @@ -50,14 +49,14 @@ sentinel: runtimeMode: runner-served-bridge assetRoot: scripts/assets/web-probe-sentinel-monitor-web envReuse: - mode: docker-layer-and-ci-node-deps + mode: k8s-buildkit-and-ci-node-deps nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules - dockerBuild: - packageMode: copy-only-dockerfile + imageBuild: + packageMode: copy-only-containerfile networkMode: host - proxySource: node.networkProfile.dockerBuildProxy + proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated - verifyPhase: pre-docker-build + verifyPhase: pre-image-build gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml index 58ed448e..1f394b5c 100644 --- a/config/hwlab-web-probe-sentinel/profiles.yaml +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -53,7 +53,6 @@ baselines: namespace: devops-infra sourceMode: sparse-git-checkout gitSshSecretName: git-mirror-github-ssh - dockerSocketPath: /var/run/docker.sock activeDeadlineSeconds: 900 ttlSecondsAfterFinished: 3600 monitorWeb: &monitor-web @@ -61,14 +60,14 @@ baselines: runtimeMode: runner-served-bridge assetRoot: scripts/assets/web-probe-sentinel-monitor-web envReuse: - mode: docker-layer-and-ci-node-deps + mode: k8s-buildkit-and-ci-node-deps nodeDepsPath: /opt/hwlab-ci-node-deps/node_modules - dockerBuild: - packageMode: copy-only-dockerfile + imageBuild: + packageMode: copy-only-containerfile networkMode: host - proxySource: node.networkProfile.dockerBuildProxy + proxySource: node.networkProfile.imageBuildProxy contextIgnore: generated - verifyPhase: pre-docker-build + verifyPhase: pre-image-build gitMirror: source: source.gitMirrorReadUrl preSync: required diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index d21066dc..38a5ceef 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -29,6 +29,7 @@ export interface HwlabProxySpec { export interface HwlabNetworkProfileSpec { readonly id: string; readonly proxy: HwlabProxySpec; + readonly imageBuildProxy: HwlabProxySpec; readonly dockerBuildProxy: HwlabProxySpec; } @@ -613,7 +614,7 @@ function optionalStringRecord(value: unknown, path: string): Record, id: string, key: "proxy" | "dockerBuildProxy", requiredNoProxy: readonly string[]): HwlabProxySpec { +function proxyConfig(raw: Record, id: string, key: "proxy" | "imageBuildProxy" | "dockerBuildProxy", requiredNoProxy: readonly string[]): HwlabProxySpec { const proxy = asRecord(raw[key], `networkProfiles.${id}.${key}`); return { http: stringField(proxy, "http", `networkProfiles.${id}.${key}`), @@ -625,10 +626,14 @@ function proxyConfig(raw: Record, id: string, key: "proxy" | "d } function networkProfileConfig(id: string, raw: Record, requiredNoProxy: readonly string[]): HwlabNetworkProfileSpec { + const imageBuildProxy = raw.imageBuildProxy === undefined + ? proxyConfig(raw, id, "dockerBuildProxy", requiredNoProxy) + : proxyConfig(raw, id, "imageBuildProxy", requiredNoProxy); return { id, proxy: proxyConfig(raw, id, "proxy", requiredNoProxy), - dockerBuildProxy: proxyConfig(raw, id, "dockerBuildProxy", requiredNoProxy), + imageBuildProxy, + dockerBuildProxy: raw.dockerBuildProxy === undefined ? imageBuildProxy : proxyConfig(raw, id, "dockerBuildProxy", requiredNoProxy), }; } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 7d4aa575..7c77638b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -370,7 +370,7 @@ function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record): Record { +function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record): Record { return { stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build", runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge", @@ -406,21 +406,22 @@ function monitorWebCicdPlan(cicd: Record): Record): "default" | "host" { - const value = stringAtNullable(cicd, "monitorWeb.dockerBuild.networkMode") ?? "host"; - if (value !== "default" && value !== "host") throw new Error(`monitorWeb.dockerBuild.networkMode must be default or host, got ${value}`); +function monitorWebImageBuildNetworkMode(cicd: Record): "default" | "host" { + const value = stringAtNullable(cicd, "monitorWeb.imageBuild.networkMode") ?? "host"; + if (value !== "default" && value !== "host") throw new Error(`monitorWeb.imageBuild.networkMode must be default or host, got ${value}`); return value; } @@ -1613,6 +1614,8 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, publishGitops: boolean): Record { const namespace = stringAt(state.cicd, "builder.namespace"); + const buildkitImage = requireSentinelBuildkitImage(state); + const proxyEnv = sentinelImageBuildProxyEnv(state); const labels = { "app.kubernetes.io/name": "web-probe-sentinel-publish", "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", @@ -1632,20 +1635,55 @@ function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, p metadata: { labels }, spec: { restartPolicy: "Never", + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 }, volumes: [ sentinelGitMirrorCacheVolume(state), { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, - { name: "docker-sock", hostPath: { path: stringAt(state.cicd, "builder.dockerSocketPath"), type: "Socket" } }, + { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, + { name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } }, + { name: "tmp", emptyDir: {} }, + ], + initContainers: [ + { + name: "source", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + env: proxyEnv, + command: ["/bin/sh", "-ec", sentinelPublishSourceShell(state, jobName)], + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }, + { + name: "image-build", + image: buildkitImage, + imagePullPolicy: "IfNotPresent", + env: [ + ...proxyEnv, + { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }, + ], + command: ["/bin/sh", "-ec", sentinelPublishImageBuildShell(state, jobName)], + securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + { name: "tmp", mountPath: "/tmp" }, + ], + }, ], containers: [{ name: "publish", image: state.image.baseImage, imagePullPolicy: "IfNotPresent", + env: proxyEnv, command: ["/bin/sh", "-ec", sentinelPublishShell(state, jobName, publishGitops)], volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, { name: "cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - { name: "docker-sock", mountPath: stringAt(state.cicd, "builder.dockerSocketPath") }, ], }], }, @@ -1660,13 +1698,31 @@ function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record { + const proxy = state.spec.networkProfile.imageBuildProxy; + const noProxy = proxy.noProxy.join(","); + return [ + { name: "HTTP_PROXY", value: proxy.http }, + { name: "http_proxy", value: proxy.http }, + { name: "HTTPS_PROXY", value: proxy.https }, + { name: "https_proxy", value: proxy.https }, + { name: "ALL_PROXY", value: proxy.all }, + { name: "all_proxy", value: proxy.all }, + { name: "NO_PROXY", value: noProxy }, + { name: "no_proxy", value: noProxy }, + ]; +} + +function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string { const monitorWeb = record(state.image.monitorWeb); - const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ - path: file.path, - contentBase64: Buffer.from(file.content, "utf8").toString("base64"), - }))), "utf8").toString("base64"); const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => { if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths"); return item; @@ -1674,11 +1730,6 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64"); const envReuseMode = stringAt(monitorWeb, "envReuseMode"); const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath"); - const dockerBuildPackageMode = stringAt(monitorWeb, "dockerBuildPackageMode"); - const dockerBuildNetworkMode = stringAt(monitorWeb, "dockerBuildNetworkMode"); - const dockerBuildProxySource = stringAt(monitorWeb, "dockerBuildProxySource"); - const dockerBuildProxy = state.spec.networkProfile.dockerBuildProxy; - const dockerBuildNoProxy = dockerBuildProxy.noProxy.join(","); return [ "set -eu", `job_name=${shellQuote(jobName)}`, @@ -1687,32 +1738,27 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`, `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`, - `image_ref=${shellQuote(state.image.ref)}`, - `image_repository=${shellQuote(state.image.repository)}`, `dockerfile_b64=${shellQuote(dockerfileB64)}`, - `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`, - `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`, - `files_b64=${shellQuote(filesB64)}`, `env_reuse_mode=${shellQuote(envReuseMode)}`, `env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`, - `docker_build_package_mode=${shellQuote(dockerBuildPackageMode)}`, - `docker_build_network_mode=${shellQuote(dockerBuildNetworkMode)}`, - `docker_build_proxy_source=${shellQuote(dockerBuildProxySource)}`, - `docker_build_http_proxy=${shellQuote(dockerBuildProxy.http)}`, - `docker_build_https_proxy=${shellQuote(dockerBuildProxy.https)}`, - `docker_build_all_proxy=${shellQuote(dockerBuildProxy.all)}`, - `docker_build_no_proxy=${shellQuote(dockerBuildNoProxy)}`, - "started_ms=$(node -e 'console.log(Date.now())')", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(node -e 'console.log(Date.now())'); node - \"$stage\" \"$status\" \"$started\" \"$finished\" <<'NODE'\nconst [stage, status, started, finished] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-stage', stage, status, elapsedMs:Number(finished)-Number(started), valuesRedacted:true }));\nNODE\n}", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", + "meta_dir=/workspace/meta", + "event_log=/workspace/publish-events.log", + "worktree=/workspace/source", + "rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json", + "mkdir -p \"$meta_dir\"", + ": > \"$event_log\"", + "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", + "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", + "started_ms=$(now_ms)", + "write_meta started_ms \"$started_ms\"", + "write_meta source_commit \"$source_commit\"", "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", - "worktree=\"/tmp/$job_name/source\"", - "rm -rf \"/tmp/$job_name\"", - "mkdir -p \"/tmp/$job_name\"", "git init \"$worktree\"", "cd \"$worktree\"", "git remote add origin \"$source_git_url\"", @@ -1725,13 +1771,16 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "fs.mkdirSync('.git/info', { recursive: true });", "fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');", "NODE", - "source_fetch_started_ms=$(node -e 'console.log(Date.now())')", + "source_fetch_started_ms=$(now_ms)", + "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"", "emit_stage source-fetch running \"$source_fetch_started_ms\"", "git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"", "git checkout --detach \"$source_commit\"", "mirror_commit=$(git rev-parse HEAD)", "test \"$mirror_commit\" = \"$source_commit\"", - "source_fetch_finished_ms=$(node -e 'console.log(Date.now())')", + "write_meta mirror_commit \"$mirror_commit\"", + "source_fetch_finished_ms=$(now_ms)", + "write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"", "emit_stage source-fetch succeeded \"$source_fetch_started_ms\"", "env_reuse_node_deps_present=false", "env_reuse_node_deps_entries=0", @@ -1739,14 +1788,24 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "env_reuse_linked_node_deps=0", "rm -rf node_modules", "if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi", + "write_meta env_reuse_mode \"$env_reuse_mode\"", + "write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"", + "write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"", + "write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"", + "write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"", "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'", "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", "NODE", - "monitor_web_verify_started_ms=$(node -e 'console.log(Date.now())')", + "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"", + "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", + "NODE", + "monitor_web_verify_started_ms=$(now_ms)", + "write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"", "emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"", "if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi", "cat /tmp/web-probe-sentinel-monitor-web-verify.log", - "monitor_web_verify_finished_ms=$(node -e 'console.log(Date.now())')", + "monitor_web_verify_finished_ms=$(now_ms)", + "write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"", "emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"", "mkdir -p .unidesk-sentinel-bin", "cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'", @@ -1756,7 +1815,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "chmod 0755 .unidesk-sentinel-bin/trans", "DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'", "const fs = require('node:fs');", - "fs.writeFileSync('Dockerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));", + "fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));", "NODE", "cat > .dockerignore <<'EOF_DOCKERIGNORE'", ".git", @@ -1779,37 +1838,131 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish ".env", ".env.*", "EOF_DOCKERIGNORE", - "docker_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')", - "docker_build_http_proxy_present=false; if [ -n \"$docker_build_http_proxy\" ]; then docker_build_http_proxy_present=true; fi", - "docker_build_https_proxy_present=false; if [ -n \"$docker_build_https_proxy\" ]; then docker_build_https_proxy_present=true; fi", - "docker_build_all_proxy_present=false; if [ -n \"$docker_build_all_proxy\" ]; then docker_build_all_proxy_present=true; fi", - "docker_build_no_proxy_present=false; if [ -n \"$docker_build_no_proxy\" ]; then docker_build_no_proxy_present=true; fi", - "docker_build_started_ms=$(node -e 'console.log(Date.now())')", - "emit_stage docker-build running \"$docker_build_started_ms\"", - "if ! env HTTP_PROXY=\"$docker_build_http_proxy\" HTTPS_PROXY=\"$docker_build_https_proxy\" ALL_PROXY=\"$docker_build_all_proxy\" NO_PROXY=\"$docker_build_no_proxy\" http_proxy=\"$docker_build_http_proxy\" https_proxy=\"$docker_build_https_proxy\" all_proxy=\"$docker_build_all_proxy\" no_proxy=\"$docker_build_no_proxy\" docker build --network \"$docker_build_network_mode\" --build-arg HTTP_PROXY=\"$docker_build_http_proxy\" --build-arg HTTPS_PROXY=\"$docker_build_https_proxy\" --build-arg ALL_PROXY=\"$docker_build_all_proxy\" --build-arg NO_PROXY=\"$docker_build_no_proxy\" --build-arg http_proxy=\"$docker_build_http_proxy\" --build-arg https_proxy=\"$docker_build_https_proxy\" --build-arg all_proxy=\"$docker_build_all_proxy\" --build-arg no_proxy=\"$docker_build_no_proxy\" -f Dockerfile.web-probe-sentinel -t \"$image_ref\" . > /tmp/web-probe-sentinel-docker-build.log 2>&1; then cat /tmp/web-probe-sentinel-docker-build.log; emit_stage docker-build failed \"$docker_build_started_ms\"; exit 1; fi", - "cat /tmp/web-probe-sentinel-docker-build.log", - "docker_build_finished_ms=$(node -e 'console.log(Date.now())')", - "emit_stage docker-build succeeded \"$docker_build_started_ms\"", - "docker_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' /tmp/web-probe-sentinel-docker-build.log 2>/dev/null || true)", - "docker_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' /tmp/web-probe-sentinel-docker-build.log 2>/dev/null || true)", - "docker_build_log_tail_b64=$(tail -n 30 /tmp/web-probe-sentinel-docker-build.log 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')", - "docker_push_started_ms=$(node -e 'console.log(Date.now())')", - "emit_stage docker-push running \"$docker_push_started_ms\"", - "if ! docker push \"$image_ref\" > /tmp/web-probe-sentinel-docker-push.log 2>&1; then cat /tmp/web-probe-sentinel-docker-push.log; emit_stage docker-push failed \"$docker_push_started_ms\"; exit 1; fi", - "cat /tmp/web-probe-sentinel-docker-push.log", - "docker_push_finished_ms=$(node -e 'console.log(Date.now())')", - "emit_stage docker-push succeeded \"$docker_push_started_ms\"", - "tag=${image_ref##*:}", - "repo_no_tag=${image_ref%:*}", - "registry_path=${repo_no_tag#127.0.0.1:5000/}", - "digest=$(awk '/digest: sha256:/ {print $3; exit}' /tmp/web-probe-sentinel-docker-push.log)", - "if [ -z \"$digest\" ]; then digest=$(curl -fsSI --max-time 10 \"http://127.0.0.1:5000/v2/$registry_path/manifests/$tag\" 2>/dev/null | awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}'); fi", + "context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')", + "write_meta context_ignore_entries \"$context_ignore_entries\"", + "chmod -R a+rwX /workspace", + "trap - EXIT", + ].join("\n"); +} + +function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string { + const monitorWeb = record(state.image.monitorWeb); + const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode"); + const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode"); + const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource"); + const imageBuildProxy = state.spec.networkProfile.imageBuildProxy; + const imageBuildNoProxy = imageBuildProxy.noProxy.join(","); + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `image_ref=${shellQuote(state.image.ref)}`, + `image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`, + `image_build_package_mode=${shellQuote(imageBuildPackageMode)}`, + `image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`, + `image_build_proxy_source=${shellQuote(imageBuildProxySource)}`, + `image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`, + `image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`, + `image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`, + `image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`, + "meta_dir=/workspace/meta", + "event_log=/workspace/publish-events.log", + "build_log=/workspace/image-build.log", + "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", + "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", + "trap emit_failed EXIT", + "cd /workspace/source", + "image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi", + "image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi", + "image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi", + "image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi", + "write_meta image_build_builder \"$image_build_builder\"", + "write_meta image_build_package_mode \"$image_build_package_mode\"", + "write_meta image_build_network_mode \"$image_build_network_mode\"", + "write_meta image_build_proxy_source \"$image_build_proxy_source\"", + "write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"", + "write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"", + "write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"", + "write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"", + "image_build_started_ms=$(now_ms)", + "write_meta image_build_started_ms \"$image_build_started_ms\"", + "emit_stage image-build running \"$image_build_started_ms\"", + "if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi", + "cat \"$build_log\"", + "image_build_finished_ms=$(now_ms)", + "write_meta image_build_finished_ms \"$image_build_finished_ms\"", + "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)", + "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)", "test -n \"$digest\"", + "repo_no_tag=${image_ref%:*}", "digest_ref=\"$repo_no_tag@$digest\"", + "write_meta digest \"$digest\"", + "write_meta digest_ref \"$digest_ref\"", + "emit_stage image-build succeeded \"$image_build_started_ms\"", + "chmod -R a+rwX /workspace", + "trap - EXIT", + ].join("\n"); +} + +function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { + const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : []; + const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ + path: file.path, + contentBase64: Buffer.from(file.content, "utf8").toString("base64"), + }))), "utf8").toString("base64"); + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `image_ref=${shellQuote(state.image.ref)}`, + `image_repository=${shellQuote(state.image.repository)}`, + `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`, + `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`, + `files_b64=${shellQuote(filesB64)}`, + "meta_dir=/workspace/meta", + "event_log=/workspace/publish-events.log", + "build_log=/workspace/image-build.log", + "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", + "read_meta() { cat \"$meta_dir/$1\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }", + "trap emit_failed EXIT", + "if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi", + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", + "started_ms=$(read_meta started_ms)", + "source_commit=$(read_meta source_commit)", + "mirror_commit=$(read_meta mirror_commit)", + "source_fetch_started_ms=$(read_meta source_fetch_started_ms)", + "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)", + "monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)", + "monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)", + "image_build_started_ms=$(read_meta image_build_started_ms)", + "image_build_finished_ms=$(read_meta image_build_finished_ms)", + "digest_ref=$(read_meta digest_ref)", + "env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')", + "env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')", + "env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)", + "env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)", + "env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)", + "image_build_builder=$(read_meta image_build_builder)", + "image_build_package_mode=$(read_meta image_build_package_mode)", + "image_build_network_mode=$(read_meta image_build_network_mode)", + "image_build_proxy_source=$(read_meta image_build_proxy_source)", + "image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)", + "image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)", + "image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)", + "image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)", + "context_ignore_entries=$(read_meta context_ignore_entries)", + "image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)", + "image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)", + "image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')", "gitops_commit=''", "changed=false", "file_count=0", - "gitops_started_ms=$(node -e 'console.log(Date.now())')", + "gitops_started_ms=$(now_ms)", "if [ \"$files_b64\" != \"W10=\" ]; then", " emit_stage gitops running \"$gitops_started_ms\"", " gitops_cache=\"/cache/${gitops_repository}.git\"", @@ -1840,13 +1993,13 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "else", " emit_stage gitops skipped \"$gitops_started_ms\"", "fi", - "gitops_finished_ms=$(node -e 'console.log(Date.now())')", - "finished_ms=$(node -e 'console.log(Date.now())')", - "node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$docker_build_started_ms\" \"$docker_build_finished_ms\" \"$docker_push_started_ms\" \"$docker_push_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$docker_build_cache_hits\" \"$docker_build_step_lines\" \"$docker_build_log_tail_b64\" \"$docker_build_package_mode\" \"$docker_build_network_mode\" \"$docker_build_proxy_source\" \"$docker_build_http_proxy_present\" \"$docker_build_https_proxy_present\" \"$docker_build_all_proxy_present\" \"$docker_build_no_proxy_present\" \"$docker_ignore_entries\" <<'NODE'", - "const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, dockerBuildStartedMs, dockerBuildFinishedMs, dockerPushStartedMs, dockerPushFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, dockerBuildCacheHits, dockerBuildStepLines, dockerBuildLogTailB64, dockerBuildPackageMode, dockerBuildNetworkMode, dockerBuildProxySource, dockerBuildHttpProxyPresent, dockerBuildHttpsProxyPresent, dockerBuildAllProxyPresent, dockerBuildNoProxyPresent, dockerIgnoreEntries] = process.argv.slice(2);", + "gitops_finished_ms=$(now_ms)", + "finished_ms=$(now_ms)", + "node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'", + "const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);", "const elapsed = (start, finish) => Number(finish) - Number(start);", - "const cacheHits = Number(dockerBuildCacheHits || 0);", - "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), dockerBuildMs: elapsed(dockerBuildStartedMs, dockerBuildFinishedMs), dockerPushMs: elapsed(dockerPushStartedMs, dockerPushFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, dockerBuild: { cacheHitLines: cacheHits, stepLines: Number(dockerBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: dockerBuildPackageMode, networkMode: dockerBuildNetworkMode, proxySource: dockerBuildProxySource, proxy: { httpProxyPresent: dockerBuildHttpProxyPresent === 'true', httpsProxyPresent: dockerBuildHttpsProxyPresent === 'true', allProxyPresent: dockerBuildAllProxyPresent === 'true', noProxyPresent: dockerBuildNoProxyPresent === 'true', valuesRedacted:true }, dockerIgnoreEntries: Number(dockerIgnoreEntries || 0), verifyLocation: 'pre-docker-build', logTail: Buffer.from(dockerBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'docker-build', 'docker-push', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", + "const cacheHits = Number(imageBuildCacheHits || 0);", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", "NODE", "trap - EXIT", ].join("\n"); @@ -1900,7 +2053,7 @@ function probeK8sJobScript(namespace: string, jobName: string): string { "pod_phase=''", "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", "logs_tail=''", - "if [ -n \"$pod\" ]; then logs_tail=$(kubectl -n \"$namespace\" logs \"$pod\" --tail=120 2>/dev/null | tail -c 12000 | base64 | tr -d '\\n'); fi", + "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null || true; done; } | tail -c 16000 | base64 | tr -d '\\n'); fi", "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", @@ -1937,7 +2090,7 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel : `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, logs: result.jobName === "-" ? "-" - : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --tail=120`, + : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`, describe: result.jobName === "-" ? "-" : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`, @@ -1994,7 +2147,7 @@ function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: rea const running = reversed.find((event) => event.status === "running"); if (running !== undefined) return text(running.stage); const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage))); - const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "docker-build", "docker-push", "gitops"] : ["source-mirror-fetch"]; + const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"]; const next = order.find((stage) => !completed.has(stage)); return next ?? result.phase; } @@ -2640,11 +2793,11 @@ function renderPublishResult(publish: Record): string { const diagnostics = record(publish.diagnostics); const diagnosticEnvReuse = record(diagnostics.envReuse); const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse; - const dockerBuild = record(payload.dockerBuild); - const dockerBuildProxy = record(dockerBuild.proxy); + const imageBuild = record(payload.imageBuild); + const imageBuildProxy = record(imageBuild.proxy); const timings = record(payload.stageTimings); const commands = record(diagnostics.commands); - const proxySummary = [dockerBuildProxy.httpProxyPresent, dockerBuildProxy.httpsProxyPresent, dockerBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none"; + const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none"; const lines = [ "PUBLISH", table(["OK", "PHASE", "JOB", "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[ @@ -2672,22 +2825,22 @@ function renderPublishResult(publish: Record): string { ]]), ); } - if (Object.keys(dockerBuild).length > 0 || Object.keys(timings).length > 0) { + if (Object.keys(imageBuild).length > 0 || Object.keys(timings).length > 0) { lines.push( "", "PUBLISH_BUILD", - table(["PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "BUILD_MS", "PUSH_MS", "GITOPS_MS", "TOTAL_MS"], [[ - dockerBuild.packageMode ?? "-", - dockerBuild.networkMode ?? "-", + table(["BUILDER", "PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "TOTAL_MS"], [[ + imageBuild.builder ?? "-", + imageBuild.packageMode ?? "-", + imageBuild.networkMode ?? "-", proxySummary, - dockerBuild.dockerIgnoreEntries ?? "-", - dockerBuild.layerCache ?? "-", - dockerBuild.cacheHitLines ?? "-", - dockerBuild.stepLines ?? "-", + imageBuild.contextIgnoreEntries ?? "-", + imageBuild.layerCache ?? "-", + imageBuild.cacheHitLines ?? "-", + imageBuild.stepLines ?? "-", timings.sourceFetchMs ?? "-", timings.monitorWebVerifyMs ?? "-", - timings.dockerBuildMs ?? "-", - timings.dockerPushMs ?? "-", + timings.imageBuildMs ?? "-", timings.gitopsMs ?? "-", timings.totalMs ?? payload.elapsedMs ?? "-", ]]), @@ -2742,7 +2895,7 @@ function renderImageResult(result: Record): string { "", table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]), "", - Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "BUILD_PKG", "BUILD_NET", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.dockerBuildPackageMode, monitorWeb.dockerBuildNetworkMode, monitorWeb.dockerBuildContextIgnore]]), + Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, monitorWeb.imageBuildContextIgnore]]), "", Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), "",