diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 7503769c..7a9906a4 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -177,10 +177,12 @@ function runSentinelImage(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = "web-probe sentinel image build"; - const publish = runSentinelPublishJob(state, false, options.timeoutSeconds); + const sourceMirrorSync = runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); + const publish = sourceMirrorSync.ok === true + ? runSentinelPublishJob(state, false, options.timeoutSeconds) + : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const registry = probeImageRegistry(state, options.timeoutSeconds); const registryReady = record(registry.probe).present === true; + const ok = state.configReady && state.sourceHead.ok && sourceMirrorSync.ok === true && publish.ok === true && registryReady; const result = { - ok: state.configReady && state.sourceHead.ok && publish.ok === true && registryReady, + ok, command, node: state.spec.nodeId, lane: state.spec.lane, @@ -527,9 +536,19 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra source: state.sourceHead, image: state.image, registry, + sourceMirrorSync, publish, - warnings: sentinelElapsedWarnings(record(publish).elapsedMs), - blocker: null, + warnings: [ + ...sentinelElapsedWarnings(record(sourceMirrorSync).elapsedMs), + ...sentinelElapsedWarnings(record(publish).elapsedMs), + ], + blocker: ok + ? null + : sourceMirrorSync.ok !== true + ? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" } + : publish.ok !== true + ? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" } + : { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" }, next: { status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`, @@ -542,7 +561,12 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const command = `web-probe sentinel control-plane ${options.action}`; const applyOnly = options.action === "apply"; - const publish = applyOnly ? null : runSentinelPublishJob(state, true, options.timeoutSeconds); + const sourceMirrorSync = applyOnly ? null : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); + const publish = applyOnly + ? null + : record(sourceMirrorSync).ok === true + ? runSentinelPublishJob(state, true, options.timeoutSeconds) + : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const flush = !applyOnly && record(publish).ok === true ? runChildCli(["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"], options.timeoutSeconds) : null; @@ -564,6 +588,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext const targetValidationOk = applyOnly || record(targetValidation).ok === true; const ok = state.configReady && state.sourceHead.ok + && (applyOnly || record(sourceMirrorSync).ok === true) && (applyOnly || record(publish).ok === true) && (applyOnly || record(flush).ok === true) && record(publicExposureApply).ok === true @@ -571,9 +596,13 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext && observedReady && targetValidationOk; const blocker = ok ? null : { - code: targetValidationOk ? "sentinel-control-plane-not-ready" : "sentinel-target-validation-failed", + code: targetValidationOk + ? record(sourceMirrorSync).ok === false ? "sentinel-source-mirror-sync-failed" : "sentinel-control-plane-not-ready" + : "sentinel-target-validation-failed", reason: targetValidationOk - ? "one or more publish, publicExposure, Argo or runtime observation checks did not pass" + ? record(sourceMirrorSync).ok === false + ? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish" + : "one or more publish, publicExposure, Argo or runtime observation checks did not pass" : text(record(targetValidation).failure ?? record(targetValidation).reason ?? "quick verify did not pass"), }; const result = { @@ -607,6 +636,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext objects: manifestObjectSummary(state.manifests), sha256: state.manifestSha256, }, + sourceMirrorSync, publish, flush, publicExposureApply, @@ -614,6 +644,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext observed, targetValidation, warnings: Array.from(new Set([ + ...sentinelElapsedWarnings(record(sourceMirrorSync).elapsedMs), ...sentinelElapsedWarnings(record(publish).elapsedMs), ...sentinelElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs), ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), @@ -688,21 +719,6 @@ function sentinelObservedReady(value: Record | SentinelObserved } function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record { - const sourceMode = stringAt(state.cicd, "builder.sourceMode"); - if (sourceMode === "sparse-git-checkout") { - return { - ok: state.sourceHead.ok, - probe: { - mode: sourceMode, - commit: state.sourceHead.commit, - expectedCommit: state.sourceHead.commit, - persistentMirrorPresent: false, - source: "commit-pinned sparse checkout declared in config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml#sentinel.cicd.source.checkoutPaths", - valuesRedacted: true, - }, - result: { exitCode: 0, timedOut: false, stdoutBytes: 0, stderrBytes: 0, stdoutPreview: "sourceMode=sparse-git-checkout", stderrPreview: "" }, - }; - } const namespace = stringAt(state.cicd, "builder.namespace"); const repository = stringAt(state.cicd, "source.repository"); const branch = stringAt(state.cicd, "source.branch"); @@ -717,10 +733,10 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re "node - \"$rc\" \"$commit\" \"$expected\" \"$repo_path\" \"$branch\" <<'NODE'", "const [rc, commit, expected, repoPath, branch] = process.argv.slice(2);", "const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');", - "console.log(JSON.stringify({ ok: present && (!expected || commit === expected), present, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, valuesRedacted: true }));", + "console.log(JSON.stringify({ ok: present && (!expected || commit === expected), mode: 'internal-git-mirror', present, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, persistentMirrorPresent: present, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));", "NODE", ].join("\n"); - const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) }; } @@ -835,6 +851,244 @@ function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Re return `${state.image.repository}@${digest}`; } +function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult { + const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`; + const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); + const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName); + const namespace = stringAt(state.cicd, "builder.namespace"); + sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); + const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + if (created.exitCode !== 0) { + sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane }); + return { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }; + } + const startedAt = Date.now(); + const timeoutMs = Math.max(30_000, Math.min(timeoutSeconds * 1000, 900_000)); + let polls = 0; + let lastProbe: Record = {}; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const probe = parseJsonObject(probeCapture.stdout) ?? {}; + lastProbe = { ...probe, capture: compactCommand(probeCapture) }; + const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); + sentinelProgressEvent("sentinel.source-mirror.progress", { + phase: "remote-job", + status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", + jobName, + polls, + elapsedMs: Date.now() - startedAt, + pod: probe.pod ?? null, + sourceCommit: state.sourceHead.commit, + node: state.spec.nodeId, + lane: state.spec.lane, + }); + if (probe.succeeded === true) { + const ok = payload.ok === true; + return { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + } + if (probe.failed === true) { + return { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + } + if (Date.now() - startedAt > 120_000) sentinelProgressEvent("sentinel.source-mirror.warning", { warning: "source mirror sync exceeded 120s; investigate env-reuse/git mirror/source build path", jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); + runCommand(["sleep", "5"], repoRoot, { timeoutMs: 6_000 }); + } + return { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; +} + +function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult { + return { + ok: false, + phase, + jobName: "-", + payload: { ok: false, status: phase, reason, valuesRedacted: true }, + valuesRedacted: true, + }; +} + +function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record { + const namespace = stringAt(state.cicd, "builder.namespace"); + const labels = { + "app.kubernetes.io/name": "web-probe-sentinel-source-mirror", + "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", + "unidesk.ai/spec-ref": "PJ2026-01060508", + "unidesk.ai/node": state.spec.nodeId, + "unidesk.ai/lane": state.spec.lane, + }; + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { name: jobName, namespace, labels }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"), + ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"), + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + volumes: [ + { name: "cache", hostPath: { path: stringAt(state.controlPlaneTarget, "gitMirror.cacheHostPath"), type: "DirectoryOrCreate" } }, + { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, + ], + containers: [{ + name: "sync", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)], + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }], + }, + }, + }, + }; +} + +function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string { + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`, + `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, + `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`, + `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, + "started_ms=$(node -e 'console.log(Date.now())')", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", + "trap emit_failed EXIT", + "test -n \"$source_commit\"", + ...sentinelSourceMirrorSshSetupShellLines(state), + "repo=\"/cache/${source_repository}.git\"", + "mkdir -p \"$(dirname \"$repo\")\"", + "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", + " git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", + "else", + " rm -rf \"$repo\"", + " git init --bare \"$repo\"", + " git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", + "fi", + "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", + "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", + "git --git-dir=\"$repo\" config http.uploadpack true", + "git --git-dir=\"$repo\" config http.receivepack true", + "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"", + "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")", + "test \"$mirror_commit\" = \"$source_commit\"", + "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"", + "git --git-dir=\"$repo\" update-server-info", + "finished_ms=$(node -e 'console.log(Date.now())')", + "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$started_ms\" \"$finished_ms\" <<'NODE'", + "const [jobName, repository, branch, sourceCommit, mirrorCommit, startedMs, finishedMs] = process.argv.slice(2);", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));", + "NODE", + "trap - EXIT", + ].join("\n"); +} + +function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] { + const proxy = record(valueAtPath(state.controlPlaneNode, "egressProxy")); + const serviceName = nonEmptyString(proxy.serviceName); + const namespace = nonEmptyString(proxy.namespace); + const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null; + const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : ""; + const useProxy = serviceName !== null && namespace !== null && port !== null; + if (!useProxy) { + return [ + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2", + "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy", + "export NO_PROXY='*'", + "export no_proxy='*'", + "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", + "#!/bin/sh", + "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 \"$@\"", + "SH_PROXY", + "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", + "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", + "unset GIT_SSH_COMMAND", + ]; + } + const proxyHost = `${serviceName}.${namespace}.svc.cluster.local`; + const proxyUrl = `http://${proxyHost}:${port}`; + const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${port} %h %p`; + return [ + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + `printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${port} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`, + `export HTTP_PROXY=${shellQuote(proxyUrl)}`, + `export HTTPS_PROXY=${shellQuote(proxyUrl)}`, + `export ALL_PROXY=${shellQuote(proxyUrl)}`, + `export http_proxy=${shellQuote(proxyUrl)}`, + `export https_proxy=${shellQuote(proxyUrl)}`, + `export all_proxy=${shellQuote(proxyUrl)}`, + `export NO_PROXY=${shellQuote(noProxy)}`, + `export no_proxy=${shellQuote(noProxy)}`, + "cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'", + "#!/usr/bin/env node", + "const net = require('node:net');", + "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", + "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", + "const targetPort = Number.parseInt(targetPortRaw || '', 10);", + "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", + " console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');", + " process.exit(64);", + "}", + "let settled = false;", + "let tunnelEstablished = false;", + "function finish(code, message) {", + " if (settled) return;", + " settled = true;", + " if (message) console.error('sentinel source-mirror proxy-connect: ' + message);", + " process.exit(code);", + "}", + "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", + "let buffer = Buffer.alloc(0);", + "socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", + "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", + "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", + "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", + "function onData(chunk) {", + " buffer = Buffer.concat([buffer, chunk]);", + " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", + " if (headerEnd === -1 && buffer.length < 8192) return;", + " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", + " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", + " const statusLine = head.split('\\r\\n', 1)[0] || '';", + " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", + " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", + " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", + " socket.destroy();", + " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", + " return;", + " }", + " socket.off('data', onData);", + " socket.setTimeout(0);", + " tunnelEstablished = true;", + " const rest = buffer.slice(headerEnd + 4);", + " if (rest.length) process.stdout.write(rest);", + " process.stdin.on('error', () => {});", + " process.stdout.on('error', () => {});", + " process.stdin.pipe(socket);", + " socket.pipe(process.stdout);", + "}", + "socket.on('data', onData);", + "NODE_PROXY", + "chmod 0700 /tmp/sentinel-github-proxy-connect.cjs", + "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", + "#!/bin/sh", + `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shellQuote(proxyCommand)} "$@"`, + "SH_PROXY", + "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", + "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", + "unset GIT_SSH_COMMAND", + ]; +} + function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult { const jobName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); const manifest = sentinelPublishJobManifest(state, jobName, publishGitops); @@ -940,7 +1194,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish `job_name=${shellQuote(jobName)}`, `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`, `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, - `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`, + `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`, `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`, `image_ref=${shellQuote(state.image.ref)}`, @@ -2324,6 +2578,8 @@ function sentinelPipelineRunName(state: SentinelCicdState): string { function renderImageResult(result: Record): string { const source = record(result.source); + const sourceMirror = record(result.sourceMirror); + const sourceMirrorSync = record(result.sourceMirrorSync); const image = record(result.image); const registry = record(result.registry); const publish = record(result.publish); @@ -2337,10 +2593,14 @@ function renderImageResult(result: Record): string { "", table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]), "", + Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]), + "", table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]), "", Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), "", + Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]), + "", Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), @@ -2366,6 +2626,7 @@ function renderControlPlaneResult(result: Record): string { const argo = record(result.argo); const validation = record(result.validation); const observed = record(result.observed); + const sourceMirrorSync = record(result.sourceMirrorSync); const publish = record(result.publish); const flush = record(result.flush); const publicExposureApply = record(result.publicExposureApply); @@ -2388,6 +2649,8 @@ function renderControlPlaneResult(result: Record): string { "", renderObservedStatus(observed), "", + Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]), + "", Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ targetValidation.ok, targetValidation.status,