diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts new file mode 100644 index 00000000..45874957 --- /dev/null +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -0,0 +1,1122 @@ +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot. +// Responsibility: source mirror and publish remote Job/PipelineRun orchestration for web-probe sentinel CI/CD. +import { runCommand } from "./command"; +import { repoRoot, rootPath } from "./config"; +import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref"; +import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel"; +import { + compactCommand, + finiteNumberOrNull, + nonEmptyString, + numberAt, + numberAtNullable, + parseJsonObject, + record, + resolveSentinelChildJson, + safeKubernetesSegment, + sentinelCliSuffix, + sentinelSourceSnapshotStageRefPrefix, + shellQuote, + short, + stringAt, + stringAtNullable, + text, + valueAtPath, + type SentinelCicdState, + type SentinelRemoteJobResult, +} from "./hwlab-node-web-sentinel-cicd-shared"; + +export function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult { + const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`; + const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); + const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName); + const namespace = stringAt(state.cicd, "builder.namespace"); + sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); + const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + if (created.exitCode !== 0) { + sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane }); + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror"); + } + const startedAt = Date.now(); + const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); + const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000; + let slowWarningSent = false; + let polls = 0; + let lastProbe: Record = {}; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-source-mirror-job-probe"); + const probe = probeResolution.parsed ?? {}; + lastProbe = { ...probe, stdoutRecovery: probeResolution.diagnostics, capture: compactCommand(probeCapture) }; + const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); + sentinelProgressEvent("sentinel.source-mirror.progress", { + phase: "remote-job", + status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", + jobName, + polls, + elapsedMs: Date.now() - startedAt, + pod: probe.pod ?? null, + sourceCommit: state.sourceHead.commit, + node: state.spec.nodeId, + lane: state.spec.lane, + }); + if (probe.succeeded === true) { + const ok = payload.ok === true; + return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); + } + if (probe.failed === true) { + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); + } + if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { + slowWarningSent = true; + sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); + } + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + } + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); +} + +export function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult { + return { + ok: false, + phase, + jobName: "-", + payload: { ok: false, status: phase, reason, valuesRedacted: true }, + valuesRedacted: true, + }; +} + +function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record { + const namespace = stringAt(state.cicd, "builder.namespace"); + const labels = { + "app.kubernetes.io/name": "web-probe-sentinel-source-mirror", + "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", + "unidesk.ai/spec-ref": "PJ2026-01060508", + "unidesk.ai/node": state.spec.nodeId, + "unidesk.ai/lane": state.spec.lane, + }; + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { name: jobName, namespace, labels }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"), + ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"), + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + volumes: [ + sentinelGitMirrorCacheVolume(state), + { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, + ], + containers: [{ + name: "sync", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)], + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }], + }, + }, + }, + }; +} + +function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string { + return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit); +} + +function sentinelSourceMirrorSyncShellFromConfig(cicd: Record, controlPlaneNode: Record, jobName: string, selectedCommit: string | null): string { + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`, + `source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`, + `source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`, + `source_commit=${shellQuote(selectedCommit ?? "")}`, + `source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`, + "started_ms=$(node -e 'console.log(Date.now())')", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", + "trap emit_failed EXIT", + ...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode), + "repo=\"/cache/${source_repository}.git\"", + "mkdir -p \"$(dirname \"$repo\")\"", + "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", + " git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", + "else", + " rm -rf \"$repo\"", + " git init --bare \"$repo\"", + " git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", + "fi", + "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", + "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", + "git --git-dir=\"$repo\" config http.uploadpack true", + "git --git-dir=\"$repo\" config http.receivepack true", + "fetch_ok=0", + "for attempt in 1 2 3; do", + " if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi", + " code=$?", + " printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2", + " sleep $((attempt * 5))", + "done", + "test \"$fetch_ok\" = 1", + "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")", + "if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi", + "git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"", + "test \"$mirror_commit\" = \"$source_commit\"", + "stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"", + "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"", + "git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"", + "git --git-dir=\"$repo\" update-server-info", + "finished_ms=$(node -e 'console.log(Date.now())')", + "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'", + "const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));", + "NODE", + "trap - EXIT", + ].join("\n"); +} + +function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] { + return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode); +} + +function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record): string[] { + const proxy = record(valueAtPath(controlPlaneNode, "egressProxy")); + const serviceName = nonEmptyString(proxy.serviceName); + const namespace = nonEmptyString(proxy.namespace); + const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null; + const hostRouteProxyUrl = nonEmptyString(proxy.proxyUrl); + const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : ""; + let proxyHost: string | null = null; + let proxyPort: number | null = null; + let proxyUrl: string | null = null; + if (serviceName !== null && namespace !== null && port !== null) { + proxyHost = `${serviceName}.${namespace}.svc.cluster.local`; + proxyPort = port; + proxyUrl = `http://${proxyHost}:${proxyPort}`; + } else if (hostRouteProxyUrl !== null) { + try { + const parsed = new URL(hostRouteProxyUrl); + const parsedPort = Number.parseInt(parsed.port || "80", 10); + if ((parsed.protocol === "http:" || parsed.protocol === "https:") && parsed.hostname.length > 0 && Number.isInteger(parsedPort)) { + proxyHost = parsed.hostname; + proxyPort = parsedPort; + proxyUrl = hostRouteProxyUrl; + } + } catch { + proxyHost = null; + proxyPort = null; + proxyUrl = null; + } + } + const useProxy = proxyHost !== null && proxyPort !== null && proxyUrl !== null; + if (!useProxy) { + return [ + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2", + "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy", + "export NO_PROXY='*'", + "export no_proxy='*'", + "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", + "#!/bin/sh", + "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"", + "SH_PROXY", + "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", + "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", + "unset GIT_SSH_COMMAND", + ]; + } + const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; + return [ + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + `printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`, + `export HTTP_PROXY=${shellQuote(proxyUrl)}`, + `export HTTPS_PROXY=${shellQuote(proxyUrl)}`, + `export ALL_PROXY=${shellQuote(proxyUrl)}`, + `export http_proxy=${shellQuote(proxyUrl)}`, + `export https_proxy=${shellQuote(proxyUrl)}`, + `export all_proxy=${shellQuote(proxyUrl)}`, + `export NO_PROXY=${shellQuote(noProxy)}`, + `export no_proxy=${shellQuote(noProxy)}`, + "cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'", + "#!/usr/bin/env node", + "const net = require('node:net');", + "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", + "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", + "const targetPort = Number.parseInt(targetPortRaw || '', 10);", + "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", + " console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');", + " process.exit(64);", + "}", + "let settled = false;", + "let tunnelEstablished = false;", + "function finish(code, message) {", + " if (settled) return;", + " settled = true;", + " if (message) console.error('sentinel source-mirror proxy-connect: ' + message);", + " process.exit(code);", + "}", + "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", + "let buffer = Buffer.alloc(0);", + "socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", + "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", + "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", + "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", + "function onData(chunk) {", + " buffer = Buffer.concat([buffer, chunk]);", + " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", + " if (headerEnd === -1 && buffer.length < 8192) return;", + " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", + " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", + " const statusLine = head.split('\\r\\n', 1)[0] || '';", + " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", + " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", + " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", + " socket.destroy();", + " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", + " return;", + " }", + " socket.off('data', onData);", + " socket.setTimeout(0);", + " tunnelEstablished = true;", + " const rest = buffer.slice(headerEnd + 4);", + " if (rest.length) process.stdout.write(rest);", + " process.stdin.on('error', () => {});", + " process.stdout.on('error', () => {});", + " process.stdin.pipe(socket);", + " socket.pipe(process.stdout);", + "}", + "socket.on('data', onData);", + "NODE_PROXY", + "chmod 0700 /tmp/sentinel-github-proxy-connect.cjs", + "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", + "#!/bin/sh", + `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`, + "SH_PROXY", + "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", + "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", + "unset GIT_SSH_COMMAND", + ]; +} + +export function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult { + const pipelineRunName = sentinelPipelineRunName(state, rerun); + const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops); + const namespace = stringAt(state.cicd, "builder.namespace"); + sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); + const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + if (created.exitCode !== 0) { + sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish"); + } + sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); + const startedAt = Date.now(); + const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); + const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000; + let slowWarningSent = false; + let polls = 0; + let lastProbe: Record = {}; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-publish-pipelinerun-probe"); + const probe = probeResolution.parsed ?? {}; + lastProbe = { ...probe, stdoutRecovery: probeResolution.diagnostics, capture: compactCommand(probeCapture) }; + const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); + sentinelProgressEvent("sentinel.publish.progress", { + phase: "remote-pipelinerun", + status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", + pipelineRun: pipelineRunName, + publishGitops, + polls, + elapsedMs: Date.now() - startedAt, + taskRun: probe.taskRun ?? null, + pod: probe.pod ?? null, + sourceCommit: state.sourceHead.commit, + node: state.spec.nodeId, + lane: state.spec.lane, + }); + if (probe.succeeded === true) { + const ok = payload.ok === true; + return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); + } + if (probe.failed === true) { + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); + } + if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { + slowWarningSent = true; + sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); + } + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + } + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); +} + +function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record { + const namespace = stringAt(state.cicd, "builder.namespace"); + const buildkitImage = requireSentinelBuildkitImage(state); + const proxyEnv = sentinelImageBuildProxyEnv(state); + const labels = { + "app.kubernetes.io/name": "web-probe-sentinel-publish", + "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", + "unidesk.ai/spec-ref": "PJ2026-01060508", + "unidesk.ai/node": state.spec.nodeId, + "unidesk.ai/lane": state.spec.lane, + "unidesk.ai/ci-system": "tekton", + }; + return { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + name: pipelineRunName, + namespace, + labels, + annotations: { + "unidesk.ai/source-commit": state.sourceHead.commit ?? "", + "unidesk.ai/source-authority": state.sourceHead.sourceAuthority, + "unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "", + "unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"), + "unidesk.ai/publish-gitops": publishGitops ? "true" : "false", + }, + }, + spec: { + timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` }, + taskRunTemplate: { + podTemplate: { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 }, + }, + }, + pipelineSpec: { + tasks: [{ + name: "publish", + taskSpec: { + volumes: [ + sentinelGitMirrorCacheVolume(state), + { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, + { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, + sentinelBuildkitStateVolume(state), + { name: "tmp", emptyDir: {} }, + ], + steps: [ + { + name: "source", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + env: proxyEnv, + script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)), + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }, + { + name: "prepare-buildkit-state", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"), + securityContext: { runAsUser: 0, runAsGroup: 0 }, + volumeMounts: [ + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + ], + }, + { + name: "image-build", + image: buildkitImage, + imagePullPolicy: "IfNotPresent", + env: [ + ...proxyEnv, + { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }, + ], + script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)), + securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + { name: "tmp", mountPath: "/tmp" }, + ], + }, + { + name: "publish", + image: state.image.baseImage, + imagePullPolicy: "IfNotPresent", + env: proxyEnv, + script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)), + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + ], + }, + ], + }, + }], + }, + }, + }; +} + +function tektonShellScript(body: string): string { + return `#!/bin/sh\n${body}`; +} + +function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record { + return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget); +} + +function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record): Record { + const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath")); + if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } }; + return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } }; +} + +function sentinelBuildkitStateVolume(state: SentinelCicdState): Record { + const buildkitState = monitorWebBuildkitStatePlan(state.cicd); + const mode = stringAt(buildkitState, "mode"); + if (mode === "hostPath") { + return { + name: "buildkit-state", + hostPath: { + path: stringAt(buildkitState, "path"), + type: stringAt(buildkitState, "type"), + }, + }; + } + if (mode === "persistentVolumeClaim") { + return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } }; + } + if (mode === "emptyDir") { + return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } }; + } + throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); +} + +function requireSentinelBuildkitImage(state: SentinelCicdState): string { + const image = state.spec.buildkit?.sidecarImage; + if (typeof image !== "string" || image.length === 0) { + throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`); + } + return image; +} + +function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> { + const proxy = state.spec.networkProfile.imageBuildProxy; + const noProxy = proxy.noProxy.join(","); + return [ + { name: "HTTP_PROXY", value: proxy.http }, + { name: "http_proxy", value: proxy.http }, + { name: "HTTPS_PROXY", value: proxy.https }, + { name: "https_proxy", value: proxy.https }, + { name: "ALL_PROXY", value: proxy.all }, + { name: "all_proxy", value: proxy.all }, + { name: "NO_PROXY", value: noProxy }, + { name: "no_proxy", value: noProxy }, + ]; +} + +function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string { + const monitorWeb = record(state.image.monitorWeb); + const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => { + if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths"); + return item; + })), "utf8").toString("base64"); + const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64"); + const envReuseMode = stringAt(monitorWeb, "envReuseMode"); + const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath"); + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`, + `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, + `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`, + `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, + `source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`, + `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`, + `dockerfile_b64=${shellQuote(dockerfileB64)}`, + `env_reuse_mode=${shellQuote(envReuseMode)}`, + `env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`, + "meta_dir=/workspace/meta", + "event_log=/workspace/publish-events.log", + "worktree=/workspace/source", + "rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json", + "mkdir -p \"$meta_dir\"", + ": > \"$event_log\"", + "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", + "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", + "trap emit_failed EXIT", + "started_ms=$(now_ms)", + "write_meta started_ms \"$started_ms\"", + "write_meta source_commit \"$source_commit\"", + "write_meta source_stage_ref \"$source_stage_ref\"", + "test -n \"$source_commit\"", + "test -n \"$source_stage_ref\"", + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", + "git init \"$worktree\"", + "cd \"$worktree\"", + "git remote add origin \"$source_git_url\"", + "git config core.sparseCheckout true", + "git config remote.origin.promisor true", + "git config remote.origin.partialclonefilter blob:none", + "CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'", + "const fs = require('node:fs');", + "const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));", + "fs.mkdirSync('.git/info', { recursive: true });", + "fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');", + "NODE", + "source_fetch_started_ms=$(now_ms)", + "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"", + "emit_stage source-fetch running \"$source_fetch_started_ms\"", + "git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"", + "git checkout --detach \"$source_commit\"", + "mirror_commit=$(git rev-parse HEAD)", + "test \"$mirror_commit\" = \"$source_commit\"", + "write_meta mirror_commit \"$mirror_commit\"", + "source_fetch_finished_ms=$(now_ms)", + "write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"", + "emit_stage source-fetch succeeded \"$source_fetch_started_ms\"", + "env_reuse_node_deps_present=false", + "env_reuse_node_deps_entries=0", + "if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi", + "env_reuse_linked_node_deps=0", + "rm -rf node_modules", + "if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi", + "write_meta env_reuse_mode \"$env_reuse_mode\"", + "write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"", + "write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"", + "write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"", + "write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"", + "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'", + "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", + "NODE", + "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"", + "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", + "NODE", + "monitor_web_verify_started_ms=$(now_ms)", + "write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"", + "emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"", + "if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi", + "cat /tmp/web-probe-sentinel-monitor-web-verify.log", + "monitor_web_verify_finished_ms=$(now_ms)", + "write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"", + "emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"", + "mkdir -p .unidesk-sentinel-bin", + "cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'", + "#!/bin/sh", + "exec bun /app/scripts/ssh-cli.ts \"$@\"", + "SH_TRANS", + "chmod 0755 .unidesk-sentinel-bin/trans", + "DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'", + "const fs = require('node:fs');", + "fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));", + "NODE", + "cat > .dockerignore <<'EOF_DOCKERIGNORE'", + ".git", + ".git/**", + ".state", + ".state/**", + "logs", + "logs/**", + "node_modules", + "node_modules/**", + "**/node_modules", + "**/node_modules/**", + "**/dist", + "**/dist/**", + "**/target", + "**/target/**", + "**/coverage", + "**/coverage/**", + "npm-debug.log*", + ".env", + ".env.*", + "EOF_DOCKERIGNORE", + "context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')", + "write_meta context_ignore_entries \"$context_ignore_entries\"", + "chmod -R a+rwX /workspace", + "trap - EXIT", + ].join("\n"); +} + +function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string { + const monitorWeb = record(state.image.monitorWeb); + const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode"); + const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode"); + const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource"); + const imageBuildProxy = state.spec.networkProfile.imageBuildProxy; + const imageBuildNoProxy = imageBuildProxy.noProxy.join(","); + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `image_ref=${shellQuote(state.image.ref)}`, + `image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`, + `image_build_package_mode=${shellQuote(imageBuildPackageMode)}`, + `image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`, + `image_build_proxy_source=${shellQuote(imageBuildProxySource)}`, + `image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`, + `image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`, + `image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`, + `image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`, + "meta_dir=/workspace/meta", + "event_log=/workspace/publish-events.log", + "build_log=/workspace/image-build.log", + "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", + "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", + "trap emit_failed EXIT", + "cd /workspace/source", + "image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi", + "image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi", + "image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi", + "image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi", + "write_meta image_build_builder \"$image_build_builder\"", + "write_meta image_build_package_mode \"$image_build_package_mode\"", + "write_meta image_build_network_mode \"$image_build_network_mode\"", + "write_meta image_build_proxy_source \"$image_build_proxy_source\"", + "write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"", + "write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"", + "write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"", + "write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"", + "image_build_started_ms=$(now_ms)", + "write_meta image_build_started_ms \"$image_build_started_ms\"", + "emit_stage image-build running \"$image_build_started_ms\"", + "if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi", + "cat \"$build_log\"", + "image_build_finished_ms=$(now_ms)", + "write_meta image_build_finished_ms \"$image_build_finished_ms\"", + "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)", + "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)", + "test -n \"$digest\"", + "repo_no_tag=${image_ref%:*}", + "digest_ref=\"$repo_no_tag@$digest\"", + "write_meta digest \"$digest\"", + "write_meta digest_ref \"$digest_ref\"", + "emit_stage image-build succeeded \"$image_build_started_ms\"", + "trap - EXIT", + ].join("\n"); +} + +function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { + const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : []; + const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ + path: file.path, + contentBase64: Buffer.from(file.content, "utf8").toString("base64"), + }))), "utf8").toString("base64"); + return [ + "set -eu", + `job_name=${shellQuote(jobName)}`, + `image_ref=${shellQuote(state.image.ref)}`, + `image_repository=${shellQuote(state.image.repository)}`, + `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`, + `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`, + `files_b64=${shellQuote(filesB64)}`, + "meta_dir=/workspace/meta", + "event_log=/workspace/publish-events.log", + "build_log=/workspace/image-build.log", + "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", + "read_meta() { cat \"$meta_dir/$1\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }", + "trap emit_failed EXIT", + "if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi", + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", + "started_ms=$(read_meta started_ms)", + "source_commit=$(read_meta source_commit)", + "source_stage_ref=$(read_meta source_stage_ref)", + "mirror_commit=$(read_meta mirror_commit)", + "source_fetch_started_ms=$(read_meta source_fetch_started_ms)", + "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)", + "monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)", + "monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)", + "image_build_started_ms=$(read_meta image_build_started_ms)", + "image_build_finished_ms=$(read_meta image_build_finished_ms)", + "digest_ref=$(read_meta digest_ref)", + "env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')", + "env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')", + "env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)", + "env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)", + "env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)", + "image_build_builder=$(read_meta image_build_builder)", + "image_build_package_mode=$(read_meta image_build_package_mode)", + "image_build_network_mode=$(read_meta image_build_network_mode)", + "image_build_proxy_source=$(read_meta image_build_proxy_source)", + "image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)", + "image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)", + "image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)", + "image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)", + "context_ignore_entries=$(read_meta context_ignore_entries)", + "image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)", + "image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)", + "image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')", + "gitops_commit=''", + "changed=false", + "file_count=0", + "gitops_started_ms=$(now_ms)", + "if [ \"$files_b64\" != \"W10=\" ]; then", + " emit_stage gitops running \"$gitops_started_ms\"", + " gitops_cache=\"/cache/${gitops_repository}.git\"", + " gitops_worktree=\"/tmp/$job_name/gitops\"", + " git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"", + " cd \"$gitops_worktree\"", + " git fetch origin \"$gitops_branch\" || true", + " if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi", + " FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'", + "const fs = require('node:fs');", + "const path = require('node:path');", + "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", + "for (const file of files) {", + " const target = path.resolve(process.cwd(), file.path);", + " if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);", + " fs.mkdirSync(path.dirname(target), { recursive: true });", + " const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);", + " fs.writeFileSync(target, text);", + "}", + "console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));", + "NODE", + " git add .", + " file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')", + " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi", + " git push origin \"HEAD:refs/heads/$gitops_branch\"", + " gitops_commit=$(git rev-parse HEAD)", + " emit_stage gitops succeeded \"$gitops_started_ms\"", + "else", + " emit_stage gitops skipped \"$gitops_started_ms\"", + "fi", + "gitops_finished_ms=$(now_ms)", + "finished_ms=$(now_ms)", + "node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'", + "const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);", + "const elapsed = (start, finish) => Number(finish) - Number(start);", + "const cacheHits = Number(imageBuildCacheHits || 0);", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", + "NODE", + "trap - EXIT", + ].join("\n"); +} + +function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] { + const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application"); + return [{ + path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`, + content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`, + }]; +} + +function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record { + const app = state.manifests.find((item) => item.kind === "Application"); + if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true }; + const yaml = `${Bun.YAML.stringify(app).trim()}\n`; + const namespace = stringAt(state.cicd, "argo.namespace"); + const applicationName = stringAt(state.cicd, "argo.applicationName"); + const script = [ + "set -eu", + "tmp=$(mktemp)", + `cat >"$tmp" <<'YAML'\n${yaml}YAML`, + "kubectl apply -f \"$tmp\"", + `kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`, + ].join("\n"); + const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true }; +} + +function createK8sJobScript(namespace: string, manifest: Record): string { + const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`; + return [ + "set -eu", + `kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`, + "tmp=$(mktemp)", + `cat >"$tmp" <<'YAML'\n${yaml}YAML`, + "kubectl apply -f \"$tmp\"", + ].join("\n"); +} + +function createTektonPipelineRunScript(namespace: string, manifest: Record): string { + const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`; + const pipelineRunName = stringAt(manifest, "metadata.name"); + return [ + "set -eu", + `pipeline_run=${shellQuote(pipelineRunName)}`, + `namespace=${shellQuote(namespace)}`, + "tmp=$(mktemp)", + `cat >"$tmp" <<'YAML'\n${yaml}YAML`, + "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then", + " printf '%s\\n' \"sentinel publish PipelineRun already exists; reusing $pipeline_run\"", + " exit 0", + "fi", + "if ! kubectl create -f \"$tmp\"; then", + " if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then", + " printf '%s\\n' \"sentinel publish PipelineRun appeared concurrently; reusing $pipeline_run\"", + " exit 0", + " fi", + " exit 1", + "fi", + ].join("\n"); +} + +function probeK8sJobScript(namespace: string, jobName: string): string { + return [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `job=${shellQuote(jobName)}`, + "succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)", + "failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)", + "active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)", + "pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)", + "pod_phase=''", + "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", + "logs_tail=''", + "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi", + "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", + "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", + "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", + "NODE", + ].join("\n"); +} + +function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string { + return [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `pipeline_run=${shellQuote(pipelineRunName)}`, + "condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)", + "condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)", + "condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)", + "if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi", + "if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi", + "if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi", + "task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", + "pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", + "pod_phase=''", + "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi", + "logs_tail=''", + "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi", + "node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", + "const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);", + "const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');", + "const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));", + "console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", + "NODE", + ].join("\n"); +} + +function sentinelPayloadFromLogs(logsTail: string): Record { + const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const line = lines[index]; + if (!line.startsWith("{") || !line.endsWith("}")) continue; + const parsed = parseJsonObject(line); + if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed; + } + return {}; +} + +function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult { + return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true }; +} + +function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record { + const namespace = stringAt(state.cicd, "builder.namespace"); + const probe = record(result.probe); + const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : ""; + const events = sentinelStageEventsFromLogs(logsTail, domain); + const envReuse = sentinelEnvReuseFromLogs(logsTail); + const completedStages = sentinelCompletedStages(events, record(result.payload)); + const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs); + const currentPhase = sentinelCurrentRemotePhase(result, events, domain); + const isPipelineRun = result.resourceKind === "PipelineRun"; + const commands = { + cliStatus: domain === "publish" + ? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}` + : `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, + logs: result.jobName === "-" + ? "-" + : isPipelineRun + ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120` + : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`, + describe: result.jobName === "-" + ? "-" + : isPipelineRun + ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}` + : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`, + gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, + gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`, + gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`, + controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, + publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, + valuesRedacted: true, + }; + return { + domain, + resourceKind: result.resourceKind ?? "Job", + pipelineRun: isPipelineRun ? result.jobName : null, + taskRun: probe.taskRun ?? null, + currentPhase, + completedStages, + stageTimings, + envReuse, + pod: probe.pod ?? null, + podPhase: probe.podPhase ?? null, + active: probe.active ?? null, + conditionStatus: probe.conditionStatus ?? null, + conditionReason: probe.conditionReason ?? null, + recentLogSummary: sentinelRecentLogSummary(logsTail), + commands, + valuesRedacted: true, + }; +} + +function sentinelEnvReuseFromLogs(logsTail: string): Record | null { + const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const parsed = parseJsonObject(lines[index]); + if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true }; + } + return null; +} + +function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record[] { + const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage"; + return logsTail + .split(/\r?\n/u) + .map((line) => parseJsonObject(line.trim())) + .filter((item): item is Record => item !== null && item.event === expectedEvent); +} + +function sentinelCompletedStages(events: readonly Record[], payload: Record): string[] { + const completed = events + .filter((event) => event.status === "succeeded" || event.status === "skipped") + .map((event) => `${text(event.stage)}:${text(event.status)}`); + const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : []; + return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-"); +} + +function sentinelStageTimingSummary(events: readonly Record[], payload: Record, fallbackTotalMs: unknown): Record { + const payloadTimings = record(payload.stageTimings); + const eventElapsed = (stage: string): number | null => { + const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed")); + return event === undefined ? null : finiteNumberOrNull(event.elapsedMs); + }; + const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch"); + const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify"); + const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build"); + const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops"); + const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null); + const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0); + const totalMs = finiteNumberOrNull(payloadTimings.totalMs) + ?? finiteNumberOrNull(payload.elapsedMs) + ?? finiteNumberOrNull(fallbackTotalMs) + ?? summedTotalMs; + const result = { + sourceFetchMs, + monitorWebVerifyMs, + imageBuildMs, + gitopsMs, + totalMs, + valuesRedacted: true, + }; + return Object.values(result).some((item) => item !== null && item !== true) ? result : {}; +} + +function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record[], domain: "source-mirror" | "publish"): string { + if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed"; + if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase; + const reversed = [...events].reverse(); + const failed = reversed.find((event) => event.status === "failed"); + if (failed !== undefined) return text(failed.stage); + const running = reversed.find((event) => event.status === "running"); + if (running !== undefined) return text(running.stage); + const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage))); + const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"]; + const next = order.find((stage) => !completed.has(stage)); + return next ?? result.phase; +} + +function sentinelRecentLogSummary(logsTail: string): string { + const lines = logsTail + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith("{")) + .slice(-5) + .map((line) => short(line)); + return lines.length === 0 ? "-" : lines.join(" | "); +} + +export function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] { + const remote = record(job); + if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return []; + const diagnostics = record(remote.diagnostics); + const commands = record(diagnostics.commands); + return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`]; +} + +export function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] { + if (!observedReady) return []; + const warnings: string[] = []; + const publishRecord = record(publish); + if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) { + warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`); + } + const flushRecord = record(flush); + if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) { + warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout."); + } + return warnings; +} + +export function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number { + return numberAt(state.cicd, "confirmWait.maxSeconds"); +} + +export function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { + const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; + const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; + if (elapsedMs === null || elapsedMs <= budgetMs) return []; + return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`]; +} + +export function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] { + const sync = record(sourceMirrorSync); + if (sync.ok === true || state.sourceHead.ok !== true) return []; + return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`]; +} + +export function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record { + return { + ok: true, + phase: "already-present", + jobName: null, + probe: record(probe), + payload: { + ok: true, + status: "already-present", + sourceCommit: state.sourceHead.commit, + mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit, + stageRef: state.sourceHead.stageRef, + sourceAuthority: state.sourceHead.sourceAuthority, + valuesRedacted: true, + }, + polls: 0, + elapsedMs: 0, + valuesRedacted: true, + }; +} diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-shared.test.ts b/scripts/src/hwlab-node-web-sentinel-cicd-shared.test.ts new file mode 100644 index 00000000..81c6ce59 --- /dev/null +++ b/scripts/src/hwlab-node-web-sentinel-cicd-shared.test.ts @@ -0,0 +1,39 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { describe, expect, test } from "bun:test"; +import { resolveSentinelChildJson } from "./hwlab-node-web-sentinel-cicd-shared"; + +describe("sentinel CI/CD child JSON recovery", () => { + test("recovers remote probe JSON from trans truncation dump", () => { + const dir = join(tmpdir(), `unidesk-sentinel-cicd-${Date.now()}-${process.pid}`); + mkdirSync(dir, { recursive: true }); + const dumpPath = join(dir, "stdout.json"); + writeFileSync(dumpPath, JSON.stringify({ + ok: true, + present: true, + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + valuesRedacted: true, + })); + const summary = { + stdout: { + stream: "stdout", + truncated: true, + dumpPath, + valuesRedacted: true, + }, + valuesRedacted: true, + }; + const resolved = resolveSentinelChildJson({ + stdout: "tail fragment that is not json", + stderr: `UNIDESK_SSH_TRUNCATION_SUMMARY ${JSON.stringify(summary)}\n`, + exitCode: 0, + timedOut: false, + }, "web-probe-sentinel-test-probe"); + expect(resolved.parsed?.ok).toBe(true); + expect(resolved.parsed?.present).toBe(true); + expect(resolved.diagnostics.stdoutKind).toBe("ssh-truncation-summary"); + expect(resolved.diagnostics.dumpPath).toBe(dumpPath); + expect(resolved.diagnostics.source).toBe("dump"); + }); +}); diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts b/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts new file mode 100644 index 00000000..97e7a5d4 --- /dev/null +++ b/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts @@ -0,0 +1,986 @@ +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot. +// Responsibility: shared types, config helpers and text render utilities for web-probe sentinel CI/CD. +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery"; +import type { CommandResult } from "./command"; +import { rootPath } from "./config"; +import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; +import type { RenderedCliResult } from "./output"; + +export type WebProbeSentinelConfigAction = "plan" | "status"; +export type WebProbeSentinelImageAction = "status" | "build"; +export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current"; +export type WebProbeSentinelPublishAction = "publish-current"; +export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; +export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger"; +export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary"; + +export type WebProbeSentinelOptions = + | { + readonly kind: "config"; + readonly action: WebProbeSentinelConfigAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + } + | { + readonly kind: "image"; + readonly action: WebProbeSentinelImageAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + } + | { + readonly kind: "control-plane"; + readonly action: WebProbeSentinelControlPlaneAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + readonly rerun: boolean; + } + | { + readonly kind: "publish"; + readonly action: WebProbeSentinelPublishAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + readonly rerun: boolean; + } + | { + readonly kind: "maintenance"; + readonly action: WebProbeSentinelMaintenanceAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + readonly releaseId: string | null; + readonly reason: string | null; + readonly quickVerify: boolean; + } + | { + readonly kind: "validate"; + readonly action: "validate"; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly dryRun: boolean; + readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; + readonly quickVerify: boolean; + } + | { + readonly kind: "report"; + readonly action: "report"; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly view: WebProbeSentinelReportView; + readonly runId: string | null; + readonly latest: boolean; + readonly traceId: string | null; + readonly sampleSeq: number | null; + readonly raw: boolean; + readonly full: boolean; + readonly timeoutSeconds: number; + } + | { + readonly kind: "dashboard"; + readonly action: WebProbeSentinelDashboardAction; + readonly node: string; + readonly lane: string; + readonly sentinelId: string | null; + readonly viewport: string; + readonly localDir: string; + readonly name: string | null; + readonly runId: string | null; + readonly timeoutMs: number; + readonly waitTimeoutMs: number; + readonly timeoutSeconds: number; + readonly commandTimeoutSeconds: number; + readonly fullPage: boolean; + readonly raw: boolean; + }; + +export interface SentinelCicdState { + readonly spec: HwlabRuntimeLaneSpec; + readonly sentinelId: string; + readonly configRefs: Record; + readonly configReady: boolean; + readonly runtime: Record; + readonly cicd: Record; + readonly scenarios: unknown; + readonly publicExposure: Record; + readonly secrets: Record; + readonly controlPlaneTarget: Record; + readonly controlPlaneNode: Record; + readonly sourceHead: SourceHead; + readonly image: SentinelImagePlan; + readonly manifests: readonly Record[]; + readonly manifestSha256: string; + readonly valuesRedacted: true; +} + +export interface SourceHead { + readonly ok: boolean; + readonly repository: string; + readonly branch: string; + readonly commit: string | null; + readonly stageRef: string | null; + readonly mirrorCommit: string | null; + readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot"; + readonly latestDrift: boolean; + readonly result: CompactCommandResult; +} + +export interface SentinelImagePlan { + readonly repository: string; + readonly tag: string; + readonly ref: string; + readonly digestRef: string | null; + readonly baseImage: string; + readonly buildContext: string; + readonly entrypoint: string; + readonly dockerfileSha256: string; + readonly dockerfilePreview: string; + readonly monitorWeb: Record; +} + +export interface SentinelObservedStatus { + readonly sourceMirror: Record; + readonly registry: Record; + readonly gitMirror: Record; + readonly gitops: Record; + readonly argo: Record; + readonly runtime: Record; + readonly cadence: Record; + readonly wait?: Record; +} + +export interface SentinelObservedExpectation { + readonly gitopsRevision: string | null; + readonly runtimeImage: string | null; +} + +export interface SentinelRemoteJobResult { + readonly ok: boolean; + readonly phase: string; + readonly resourceKind?: "Job" | "PipelineRun"; + readonly jobName: string; + readonly payload: Record; + readonly polls?: number; + readonly elapsedMs?: number; + readonly create?: Record; + readonly probe?: Record; + readonly diagnostics?: Record; + readonly valuesRedacted: true; +} + +export interface CompactCommandResult { + readonly exitCode: number | null; + readonly timedOut: boolean; + readonly stdoutBytes: number; + readonly stderrBytes: number; + readonly stdoutPreview: string; + readonly stderrPreview: string; +} + +export interface ChildCliResult { + readonly ok: boolean; + readonly parsed: Record | null; + readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string }; +} + +export function sentinelSourceSnapshotStageRefPrefix(cicd: Record): string { + const branch = stringAt(cicd, "source.branch"); + const repository = stringAt(cicd, "source.repository"); + const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix") + .replaceAll("{branch}", branch) + .replaceAll("{repository}", repository) + .replace(/\/+$/u, ""); + if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix"); + return prefix; +} + +export function sentinelSourceSnapshotRef(cicd: Record, commit: string): string { + return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`; +} + +export function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record): Record { + return { + stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build", + runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge", + assetRoot: stringAtNullable(cicd, "monitorWeb.assetRoot") ?? "scripts/assets/web-probe-sentinel-monitor-web", + verifyCommand: "bun scripts/verify-web-probe-sentinel-monitor-web.ts", + gitMirrorReadUrl: stringAt(cicd, "source.gitMirrorReadUrl"), + sourceMode: stringAt(cicd, "builder.sourceMode"), + envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "k8s-buildkit-and-ci-node-deps", + envReuseNodeDepsPath: stringAtNullable(cicd, "monitorWeb.envReuse.nodeDepsPath") ?? "/opt/hwlab-ci-node-deps/node_modules", + verifyPhase: stringAtNullable(cicd, "monitorWeb.imageBuild.verifyPhase") ?? "pre-image-build", + imageBuildBuilder: spec.buildkit?.sidecarImage ?? null, + imageBuildPackageMode: stringAtNullable(cicd, "monitorWeb.imageBuild.packageMode") ?? "copy-only-containerfile", + imageBuildNetworkMode: monitorWebImageBuildNetworkMode(cicd), + imageBuildProxySource: stringAtNullable(cicd, "monitorWeb.imageBuild.proxySource") ?? "node.networkProfile.imageBuildProxy", + imageBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.imageBuild.contextIgnore") ?? "generated", + imageBuildState: monitorWebBuildkitStatePlan(cicd), + ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"), + valuesRedacted: true, + }; +} + +export function monitorWebImageBuildNetworkMode(cicd: Record): "default" | "host" { + const value = stringAtNullable(cicd, "monitorWeb.imageBuild.networkMode") ?? "host"; + if (value !== "default" && value !== "host") throw new Error(`monitorWeb.imageBuild.networkMode must be default or host, got ${value}`); + return value; +} + +export function monitorWebBuildkitStatePlan(cicd: Record): Record { + const state = recordTarget(valueAtPath(cicd, "monitorWeb.imageBuild.buildkitState"), "monitorWeb.imageBuild.buildkitState"); + const mode = stringAt(state, "mode"); + if (mode === "hostPath") { + return { + mode, + path: stringAt(state, "path"), + type: stringAt(state, "type"), + valuesRedacted: true, + }; + } + if (mode === "persistentVolumeClaim") { + return { + mode, + claimName: stringAt(state, "claimName"), + valuesRedacted: true, + }; + } + if (mode === "emptyDir") { + return { + mode, + sizeLimit: stringAt(state, "sizeLimit"), + valuesRedacted: true, + }; + } + throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); +} + +export function secretSourcePaths(sourceRef: string): string[] { + if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); + const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); + return [...new Set(paths)]; +} + +export function ownerFileSourcePaths(sourceRef: string): string[] { + if (sourceRef.includes("..") || sourceRef.includes("\0")) return []; + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot]; + return [...new Set(roots.map((root) => join(root, sourceRef)))]; +} + +export function parseEnvFile(textValue: string): Record { + const values: Record = {}; + for (const rawLine of textValue.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const index = line.indexOf("="); + if (index <= 0) continue; + const key = line.slice(0, index).trim(); + let value = line.slice(index + 1).trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1); + values[key] = value; + } + return values; +} + +export function stringAtNullable(value: unknown, path: string): string | null { + const found = valueAtPath(value, path); + return typeof found === "string" && found.length > 0 ? found : null; +} + +export function numberAtNullable(value: unknown, path: string): number | null { + const found = valueAtPath(value, path); + return typeof found === "number" && Number.isFinite(found) ? found : null; +} + +export function booleanAtNullable(value: unknown, path: string): boolean | null { + const found = valueAtPath(value, path); + return typeof found === "boolean" ? found : null; +} + +export function displayPath(pathValue: string): string { + if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1); + const marker = "/.worktree/"; + const index = repoRoot.indexOf(marker); + if (index >= 0) { + const mainRoot = repoRoot.slice(0, index); + if (pathValue.startsWith(`${mainRoot}/`)) return pathValue.slice(mainRoot.length + 1); + } + return pathValue; +} + +export function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string { + const commit = state.sourceHead.commit ?? "source"; + const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`; + if (!rerun) return base; + const suffix = `-r${Date.now().toString(36)}`; + return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`; +} + +export function sentinelCliSuffix(state: SentinelCicdState): string { + return ` --sentinel ${state.sentinelId}`; +} + +export function safeJobSegment(value: string): string { + return value.replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "").slice(0, 48) || "sentinel"; +} + +export function safeKubernetesSegment(value: string, maxLength: number): string { + const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, ""); + return (normalized || "sentinel").slice(0, Math.max(1, maxLength)).replace(/-+$/u, "") || "sentinel"; +} + +export function renderPublishResult(publish: Record): string { + const payload = record(publish.payload); + const diagnostics = record(publish.diagnostics); + const diagnosticEnvReuse = record(diagnostics.envReuse); + const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse; + const imageBuild = record(payload.imageBuild); + const imageBuildProxy = record(imageBuild.proxy); + const payloadStageTimings = record(payload.stageTimings); + const diagnosticStageTimings = record(diagnostics.stageTimings); + const timings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; + const commands = record(diagnostics.commands); + const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none"; + const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB"; + const lines = [ + "PUBLISH", + table(["OK", "PHASE", runColumn, "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[ + publish.ok, + publish.phase, + publish.jobName, + publish.elapsedMs ?? "-", + diagnostics.pod ?? "-", + diagnostics.currentPhase ?? "-", + short(payload.digestRef), + short(payload.gitopsCommit), + ]]), + ]; + if (Object.keys(envReuse).length > 0) { + lines.push( + "", + "PUBLISH_ENV_REUSE", + table(["MODE", "NODE_DEPS", "PRESENT", "ENTRIES", "LINKED", "DEPENDENCY"], [[ + envReuse.mode, + envReuse.nodeDepsPath, + envReuse.nodeDepsPresent, + envReuse.nodeDepsEntries, + envReuse.linkedNodeDeps ?? "-", + envReuse.dependencyReuse, + ]]), + ); + } + if (Object.keys(imageBuild).length > 0 || Object.keys(timings).length > 0) { + lines.push( + "", + "PUBLISH_BUILD", + table(["BUILDER", "PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "TOTAL_MS"], [[ + imageBuild.builder ?? "-", + imageBuild.packageMode ?? "-", + imageBuild.networkMode ?? "-", + proxySummary, + imageBuild.contextIgnoreEntries ?? "-", + imageBuild.layerCache ?? "-", + imageBuild.cacheHitLines ?? "-", + imageBuild.stepLines ?? "-", + timings.sourceFetchMs ?? "-", + timings.monitorWebVerifyMs ?? "-", + timings.imageBuildMs ?? "-", + timings.gitopsMs ?? "-", + timings.totalMs ?? payload.elapsedMs ?? "-", + ]]), + ); + } + if (Object.keys(diagnostics).length > 0) { + lines.push( + "", + "PUBLISH_DIAGNOSTICS", + table(["TASKRUN", "POD_PHASE", "ACTIVE", "CONDITION", "COMPLETED", "RECENT_LOG"], [[ + diagnostics.taskRun ?? "-", + diagnostics.podPhase ?? "-", + diagnostics.active ?? "-", + diagnostics.conditionReason ?? diagnostics.conditionStatus ?? "-", + Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "-", + diagnostics.recentLogSummary ?? "-", + ]]), + ); + } + if (publish.ok !== true && Object.keys(commands).length > 0) { + lines.push( + "", + "PUBLISH_DRILLDOWN", + ` status: ${commands.cliStatus ?? "-"}`, + ` logs: ${commands.logs ?? "-"}`, + ` describe: ${commands.describe ?? "-"}`, + ` publish-current: ${commands.publishCurrent ?? "-"}`, + ` git-mirror: ${commands.gitMirrorStatus ?? "-"}`, + ` sync: ${commands.gitMirrorSync ?? "-"}`, + ` flush: ${commands.gitMirrorFlush ?? "-"}`, + ` apply: ${commands.controlPlaneApply ?? "-"}`, + ); + } + return lines.join("\n"); +} + +export function renderPublishCurrentResult(result: Record): string { + const source = record(result.source); + const image = record(result.image); + const controlPlane = record(result.controlPlane); + const publish = record(controlPlane.publish); + const publishPayload = record(publish.payload); + const observed = record(controlPlane.observed); + const gitops = record(observed.gitops); + const argo = record(observed.argo); + const runtime = record(observed.runtime); + const runtimeDeployment = record(record(runtime.probe).deployment); + const health = record(result.health); + const healthBody = record(record(health.health).bodyJson); + const timings = record(result.timings); + const budget = record(result.budget); + const stageBudgets = record(result.stageBudgets); + const validationPlan = record(result.validationPlan); + const blocker = record(result.blocker); + const recoveryNext = record(controlPlane.recoveryNext); + const next = record(result.next); + const warnings = Array.isArray(result.warnings) ? result.warnings : []; + const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : []; + const lines = [ + String(result.command), + "", + table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[ + result.node, + result.lane, + result.sentinelId, + result.ok === true ? "ok" : "blocked", + result.mode, + budget.maxSeconds ?? "-", + finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000), + ]]), + "", + table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[ + `${source.repository ?? "-"}@${source.branch ?? "-"}`, + short(source.commit), + source.sourceAuthority ?? "-", + short(source.stageRef), + image.ref ?? "-", + short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest), + result.pipelineRun ?? publish.jobName ?? "-", + ]]), + "", + table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "HEALTH"], [[ + short(gitops.revision), + short(argo.revision), + `${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`, + short(runtimeDeployment.image), + `${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`, + health.ok === true ? "pass" : health.skipped === true ? `skipped:${text(health.reason)}` : Object.keys(health).length === 0 ? "planned" : "blocked", + ]]), + "", + table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "VALIDATION_MS", "TOTAL_MS"], [[ + timings.sourceSyncMs ?? "-", + timings.sourceFetchMs ?? "-", + timings.monitorWebVerifyMs ?? "-", + timings.imageBuildMs ?? "-", + timings.gitopsMs ?? "-", + timings.argoRuntimeMs ?? "-", + timings.healthValidationMs ?? "-", + timings.totalMs ?? "-", + ]]), + "", + table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "VALIDATION"], [[ + "YAML publishCurrent", + stageBudgets.sourceSyncSeconds ?? "-", + stageBudgets.sourceFetchSeconds ?? "-", + stageBudgets.monitorWebVerifySeconds ?? "-", + stageBudgets.imageBuildSeconds ?? "-", + stageBudgets.gitopsSeconds ?? "-", + stageBudgets.argoRuntimeSeconds ?? "-", + stageBudgets.dashboardVerifySeconds ?? "-", + ]]), + ]; + if (Object.keys(publish).length > 0) { + const payloadImageBuild = record(publishPayload.imageBuild); + const payloadEnvReuse = record(publishPayload.envReuse); + lines.push( + "", + table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[ + payloadEnvReuse.dependencyReuse ?? "-", + payloadEnvReuse.nodeDepsPresent ?? "-", + payloadImageBuild.packageMode ?? "-", + payloadImageBuild.networkMode ?? "-", + payloadImageBuild.layerCache ?? "-", + payloadImageBuild.cacheHitLines ?? "-", + ]]), + ); + } + lines.push( + "", + Object.keys(health).length === 0 + ? "HEALTH_VALIDATION\n-" + : table(["ENDPOINT", "HTTP", "OK", "STATUS", "PUBLIC_URL", "INTERNAL_URL"], [[ + health.endpoint ?? validationPlan.endpoint ?? "-", + health.httpStatus ?? "-", + healthBody.ok ?? "-", + healthBody.status ?? "-", + health.publicUrl ?? "-", + health.internalUrl ?? "-", + ]]), + "", + slowStages.length === 0 ? "SLOW_STAGES\n-" : [ + "SLOW_STAGES", + table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])), + ].join("\n"), + "", + warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), + "", + Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ + "RECOVERY_NEXT", + table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), + ` publish-current: ${recoveryNext.publishCurrent ?? "-"}`, + ` status: ${recoveryNext.nextStatus ?? "-"}`, + ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, + ` sync: ${recoveryNext.gitMirrorSync ?? "-"}`, + ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, + ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, + ].join("\n"), + "", + "NEXT", + ` publish-current: ${next.publishCurrent ?? "-"}`, + ` status: ${next.controlPlaneStatus ?? "-"}`, + ` post-deploy-dashboard: ${next.dashboardVerify ?? "-"}`, + ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, + ` sync: ${next.gitMirrorSync ?? "-"}`, + ` flush: ${next.gitMirrorFlush ?? "-"}`, + "", + "DISCLOSURE", + ` end-to-end and stage budgets are read from ${Object.keys(validationPlan).length > 0 ? "publishCurrent YAML and runtime.healthPath" : "YAML-required publishCurrent fields"}.`, + " CI/CD validation only checks the configured health endpoint; web-probe, Playwright and browser dashboard checks are post-deploy evidence, not this gate.", + " image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.", + ); + return lines.join("\n"); +} + +export function renderImageResult(result: Record): string { + const source = record(result.source); + const sourceMirror = record(result.sourceMirror); + const sourceMirrorSync = record(result.sourceMirrorSync); + const image = record(result.image); + const monitorWeb = record(image.monitorWeb); + const registry = record(result.registry); + const publish = record(result.publish); + const blocker = record(result.blocker); + const next = record(result.next); + const warnings = Array.isArray(result.warnings) ? result.warnings : []; + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]), + "", + table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]), + "", + Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]), + "", + table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]), + "", + Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "BUILD_STATE", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, `${record(monitorWeb.imageBuildState).mode ?? "-"}:${record(monitorWeb.imageBuildState).path ?? record(monitorWeb.imageBuildState).claimName ?? record(monitorWeb.imageBuildState).sizeLimit ?? "-"}`, monitorWeb.imageBuildContextIgnore]]), + "", + Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), + "", + Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), + "", + Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), + "", + warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), + "", + "NEXT", + ` status: ${next.status ?? "-"}`, + ` dry-run: ${next.dryRun ?? "-"}`, + ` confirm: ${next.confirm ?? "-"}`, + ` trigger: ${next.controlPlaneTrigger ?? "-"}`, + ` control-plane: ${next.controlPlanePlan ?? "-"}`, + "", + "DISCLOSURE", + " valuesRedacted=true; image status shows refs, hashes and object names only.", + ].join("\n"); +} + +export function renderControlPlaneResult(result: Record): string { + const source = record(result.source); + const image = record(result.image); + const gitops = record(result.gitops); + const argo = record(result.argo); + const validation = record(result.validation); + const observability = record(result.observability); + const observed = record(result.observed); + const sourceMirrorSync = record(result.sourceMirrorSync); + const publish = record(result.publish); + const flush = record(result.flush); + const runtimeSecretsApply = record(result.runtimeSecretsApply); + const publicExposureApply = record(result.publicExposureApply); + const publicExposureCaddy = record(publicExposureApply.caddy); + const argoApply = record(result.argoApply); + const blocker = record(result.blocker); + const statusDiagnosis = record(result.statusDiagnosis); + const targetValidation = record(result.targetValidation); + const targetValidationBusiness = record(targetValidation.businessStatus); + const recoveryNext = record(result.recoveryNext); + const next = record(result.next); + const warnings = Array.isArray(result.warnings) ? result.warnings : []; + return [ + String(result.command), + "", + table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]), + "", + table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]), + "", + table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]), + "", + table(["SCENARIO", "MAX_SECONDS", "CI_WAIT", "QVERIFY", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.controlPlaneWaitMaxSeconds ?? "-", validation.quickVerifyMode ?? "-", validation.automaticSecondPath]]), + "", + Object.keys(observability).length === 0 ? "OTEL\n-" : table(["ENABLED", "ENDPOINT", "SERVICE", "COVERAGE"], [[observability.enabled, observability.endpointConfigured, observability.serviceName, observability.coverage]]), + "", + renderObservedStatus(observed), + "", + Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [ + "STATUS_DIAGNOSIS", + table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[ + statusDiagnosis.code, + statusDiagnosis.phase, + statusDiagnosis.pipelineRun, + statusDiagnosis.sourceMirror, + statusDiagnosis.registry, + statusDiagnosis.gitMirror, + statusDiagnosis.gitops, + statusDiagnosis.argo, + statusDiagnosis.runtime, + ]]), + ].join("\n"), + "", + Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), + "", + Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "ERROR_TITLE", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ + targetValidation.ok, + targetValidation.status, + targetValidationBusiness.status ?? "-", + targetValidation.errorTitleZh ?? targetValidation.failureTitleZh ?? targetValidationBusiness.errorTitleZh ?? "-", + targetValidation.scenarioId, + targetValidation.runId, + targetValidation.observerId, + short(targetValidation.reportJsonSha256), + targetValidation.findingCount, + targetValidation.artifactCount, + ]]), + "", + Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), + "", + Object.keys(flush).length === 0 + ? "FLUSH\n-" + : flush.mode === "async-job" + ? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]]) + : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]), + "", + Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]), + "", + Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST", "ROUTE_HTTP"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname, record(publicExposureApply.caddy).routeProbeHttpStatus ?? "-"]]), + "", + Object.keys(publicExposureCaddy).length === 0 || publicExposureCaddy.ok === true + ? "CADDY_APPLY_DETAIL\n-" + : table(["PY", "VALIDATE", "RELOAD", "PROBE", "HTTP", "BLOCK", "ACTIVE", "ERROR", "STDOUT", "STDERR"], [[publicExposureCaddy.pythonExitCode, publicExposureCaddy.validateExitCode, publicExposureCaddy.reloadExitCode, publicExposureCaddy.routeProbeExitCode, publicExposureCaddy.routeProbeHttpStatus, publicExposureCaddy.afterBlockPresent, publicExposureCaddy.active, short(publicExposureCaddy.errorPreview), short(record(publicExposureCaddy.result).stdoutPreview), short(record(publicExposureCaddy.result).stderrPreview)]]), + "", + Object.keys(argoApply).length === 0 ? "ARGO_APPLY\n-" : table(["OK", "EXIT", "PREVIEW"], [[argoApply.ok, record(argoApply.result).exitCode, record(argoApply.result).stdoutPreview]]), + "", + warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), + "", + Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), + "", + Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ + "RECOVERY_NEXT", + table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), + ` publish-current: ${recoveryNext.publishCurrent ?? "-"}`, + ` status: ${recoveryNext.nextStatus ?? "-"}`, + ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, + ` sync: ${recoveryNext.gitMirrorSync ?? "-"}`, + ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, + ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, + ].join("\n"), + "", + "NEXT", + ` plan: ${next.plan ?? "-"}`, + ` status: ${next.status ?? "-"}`, + ` image: ${next.image ?? "-"}`, + ` trigger-current: ${next.triggerCurrent ?? "-"}`, + ` apply: ${next.apply ?? "-"}`, + ` validate: ${next.validate ?? "-"}`, + ` quick-verify: ${next.quickVerify ?? "-"}`, + ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, + ` sync: ${next.gitMirrorSync ?? "-"}`, + ` flush: ${next.gitMirrorFlush ?? "-"}`, + "", + "DISCLOSURE", + " default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.", + " sentinel unavailable policy is structured-failure; no automatic second execution path is rendered.", + ].join("\n"); +} + +export function renderObservedStatus(observed: Record): string { + const rows = [ + observedStatusRow("source", observed.sourceMirror), + observedStatusRow("registry", observed.registry), + observedStatusRow("git-mirror", observed.gitMirror), + observedStatusRow("gitops", observed.gitops), + observedStatusRow("argo", observed.argo), + observedStatusRow("runtime", observed.runtime), + observedStatusRow("cadence", observed.cadence), + ].filter((row) => row !== null); + if (rows.length === 0) return "OBSERVED\n-"; + return table(["CHECK", "OK", "DETAIL", "EXIT", "TIMED_OUT", "PREVIEW"], rows); +} + +export function observedStatusRow(name: string, value: unknown): unknown[] | null { + const item = record(value); + if (Object.keys(item).length === 0) return null; + const result = record(item.result); + return [name, item.ok, observedDetail(name, item), result.exitCode, result.timedOut, result.stdoutPreview]; +} + +export function observedDetail(name: string, item: Record): string { + if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`; + if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`; + if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`; + if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`; + if (name === "argo") { + const diagnostics = record(item.diagnostics); + const problems = Array.isArray(diagnostics.problemResources) ? diagnostics.problemResources : []; + const first = problems.find((entry) => typeof entry === "object" && entry !== null && !Array.isArray(entry)) as Record | undefined; + const problemText = Number(diagnostics.problemResourceCount ?? 0) > 0 + ? ` degraded=${diagnostics.problemResourceCount}:${first?.kind ?? "-"} ${first?.namespace ?? "-"}/${first?.name ?? "-"} ${first?.healthStatus ?? first?.status ?? "-"}` + : ""; + return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}${problemText}`; + } + if (name === "runtime") { + const probe = record(item.probe); + const deployment = record(probe.deployment); + return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`; + } + if (name === "cadence") { + if (item.skipped === true) return `${item.reason ?? "skipped"}`; + const probe = record(item.probe); + return `${probe.code ?? "ok"} schedule=${probe.schedule ?? "-"}/${probe.expectedSchedule ?? "-"} last=${probe.lastScheduleTime ?? "-"} jobs=${probe.jobCount ?? "-"}`; + } + return "-"; +} + +export function renderAsyncJobResult(result: Record): string { + const job = record(result.job); + const next = record(result.next); + return [ + String(result.command), + "", + table(["NODE", "LANE", "MODE", "MUTATION", "JOB"], [[result.node, result.lane, result.mode, result.mutation, job.id]]), + "", + table(["STATUS", "NAME", "CREATED"], [[job.status, job.name, job.createdAt]]), + "", + "NEXT", + ` status: ${next.status ?? "-"}`, + ` wait: ${next.wait ?? "-"}`, + "", + "DISCLOSURE", + " confirmed operation is delegated to UniDesk job status to keep interactive calls bounded.", + ].join("\n"); +} + +export function rendered(ok: boolean, command: string, text: string): RenderedCliResult { + return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" }; +} + +export function readConfigFile(file: string): unknown { + if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`); + const abs = rootPath(file); + if (!existsSync(abs)) throw new Error(`${file} does not exist`); + return Bun.YAML.parse(readFileSync(abs, "utf8")) as unknown; +} + +export function configRefFile(ref: string): string { + const [file, path, extra] = ref.split("#"); + if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) throw new Error(`invalid configRef: ${ref}`); + return file; +} + +export function valueAtPath(value: unknown, path: string): unknown { + let current: unknown = value; + for (const segment of path.split(".")) { + const match = /^([A-Za-z0-9_-]+)?(?:\[(\d+)\])?$/u.exec(segment); + if (match === null) return undefined; + if (match[1] !== undefined) { + if (!isRecord(current)) return undefined; + current = current[match[1]]; + } + if (match[2] !== undefined) { + if (!Array.isArray(current)) return undefined; + current = current[Number(match[2])]; + } + } + return current; +} + +export function stringAt(value: unknown, path: string): string { + const found = valueAtPath(value, path); + if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`); + return found; +} + +export function nonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +export function stringField(value: Record, path: string): string { + return stringAt(value, path); +} + +export function stringTarget(value: unknown, label: string): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must resolve to a non-empty string`); + return value; +} + +export function numberAt(value: unknown, path: string): number { + const found = valueAtPath(value, path); + if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`); + return found; +} + +export function booleanAt(value: unknown, path: string): boolean { + const found = valueAtPath(value, path); + if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`); + return found; +} + +export function finiteNumberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +export function arrayAt(value: unknown, path: string): unknown[] { + const found = valueAtPath(value, path); + if (!Array.isArray(found)) throw new Error(`${path} must be an array`); + return found; +} + +export function arrayAtNullable(value: unknown, path: string): Record[] { + const found = valueAtPath(value, path); + return Array.isArray(found) ? found.map(record) : []; +} + +export function recordTarget(value: unknown, label: string): Record { + if (!isRecord(value)) throw new Error(`${label} must resolve to an object`); + return value; +} + +export function record(value: unknown): Record { + return isRecord(value) ? value : {}; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function manifestObjectSummary(items: readonly Record[]): readonly Record[] { + return items.map((item) => ({ + kind: item.kind ?? null, + name: record(item.metadata).name ?? null, + namespace: record(item.metadata).namespace ?? null, + })); +} + +export function compactCommand(result: CommandResult): CompactCommandResult { + return { + exitCode: result.exitCode, + timedOut: result.timedOut, + stdoutBytes: Buffer.byteLength(result.stdout), + stderrBytes: Buffer.byteLength(result.stderr), + stdoutPreview: result.stdout.trim().slice(0, 500), + stderrPreview: result.stderr.trim().slice(0, 500), + }; +} + +export function resolveSentinelChildJson(result: Pick, requestedStdoutType: string): { parsed: Record | null; diagnostics: Record } { + const resolved = resolveCliChildJsonCommandResult({ + result, + requestedStdoutType, + acceptParsed: (value) => Object.keys(value).length > 0, + }); + return { parsed: resolved.parsed, diagnostics: resolved.diagnostics }; +} + +export function parseJsonObject(text: string): Record | null { + const trimmed = text.trim(); + if (trimmed.length === 0) return null; + try { + const parsed = JSON.parse(trimmed) as unknown; + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } +} + +export function table(headers: string[], rows: unknown[][]): string { + const normalized = [headers, ...rows.map((row) => row.map(text))]; + const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length))); + return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n"); +} + +export function text(value: unknown): string { + if (value === undefined || value === null || value === "") return "-"; + if (typeof value === "boolean") return value ? "true" : "false"; + return String(value).replace(/\s+/gu, " ").trim(); +} + +export function short(value: unknown): string { + const raw = text(value); + if (raw === "-") return raw; + if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`; + if (/^[0-9a-f]{40}$/iu.test(raw)) return raw.slice(0, 12); + return raw.length > 42 ? `${raw.slice(0, 39)}...` : raw; +} + +export function sha256(textValue: string): string { + return `sha256:${createHash("sha256").update(textValue).digest("hex")}`; +} + +export function shellQuote(value: string): string { + return `'${value.replace(/'/gu, "'\\''")}'`; +} + +export function tomlEscape(value: string): string { + return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"'); +} diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 8e9fe300..24b3ea1c 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -23,206 +23,103 @@ import type { RenderedCliResult } from "./output"; import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5"; import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; +import { + arrayAt, + arrayAtNullable, + booleanAt, + compactCommand, + displayPath, + finiteNumberOrNull, + isRecord, + manifestObjectSummary, + monitorWebCicdPlan, + nonEmptyString, + numberAt, + numberAtNullable, + parseEnvFile, + parseJsonObject, + record, + recordTarget, + rendered, + renderAsyncJobResult, + renderControlPlaneResult, + renderImageResult, + renderPublishCurrentResult, + renderPublishResult, + resolveSentinelChildJson, + safeJobSegment, + secretSourcePaths, + sentinelCliSuffix, + sentinelSourceSnapshotRef, + sentinelSourceSnapshotStageRefPrefix, + sha256, + shellQuote, + short, + stringAt, + stringAtNullable, + stringField, + stringTarget, + table, + text, + tomlEscape, + valueAtPath, + type ChildCliResult, + type CompactCommandResult, + type SentinelCicdState, + type SentinelImagePlan, + type SentinelObservedExpectation, + type SentinelObservedStatus, + type SentinelRemoteJobResult, + type SourceHead, + type WebProbeSentinelDashboardAction, + type WebProbeSentinelOptions, + type WebProbeSentinelReportView, +} from "./hwlab-node-web-sentinel-cicd-shared"; +import { + controlPlaneWaitWarningSeconds, + publishSatisfiedByObservedWarnings, + runSentinelPublishJob, + runSentinelSourceMirrorSyncJob, + sentinelBlockedRemoteResult, + sentinelCicdElapsedWarnings, + sentinelRemoteJobTimeoutWarnings, + sentinelSourceMirrorAlreadyPresentResult, + sourceMirrorAlreadyReadyWarnings, +} from "./hwlab-node-web-sentinel-cicd-jobs"; -export type WebProbeSentinelConfigAction = "plan" | "status"; -export type WebProbeSentinelImageAction = "status" | "build"; -export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current"; -export type WebProbeSentinelPublishAction = "publish-current"; -export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; -export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger"; -export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary"; +export type { + ChildCliResult, + CompactCommandResult, + SentinelCicdState, + WebProbeSentinelDashboardAction, + WebProbeSentinelOptions, + WebProbeSentinelReportView, +} from "./hwlab-node-web-sentinel-cicd-shared"; +export { + arrayAt, + compactCommand, + displayPath, + isRecord, + nonEmptyString, + numberAt, + numberAtNullable, + parseEnvFile, + parseJsonObject, + record, + recordTarget, + rendered, + renderAsyncJobResult, + safeJobSegment, + secretSourcePaths, + sentinelCliSuffix, + shellQuote, + short, + stringAt, + stringAtNullable, + table, + text, +}; -export type WebProbeSentinelOptions = - | { - readonly kind: "config"; - readonly action: WebProbeSentinelConfigAction; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly dryRun: boolean; - } - | { - readonly kind: "image"; - readonly action: WebProbeSentinelImageAction; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly dryRun: boolean; - readonly confirm: boolean; - readonly wait: boolean; - readonly timeoutSeconds: number; - } - | { - readonly kind: "control-plane"; - readonly action: WebProbeSentinelControlPlaneAction; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly dryRun: boolean; - readonly confirm: boolean; - readonly wait: boolean; - readonly timeoutSeconds: number; - readonly rerun: boolean; - } - | { - readonly kind: "publish"; - readonly action: WebProbeSentinelPublishAction; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly dryRun: boolean; - readonly confirm: boolean; - readonly wait: boolean; - readonly timeoutSeconds: number; - readonly rerun: boolean; - } - | { - readonly kind: "maintenance"; - readonly action: WebProbeSentinelMaintenanceAction; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly dryRun: boolean; - readonly confirm: boolean; - readonly wait: boolean; - readonly timeoutSeconds: number; - readonly releaseId: string | null; - readonly reason: string | null; - readonly quickVerify: boolean; - } - | { - readonly kind: "validate"; - readonly action: "validate"; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly dryRun: boolean; - readonly confirm: boolean; - readonly wait: boolean; - readonly timeoutSeconds: number; - readonly quickVerify: boolean; - } - | { - readonly kind: "report"; - readonly action: "report"; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly view: WebProbeSentinelReportView; - readonly runId: string | null; - readonly latest: boolean; - readonly traceId: string | null; - readonly sampleSeq: number | null; - readonly raw: boolean; - readonly full: boolean; - readonly timeoutSeconds: number; - } - | { - readonly kind: "dashboard"; - readonly action: WebProbeSentinelDashboardAction; - readonly node: string; - readonly lane: string; - readonly sentinelId: string | null; - readonly viewport: string; - readonly localDir: string; - readonly name: string | null; - readonly runId: string | null; - readonly timeoutMs: number; - readonly waitTimeoutMs: number; - readonly timeoutSeconds: number; - readonly commandTimeoutSeconds: number; - readonly fullPage: boolean; - readonly raw: boolean; - }; - -export interface SentinelCicdState { - readonly spec: HwlabRuntimeLaneSpec; - readonly sentinelId: string; - readonly configRefs: Record; - readonly configReady: boolean; - readonly runtime: Record; - readonly cicd: Record; - readonly scenarios: unknown; - readonly publicExposure: Record; - readonly secrets: Record; - readonly controlPlaneTarget: Record; - readonly controlPlaneNode: Record; - readonly sourceHead: SourceHead; - readonly image: SentinelImagePlan; - readonly manifests: readonly Record[]; - readonly manifestSha256: string; - readonly valuesRedacted: true; -} - -interface SourceHead { - readonly ok: boolean; - readonly repository: string; - readonly branch: string; - readonly commit: string | null; - readonly stageRef: string | null; - readonly mirrorCommit: string | null; - readonly sourceAuthority: "git-mirror-cache" | "git-mirror-snapshot"; - readonly latestDrift: boolean; - readonly result: CompactCommandResult; -} - -interface SentinelImagePlan { - readonly repository: string; - readonly tag: string; - readonly ref: string; - readonly digestRef: string | null; - readonly baseImage: string; - readonly buildContext: string; - readonly entrypoint: string; - readonly dockerfileSha256: string; - readonly dockerfilePreview: string; - readonly monitorWeb: Record; -} - -interface SentinelObservedStatus { - readonly sourceMirror: Record; - readonly registry: Record; - readonly gitMirror: Record; - readonly gitops: Record; - readonly argo: Record; - readonly runtime: Record; - readonly cadence: Record; - readonly wait?: Record; -} - -interface SentinelObservedExpectation { - readonly gitopsRevision: string | null; - readonly runtimeImage: string | null; -} - -interface SentinelRemoteJobResult { - readonly ok: boolean; - readonly phase: string; - readonly resourceKind?: "Job" | "PipelineRun"; - readonly jobName: string; - readonly payload: Record; - readonly polls?: number; - readonly elapsedMs?: number; - readonly create?: Record; - readonly probe?: Record; - readonly diagnostics?: Record; - readonly valuesRedacted: true; -} - -export interface CompactCommandResult { - readonly exitCode: number | null; - readonly timedOut: boolean; - readonly stdoutBytes: number; - readonly stderrBytes: number; - readonly stdoutPreview: string; - readonly stderrPreview: string; -} - -export interface ChildCliResult { - readonly ok: boolean; - readonly parsed: Record | null; - readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string }; -} const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot"; @@ -712,10 +609,11 @@ function resolveSourceHeadWithK8sSnapshot( while (Date.now() - startedAt < timeoutMs) { const probeCapture = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); lastCapture = probeCapture; - const probe = parseJsonObject(probeCapture.stdout) ?? {}; + const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-source-resolve-job-probe"); + const probe = probeResolution.parsed ?? {}; const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); - if (probe.succeeded === true) return { ok: payload.ok === true, probe: payload, result: probeCapture }; - if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, valuesRedacted: true } : payload, result: probeCapture }; + if (probe.succeeded === true) return { ok: payload.ok === true, probe: { ...payload, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result: probeCapture }; + if (probe.failed === true) return { ok: false, probe: Object.keys(payload).length === 0 ? { ok: false, status: "failed", jobName, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true } : { ...payload, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result: probeCapture }; runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } return { ok: false, probe: { ok: false, status: "timeout", jobName, valuesRedacted: true }, result: lastCapture }; @@ -787,20 +685,6 @@ function validateSentinelSourceAuthority(cicd: Record): void { sentinelSourceSnapshotStageRefPrefix(cicd); } -function sentinelSourceSnapshotStageRefPrefix(cicd: Record): string { - const branch = stringAt(cicd, "source.branch"); - const repository = stringAt(cicd, "source.repository"); - const prefix = stringAt(cicd, "sourceSnapshot.stageRefPrefix") - .replaceAll("{branch}", branch) - .replaceAll("{repository}", repository) - .replace(/\/+$/u, ""); - if (!prefix.startsWith("refs/")) throw new Error("sourceSnapshot.stageRefPrefix must resolve to a git ref prefix"); - return prefix; -} - -function sentinelSourceSnapshotRef(cicd: Record, commit: string): string { - return `${sentinelSourceSnapshotStageRefPrefix(cicd)}/${commit}`; -} function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record, sourceHead: SourceHead): SentinelImagePlan { const repository = stringAt(cicd, "image.repository"); @@ -835,63 +719,6 @@ function sentinelDockerfile(baseImage: string, entrypoint: string): string { "", ].join("\n"); } - -function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record): Record { - return { - stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build", - runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge", - assetRoot: stringAtNullable(cicd, "monitorWeb.assetRoot") ?? "scripts/assets/web-probe-sentinel-monitor-web", - verifyCommand: "bun scripts/verify-web-probe-sentinel-monitor-web.ts", - gitMirrorReadUrl: stringAt(cicd, "source.gitMirrorReadUrl"), - sourceMode: stringAt(cicd, "builder.sourceMode"), - envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "k8s-buildkit-and-ci-node-deps", - envReuseNodeDepsPath: stringAtNullable(cicd, "monitorWeb.envReuse.nodeDepsPath") ?? "/opt/hwlab-ci-node-deps/node_modules", - verifyPhase: stringAtNullable(cicd, "monitorWeb.imageBuild.verifyPhase") ?? "pre-image-build", - imageBuildBuilder: spec.buildkit?.sidecarImage ?? null, - imageBuildPackageMode: stringAtNullable(cicd, "monitorWeb.imageBuild.packageMode") ?? "copy-only-containerfile", - imageBuildNetworkMode: monitorWebImageBuildNetworkMode(cicd), - imageBuildProxySource: stringAtNullable(cicd, "monitorWeb.imageBuild.proxySource") ?? "node.networkProfile.imageBuildProxy", - imageBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.imageBuild.contextIgnore") ?? "generated", - imageBuildState: monitorWebBuildkitStatePlan(cicd), - ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"), - valuesRedacted: true, - }; -} - -function monitorWebImageBuildNetworkMode(cicd: Record): "default" | "host" { - const value = stringAtNullable(cicd, "monitorWeb.imageBuild.networkMode") ?? "host"; - if (value !== "default" && value !== "host") throw new Error(`monitorWeb.imageBuild.networkMode must be default or host, got ${value}`); - return value; -} - -function monitorWebBuildkitStatePlan(cicd: Record): Record { - const state = recordTarget(valueAtPath(cicd, "monitorWeb.imageBuild.buildkitState"), "monitorWeb.imageBuild.buildkitState"); - const mode = stringAt(state, "mode"); - if (mode === "hostPath") { - return { - mode, - path: stringAt(state, "path"), - type: stringAt(state, "type"), - valuesRedacted: true, - }; - } - if (mode === "persistentVolumeClaim") { - return { - mode, - claimName: stringAt(state, "claimName"), - valuesRedacted: true, - }; - } - if (mode === "emptyDir") { - return { - mode, - sizeLimit: stringAt(state, "sizeLimit"), - valuesRedacted: true, - }; - } - throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); -} - function publishCurrentBudget(state: SentinelCicdState): Record { const budget = recordTarget(valueAtPath(state.cicd, "publishCurrent.endToEndBudget"), "publishCurrent.endToEndBudget"); return { @@ -1602,9 +1429,11 @@ function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): R const result = probeMode === "k8s-service" ? runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "kubectl", "-n", stringAt(state.cicd, "builder.namespace"), "exec", "deploy/git-mirror-http", "--", "sh", "-lc", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }) : runCommand(["trans", stringAt(state.controlPlaneNode, "route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); + const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-image-registry-probe"); return { ok: result.exitCode === 0, - probe: parseJsonObject(result.stdout), + probe: probeResolution.parsed, + stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result), }; } @@ -2075,7 +1904,9 @@ function probeSourceMirrorCache(cicd: Record, controlPlaneNode: "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout) ?? {}, result }; + const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-source-mirror-cache-probe"); + const probe = probeResolution.parsed ?? {}; + return { ok: result.exitCode === 0 && probe.ok === true, probe: { ...probe, stdoutRecovery: probeResolution.diagnostics, valuesRedacted: true }, result }; } function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, expectedRevision: string | null): Record { @@ -2112,7 +1943,8 @@ function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number, function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSeconds: number, namespace: string, applicationName: string): Record { const route = stringAt(state.controlPlaneNode, "kubeRoute"); const appResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const parsed = parseJsonObject(appResult.stdout); + const appResolution = resolveSentinelChildJson(appResult, "web-probe-sentinel-argo-application-json"); + const parsed = appResolution.parsed; const status = record(parsed?.status); const resourcesRaw = Array.isArray(status.resources) ? status.resources : []; const resources = resourcesRaw @@ -2145,7 +1977,8 @@ function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSecond })); const operationState = record(status.operationState); const eventResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "events", "--field-selector", `involvedObject.name=${applicationName}`, "--sort-by=.lastTimestamp", "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const eventsJson = parseJsonObject(eventResult.stdout); + const eventResolution = resolveSentinelChildJson(eventResult, "web-probe-sentinel-argo-events-json"); + const eventsJson = eventResolution.parsed; const events = (Array.isArray(eventsJson?.items) ? eventsJson.items : []) .filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) .slice(-8) @@ -2171,6 +2004,11 @@ function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSecond events, result: compactCommand(appResult), eventsResult: compactCommand(eventResult), + stdoutRecovery: { + application: appResolution.diagnostics, + events: eventResolution.diagnostics, + valuesRedacted: true, + }, drillDown: { application: `trans ${route} kubectl -n ${namespace} get application ${applicationName} -o json`, events: `trans ${route} kubectl -n ${namespace} get events --field-selector involvedObject.name=${applicationName} --sort-by=.lastTimestamp`, @@ -2256,8 +2094,9 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const probe = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) }; + const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-runtime-objects-probe"); + const probe = probeResolution.parsed; + return { ok: result.exitCode === 0 && probe?.ok === true, probe, stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result) }; } function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record { @@ -2301,7 +2140,8 @@ function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const probe = parseJsonObject(result.stdout); + const probeResolution = resolveSentinelChildJson(result, "web-probe-sentinel-cadence-cronjob-probe"); + const probe = probeResolution.parsed; const ok = result.exitCode === 0 && probe?.ok === true; emitWebProbeSentinelSpan({ node: state.spec.nodeId, @@ -2322,6 +2162,7 @@ function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): return { ok, probe, + stdoutRecovery: probeResolution.diagnostics, result: compactCommand(result), warning: ok ? null : `cadence CronJob is not ready: ${text(probe?.code ?? "probe-failed")}`, valuesRedacted: true, @@ -2334,1099 +2175,6 @@ function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Re return `${state.image.repository}@${digest}`; } -function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult { - const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`; - const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63); - const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName); - const namespace = stringAt(state.cicd, "builder.namespace"); - sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); - const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - if (created.exitCode !== 0) { - sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane }); - return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror"); - } - const startedAt = Date.now(); - const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); - const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000; - let slowWarningSent = false; - let polls = 0; - let lastProbe: Record = {}; - while (Date.now() - startedAt < timeoutMs) { - polls += 1; - const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const probe = parseJsonObject(probeCapture.stdout) ?? {}; - lastProbe = { ...probe, capture: compactCommand(probeCapture) }; - const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); - sentinelProgressEvent("sentinel.source-mirror.progress", { - phase: "remote-job", - status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", - jobName, - polls, - elapsedMs: Date.now() - startedAt, - pod: probe.pod ?? null, - sourceCommit: state.sourceHead.commit, - node: state.spec.nodeId, - lane: state.spec.lane, - }); - if (probe.succeeded === true) { - const ok = payload.ok === true; - return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); - } - if (probe.failed === true) { - return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); - } - if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { - slowWarningSent = true; - sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); - } - runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); - } - return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); -} - -function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult { - return { - ok: false, - phase, - jobName: "-", - payload: { ok: false, status: phase, reason, valuesRedacted: true }, - valuesRedacted: true, - }; -} - -function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record { - const namespace = stringAt(state.cicd, "builder.namespace"); - const labels = { - "app.kubernetes.io/name": "web-probe-sentinel-source-mirror", - "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", - "unidesk.ai/spec-ref": "PJ2026-01060508", - "unidesk.ai/node": state.spec.nodeId, - "unidesk.ai/lane": state.spec.lane, - }; - return { - apiVersion: "batch/v1", - kind: "Job", - metadata: { name: jobName, namespace, labels }, - spec: { - backoffLimit: 0, - activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"), - ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"), - template: { - metadata: { labels }, - spec: { - restartPolicy: "Never", - volumes: [ - sentinelGitMirrorCacheVolume(state), - { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, - ], - containers: [{ - name: "sync", - image: state.image.baseImage, - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)], - volumeMounts: [ - { name: "cache", mountPath: "/cache" }, - { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - ], - }], - }, - }, - }, - }; -} - -function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string { - return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit); -} - -function sentinelSourceMirrorSyncShellFromConfig(cicd: Record, controlPlaneNode: Record, jobName: string, selectedCommit: string | null): string { - return [ - "set -eu", - `job_name=${shellQuote(jobName)}`, - `source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`, - `source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`, - `source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`, - `source_commit=${shellQuote(selectedCommit ?? "")}`, - `source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`, - "started_ms=$(node -e 'console.log(Date.now())')", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", - "trap emit_failed EXIT", - ...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode), - "repo=\"/cache/${source_repository}.git\"", - "mkdir -p \"$(dirname \"$repo\")\"", - "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", - " git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", - "else", - " rm -rf \"$repo\"", - " git init --bare \"$repo\"", - " git --git-dir=\"$repo\" remote add origin \"$source_git_url\"", - "fi", - "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", - "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", - "git --git-dir=\"$repo\" config http.uploadpack true", - "git --git-dir=\"$repo\" config http.receivepack true", - "fetch_ok=0", - "for attempt in 1 2 3; do", - " if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi", - " code=$?", - " printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2", - " sleep $((attempt * 5))", - "done", - "test \"$fetch_ok\" = 1", - "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")", - "if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi", - "git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"", - "test \"$mirror_commit\" = \"$source_commit\"", - "stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"", - "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"", - "git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"", - "git --git-dir=\"$repo\" update-server-info", - "finished_ms=$(node -e 'console.log(Date.now())')", - "node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'", - "const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);", - "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));", - "NODE", - "trap - EXIT", - ].join("\n"); -} - -function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] { - return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode); -} - -function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record): string[] { - const proxy = record(valueAtPath(controlPlaneNode, "egressProxy")); - const serviceName = nonEmptyString(proxy.serviceName); - const namespace = nonEmptyString(proxy.namespace); - const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null; - const hostRouteProxyUrl = nonEmptyString(proxy.proxyUrl); - const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : ""; - let proxyHost: string | null = null; - let proxyPort: number | null = null; - let proxyUrl: string | null = null; - if (serviceName !== null && namespace !== null && port !== null) { - proxyHost = `${serviceName}.${namespace}.svc.cluster.local`; - proxyPort = port; - proxyUrl = `http://${proxyHost}:${proxyPort}`; - } else if (hostRouteProxyUrl !== null) { - try { - const parsed = new URL(hostRouteProxyUrl); - const parsedPort = Number.parseInt(parsed.port || "80", 10); - if ((parsed.protocol === "http:" || parsed.protocol === "https:") && parsed.hostname.length > 0 && Number.isInteger(parsedPort)) { - proxyHost = parsed.hostname; - proxyPort = parsedPort; - proxyUrl = hostRouteProxyUrl; - } - } catch { - proxyHost = null; - proxyPort = null; - proxyUrl = null; - } - } - const useProxy = proxyHost !== null && proxyPort !== null && proxyUrl !== null; - if (!useProxy) { - return [ - "mkdir -p /root/.ssh", - "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - "printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2", - "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy", - "export NO_PROXY='*'", - "export no_proxy='*'", - "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", - "#!/bin/sh", - "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"", - "SH_PROXY", - "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", - "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", - "unset GIT_SSH_COMMAND", - ]; - } - const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; - return [ - "mkdir -p /root/.ssh", - "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - `printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`, - `export HTTP_PROXY=${shellQuote(proxyUrl)}`, - `export HTTPS_PROXY=${shellQuote(proxyUrl)}`, - `export ALL_PROXY=${shellQuote(proxyUrl)}`, - `export http_proxy=${shellQuote(proxyUrl)}`, - `export https_proxy=${shellQuote(proxyUrl)}`, - `export all_proxy=${shellQuote(proxyUrl)}`, - `export NO_PROXY=${shellQuote(noProxy)}`, - `export no_proxy=${shellQuote(noProxy)}`, - "cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'", - "#!/usr/bin/env node", - "const net = require('node:net');", - "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", - "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", - "const targetPort = Number.parseInt(targetPortRaw || '', 10);", - "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", - " console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');", - " process.exit(64);", - "}", - "let settled = false;", - "let tunnelEstablished = false;", - "function finish(code, message) {", - " if (settled) return;", - " settled = true;", - " if (message) console.error('sentinel source-mirror proxy-connect: ' + message);", - " process.exit(code);", - "}", - "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", - "let buffer = Buffer.alloc(0);", - "socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", - "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", - "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", - "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", - "function onData(chunk) {", - " buffer = Buffer.concat([buffer, chunk]);", - " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", - " if (headerEnd === -1 && buffer.length < 8192) return;", - " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", - " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", - " const statusLine = head.split('\\r\\n', 1)[0] || '';", - " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", - " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", - " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", - " socket.destroy();", - " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", - " return;", - " }", - " socket.off('data', onData);", - " socket.setTimeout(0);", - " tunnelEstablished = true;", - " const rest = buffer.slice(headerEnd + 4);", - " if (rest.length) process.stdout.write(rest);", - " process.stdin.on('error', () => {});", - " process.stdout.on('error', () => {});", - " process.stdin.pipe(socket);", - " socket.pipe(process.stdout);", - "}", - "socket.on('data', onData);", - "NODE_PROXY", - "chmod 0700 /tmp/sentinel-github-proxy-connect.cjs", - "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", - "#!/bin/sh", - `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`, - "SH_PROXY", - "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", - "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", - "unset GIT_SSH_COMMAND", - ]; -} - -function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult { - const pipelineRunName = sentinelPipelineRunName(state, rerun); - const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops); - const namespace = stringAt(state.cicd, "builder.namespace"); - sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane }); - const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - if (created.exitCode !== 0) { - sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); - return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish"); - } - sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); - const startedAt = Date.now(); - const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); - const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000; - let slowWarningSent = false; - let polls = 0; - let lastProbe: Record = {}; - while (Date.now() - startedAt < timeoutMs) { - polls += 1; - const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const probe = parseJsonObject(probeCapture.stdout) ?? {}; - lastProbe = { ...probe, capture: compactCommand(probeCapture) }; - const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); - sentinelProgressEvent("sentinel.publish.progress", { - phase: "remote-pipelinerun", - status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running", - pipelineRun: pipelineRunName, - publishGitops, - polls, - elapsedMs: Date.now() - startedAt, - taskRun: probe.taskRun ?? null, - pod: probe.pod ?? null, - sourceCommit: state.sourceHead.commit, - node: state.spec.nodeId, - lane: state.spec.lane, - }); - if (probe.succeeded === true) { - const ok = payload.ok === true; - return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); - } - if (probe.failed === true) { - return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); - } - if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { - slowWarningSent = true; - sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane }); - } - runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); - } - return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); -} - -function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record { - const namespace = stringAt(state.cicd, "builder.namespace"); - const buildkitImage = requireSentinelBuildkitImage(state); - const proxyEnv = sentinelImageBuildProxyEnv(state); - const labels = { - "app.kubernetes.io/name": "web-probe-sentinel-publish", - "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", - "unidesk.ai/spec-ref": "PJ2026-01060508", - "unidesk.ai/node": state.spec.nodeId, - "unidesk.ai/lane": state.spec.lane, - "unidesk.ai/ci-system": "tekton", - }; - return { - apiVersion: "tekton.dev/v1", - kind: "PipelineRun", - metadata: { - name: pipelineRunName, - namespace, - labels, - annotations: { - "unidesk.ai/source-commit": state.sourceHead.commit ?? "", - "unidesk.ai/source-authority": state.sourceHead.sourceAuthority, - "unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "", - "unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"), - "unidesk.ai/publish-gitops": publishGitops ? "true" : "false", - }, - }, - spec: { - timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` }, - taskRunTemplate: { - podTemplate: { - hostNetwork: true, - dnsPolicy: "ClusterFirstWithHostNet", - securityContext: { fsGroup: 1000 }, - }, - }, - pipelineSpec: { - tasks: [{ - name: "publish", - taskSpec: { - volumes: [ - sentinelGitMirrorCacheVolume(state), - { name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } }, - { name: "workspace", emptyDir: { sizeLimit: "8Gi" } }, - sentinelBuildkitStateVolume(state), - { name: "tmp", emptyDir: {} }, - ], - steps: [ - { - name: "source", - image: state.image.baseImage, - imagePullPolicy: "IfNotPresent", - env: proxyEnv, - script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)), - volumeMounts: [ - { name: "workspace", mountPath: "/workspace" }, - { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - ], - }, - { - name: "prepare-buildkit-state", - image: state.image.baseImage, - imagePullPolicy: "IfNotPresent", - script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"), - securityContext: { runAsUser: 0, runAsGroup: 0 }, - volumeMounts: [ - { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, - ], - }, - { - name: "image-build", - image: buildkitImage, - imagePullPolicy: "IfNotPresent", - env: [ - ...proxyEnv, - { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" }, - ], - script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)), - securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 }, - volumeMounts: [ - { name: "workspace", mountPath: "/workspace" }, - { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, - { name: "tmp", mountPath: "/tmp" }, - ], - }, - { - name: "publish", - image: state.image.baseImage, - imagePullPolicy: "IfNotPresent", - env: proxyEnv, - script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)), - volumeMounts: [ - { name: "workspace", mountPath: "/workspace" }, - { name: "cache", mountPath: "/cache" }, - { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, - ], - }, - ], - }, - }], - }, - }, - }; -} - -function tektonShellScript(body: string): string { - return `#!/bin/sh\n${body}`; -} - -function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record { - return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget); -} - -function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record): Record { - const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath")); - if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } }; - return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } }; -} - -function sentinelBuildkitStateVolume(state: SentinelCicdState): Record { - const buildkitState = monitorWebBuildkitStatePlan(state.cicd); - const mode = stringAt(buildkitState, "mode"); - if (mode === "hostPath") { - return { - name: "buildkit-state", - hostPath: { - path: stringAt(buildkitState, "path"), - type: stringAt(buildkitState, "type"), - }, - }; - } - if (mode === "persistentVolumeClaim") { - return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } }; - } - if (mode === "emptyDir") { - return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } }; - } - throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`); -} - -function requireSentinelBuildkitImage(state: SentinelCicdState): string { - const image = state.spec.buildkit?.sidecarImage; - if (typeof image !== "string" || image.length === 0) { - throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`); - } - return image; -} - -function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> { - const proxy = state.spec.networkProfile.imageBuildProxy; - const noProxy = proxy.noProxy.join(","); - return [ - { name: "HTTP_PROXY", value: proxy.http }, - { name: "http_proxy", value: proxy.http }, - { name: "HTTPS_PROXY", value: proxy.https }, - { name: "https_proxy", value: proxy.https }, - { name: "ALL_PROXY", value: proxy.all }, - { name: "all_proxy", value: proxy.all }, - { name: "NO_PROXY", value: noProxy }, - { name: "no_proxy", value: noProxy }, - ]; -} - -function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string { - const monitorWeb = record(state.image.monitorWeb); - const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => { - if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths"); - return item; - })), "utf8").toString("base64"); - const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64"); - const envReuseMode = stringAt(monitorWeb, "envReuseMode"); - const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath"); - return [ - "set -eu", - `job_name=${shellQuote(jobName)}`, - `source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`, - `source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`, - `source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`, - `source_commit=${shellQuote(state.sourceHead.commit ?? "")}`, - `source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`, - `checkout_paths_b64=${shellQuote(checkoutPathsB64)}`, - `dockerfile_b64=${shellQuote(dockerfileB64)}`, - `env_reuse_mode=${shellQuote(envReuseMode)}`, - `env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`, - "meta_dir=/workspace/meta", - "event_log=/workspace/publish-events.log", - "worktree=/workspace/source", - "rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json", - "mkdir -p \"$meta_dir\"", - ": > \"$event_log\"", - "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", - "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", - "trap emit_failed EXIT", - "started_ms=$(now_ms)", - "write_meta started_ms \"$started_ms\"", - "write_meta source_commit \"$source_commit\"", - "write_meta source_stage_ref \"$source_stage_ref\"", - "test -n \"$source_commit\"", - "test -n \"$source_stage_ref\"", - "mkdir -p /root/.ssh", - "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", - "git init \"$worktree\"", - "cd \"$worktree\"", - "git remote add origin \"$source_git_url\"", - "git config core.sparseCheckout true", - "git config remote.origin.promisor true", - "git config remote.origin.partialclonefilter blob:none", - "CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'", - "const fs = require('node:fs');", - "const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));", - "fs.mkdirSync('.git/info', { recursive: true });", - "fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');", - "NODE", - "source_fetch_started_ms=$(now_ms)", - "write_meta source_fetch_started_ms \"$source_fetch_started_ms\"", - "emit_stage source-fetch running \"$source_fetch_started_ms\"", - "git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"", - "git checkout --detach \"$source_commit\"", - "mirror_commit=$(git rev-parse HEAD)", - "test \"$mirror_commit\" = \"$source_commit\"", - "write_meta mirror_commit \"$mirror_commit\"", - "source_fetch_finished_ms=$(now_ms)", - "write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"", - "emit_stage source-fetch succeeded \"$source_fetch_started_ms\"", - "env_reuse_node_deps_present=false", - "env_reuse_node_deps_entries=0", - "if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi", - "env_reuse_linked_node_deps=0", - "rm -rf node_modules", - "if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi", - "write_meta env_reuse_mode \"$env_reuse_mode\"", - "write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"", - "write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"", - "write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"", - "write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"", - "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'", - "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", - "NODE", - "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"", - "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", - "NODE", - "monitor_web_verify_started_ms=$(now_ms)", - "write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"", - "emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"", - "if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi", - "cat /tmp/web-probe-sentinel-monitor-web-verify.log", - "monitor_web_verify_finished_ms=$(now_ms)", - "write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"", - "emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"", - "mkdir -p .unidesk-sentinel-bin", - "cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'", - "#!/bin/sh", - "exec bun /app/scripts/ssh-cli.ts \"$@\"", - "SH_TRANS", - "chmod 0755 .unidesk-sentinel-bin/trans", - "DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'", - "const fs = require('node:fs');", - "fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));", - "NODE", - "cat > .dockerignore <<'EOF_DOCKERIGNORE'", - ".git", - ".git/**", - ".state", - ".state/**", - "logs", - "logs/**", - "node_modules", - "node_modules/**", - "**/node_modules", - "**/node_modules/**", - "**/dist", - "**/dist/**", - "**/target", - "**/target/**", - "**/coverage", - "**/coverage/**", - "npm-debug.log*", - ".env", - ".env.*", - "EOF_DOCKERIGNORE", - "context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')", - "write_meta context_ignore_entries \"$context_ignore_entries\"", - "chmod -R a+rwX /workspace", - "trap - EXIT", - ].join("\n"); -} - -function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string { - const monitorWeb = record(state.image.monitorWeb); - const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode"); - const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode"); - const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource"); - const imageBuildProxy = state.spec.networkProfile.imageBuildProxy; - const imageBuildNoProxy = imageBuildProxy.noProxy.join(","); - return [ - "set -eu", - `job_name=${shellQuote(jobName)}`, - `image_ref=${shellQuote(state.image.ref)}`, - `image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`, - `image_build_package_mode=${shellQuote(imageBuildPackageMode)}`, - `image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`, - `image_build_proxy_source=${shellQuote(imageBuildProxySource)}`, - `image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`, - `image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`, - `image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`, - `image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`, - "meta_dir=/workspace/meta", - "event_log=/workspace/publish-events.log", - "build_log=/workspace/image-build.log", - "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", - "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", - "trap emit_failed EXIT", - "cd /workspace/source", - "image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi", - "image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi", - "image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi", - "image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi", - "write_meta image_build_builder \"$image_build_builder\"", - "write_meta image_build_package_mode \"$image_build_package_mode\"", - "write_meta image_build_network_mode \"$image_build_network_mode\"", - "write_meta image_build_proxy_source \"$image_build_proxy_source\"", - "write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"", - "write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"", - "write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"", - "write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"", - "image_build_started_ms=$(now_ms)", - "write_meta image_build_started_ms \"$image_build_started_ms\"", - "emit_stage image-build running \"$image_build_started_ms\"", - "if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi", - "cat \"$build_log\"", - "image_build_finished_ms=$(now_ms)", - "write_meta image_build_finished_ms \"$image_build_finished_ms\"", - "metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)", - "digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)", - "test -n \"$digest\"", - "repo_no_tag=${image_ref%:*}", - "digest_ref=\"$repo_no_tag@$digest\"", - "write_meta digest \"$digest\"", - "write_meta digest_ref \"$digest_ref\"", - "emit_stage image-build succeeded \"$image_build_started_ms\"", - "trap - EXIT", - ].join("\n"); -} - -function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { - const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : []; - const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ - path: file.path, - contentBase64: Buffer.from(file.content, "utf8").toString("base64"), - }))), "utf8").toString("base64"); - return [ - "set -eu", - `job_name=${shellQuote(jobName)}`, - `image_ref=${shellQuote(state.image.ref)}`, - `image_repository=${shellQuote(state.image.repository)}`, - `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`, - `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`, - `files_b64=${shellQuote(filesB64)}`, - "meta_dir=/workspace/meta", - "event_log=/workspace/publish-events.log", - "build_log=/workspace/image-build.log", - "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", - "read_meta() { cat \"$meta_dir/$1\"; }", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }", - "trap emit_failed EXIT", - "if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi", - "mkdir -p /root/.ssh", - "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - "export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", - "started_ms=$(read_meta started_ms)", - "source_commit=$(read_meta source_commit)", - "source_stage_ref=$(read_meta source_stage_ref)", - "mirror_commit=$(read_meta mirror_commit)", - "source_fetch_started_ms=$(read_meta source_fetch_started_ms)", - "source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)", - "monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)", - "monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)", - "image_build_started_ms=$(read_meta image_build_started_ms)", - "image_build_finished_ms=$(read_meta image_build_finished_ms)", - "digest_ref=$(read_meta digest_ref)", - "env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')", - "env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')", - "env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)", - "env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)", - "env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)", - "image_build_builder=$(read_meta image_build_builder)", - "image_build_package_mode=$(read_meta image_build_package_mode)", - "image_build_network_mode=$(read_meta image_build_network_mode)", - "image_build_proxy_source=$(read_meta image_build_proxy_source)", - "image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)", - "image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)", - "image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)", - "image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)", - "context_ignore_entries=$(read_meta context_ignore_entries)", - "image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)", - "image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)", - "image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')", - "gitops_commit=''", - "changed=false", - "file_count=0", - "gitops_started_ms=$(now_ms)", - "if [ \"$files_b64\" != \"W10=\" ]; then", - " emit_stage gitops running \"$gitops_started_ms\"", - " gitops_cache=\"/cache/${gitops_repository}.git\"", - " gitops_worktree=\"/tmp/$job_name/gitops\"", - " git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"", - " cd \"$gitops_worktree\"", - " git fetch origin \"$gitops_branch\" || true", - " if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi", - " FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'", - "const fs = require('node:fs');", - "const path = require('node:path');", - "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", - "for (const file of files) {", - " const target = path.resolve(process.cwd(), file.path);", - " if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);", - " fs.mkdirSync(path.dirname(target), { recursive: true });", - " const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);", - " fs.writeFileSync(target, text);", - "}", - "console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));", - "NODE", - " git add .", - " file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')", - " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi", - " git push origin \"HEAD:refs/heads/$gitops_branch\"", - " gitops_commit=$(git rev-parse HEAD)", - " emit_stage gitops succeeded \"$gitops_started_ms\"", - "else", - " emit_stage gitops skipped \"$gitops_started_ms\"", - "fi", - "gitops_finished_ms=$(now_ms)", - "finished_ms=$(now_ms)", - "node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'", - "const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);", - "const elapsed = (start, finish) => Number(finish) - Number(start);", - "const cacheHits = Number(imageBuildCacheHits || 0);", - "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", - "NODE", - "trap - EXIT", - ].join("\n"); -} - -function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] { - const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application"); - return [{ - path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`, - content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`, - }]; -} - -function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record { - const app = state.manifests.find((item) => item.kind === "Application"); - if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true }; - const yaml = `${Bun.YAML.stringify(app).trim()}\n`; - const namespace = stringAt(state.cicd, "argo.namespace"); - const applicationName = stringAt(state.cicd, "argo.applicationName"); - const script = [ - "set -eu", - "tmp=$(mktemp)", - `cat >"$tmp" <<'YAML'\n${yaml}YAML`, - "kubectl apply -f \"$tmp\"", - `kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`, - ].join("\n"); - const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true }; -} - -function createK8sJobScript(namespace: string, manifest: Record): string { - const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`; - return [ - "set -eu", - `kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`, - "tmp=$(mktemp)", - `cat >"$tmp" <<'YAML'\n${yaml}YAML`, - "kubectl apply -f \"$tmp\"", - ].join("\n"); -} - -function createTektonPipelineRunScript(namespace: string, manifest: Record): string { - const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`; - const pipelineRunName = stringAt(manifest, "metadata.name"); - return [ - "set -eu", - `pipeline_run=${shellQuote(pipelineRunName)}`, - `namespace=${shellQuote(namespace)}`, - "tmp=$(mktemp)", - `cat >"$tmp" <<'YAML'\n${yaml}YAML`, - "if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then", - " printf '%s\\n' \"sentinel publish PipelineRun already exists; reusing $pipeline_run\"", - " exit 0", - "fi", - "if ! kubectl create -f \"$tmp\"; then", - " if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then", - " printf '%s\\n' \"sentinel publish PipelineRun appeared concurrently; reusing $pipeline_run\"", - " exit 0", - " fi", - " exit 1", - "fi", - ].join("\n"); -} - -function probeK8sJobScript(namespace: string, jobName: string): string { - return [ - "set +e", - `namespace=${shellQuote(namespace)}`, - `job=${shellQuote(jobName)}`, - "succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)", - "failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)", - "active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)", - "pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)", - "pod_phase=''", - "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", - "logs_tail=''", - "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi", - "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", - "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", - "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", - "NODE", - ].join("\n"); -} - -function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string { - return [ - "set +e", - `namespace=${shellQuote(namespace)}`, - `pipeline_run=${shellQuote(pipelineRunName)}`, - "condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)", - "condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)", - "condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)", - "if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi", - "if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi", - "if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi", - "task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", - "pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)", - "pod_phase=''", - "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi", - "logs_tail=''", - "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi", - "node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", - "const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);", - "const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');", - "const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));", - "console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", - "NODE", - ].join("\n"); -} - -function sentinelPayloadFromLogs(logsTail: string): Record { - const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); - for (let index = lines.length - 1; index >= 0; index -= 1) { - const line = lines[index]; - if (!line.startsWith("{") || !line.endsWith("}")) continue; - const parsed = parseJsonObject(line); - if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed; - } - return {}; -} - -function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult { - return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true }; -} - -function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record { - const namespace = stringAt(state.cicd, "builder.namespace"); - const probe = record(result.probe); - const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : ""; - const events = sentinelStageEventsFromLogs(logsTail, domain); - const envReuse = sentinelEnvReuseFromLogs(logsTail); - const completedStages = sentinelCompletedStages(events, record(result.payload)); - const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs); - const currentPhase = sentinelCurrentRemotePhase(result, events, domain); - const isPipelineRun = result.resourceKind === "PipelineRun"; - const commands = { - cliStatus: domain === "publish" - ? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}` - : `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, - logs: result.jobName === "-" - ? "-" - : isPipelineRun - ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120` - : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`, - describe: result.jobName === "-" - ? "-" - : isPipelineRun - ? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}` - : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`, - gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, - gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`, - gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`, - controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, - publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, - valuesRedacted: true, - }; - return { - domain, - resourceKind: result.resourceKind ?? "Job", - pipelineRun: isPipelineRun ? result.jobName : null, - taskRun: probe.taskRun ?? null, - currentPhase, - completedStages, - stageTimings, - envReuse, - pod: probe.pod ?? null, - podPhase: probe.podPhase ?? null, - active: probe.active ?? null, - conditionStatus: probe.conditionStatus ?? null, - conditionReason: probe.conditionReason ?? null, - recentLogSummary: sentinelRecentLogSummary(logsTail), - commands, - valuesRedacted: true, - }; -} - -function sentinelEnvReuseFromLogs(logsTail: string): Record | null { - const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); - for (let index = lines.length - 1; index >= 0; index -= 1) { - const parsed = parseJsonObject(lines[index]); - if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true }; - } - return null; -} - -function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record[] { - const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage"; - return logsTail - .split(/\r?\n/u) - .map((line) => parseJsonObject(line.trim())) - .filter((item): item is Record => item !== null && item.event === expectedEvent); -} - -function sentinelCompletedStages(events: readonly Record[], payload: Record): string[] { - const completed = events - .filter((event) => event.status === "succeeded" || event.status === "skipped") - .map((event) => `${text(event.stage)}:${text(event.status)}`); - const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : []; - return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-"); -} - -function sentinelStageTimingSummary(events: readonly Record[], payload: Record, fallbackTotalMs: unknown): Record { - const payloadTimings = record(payload.stageTimings); - const eventElapsed = (stage: string): number | null => { - const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed")); - return event === undefined ? null : finiteNumberOrNull(event.elapsedMs); - }; - const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch"); - const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify"); - const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build"); - const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops"); - const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null); - const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0); - const totalMs = finiteNumberOrNull(payloadTimings.totalMs) - ?? finiteNumberOrNull(payload.elapsedMs) - ?? finiteNumberOrNull(fallbackTotalMs) - ?? summedTotalMs; - const result = { - sourceFetchMs, - monitorWebVerifyMs, - imageBuildMs, - gitopsMs, - totalMs, - valuesRedacted: true, - }; - return Object.values(result).some((item) => item !== null && item !== true) ? result : {}; -} - -function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record[], domain: "source-mirror" | "publish"): string { - if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed"; - if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase; - const reversed = [...events].reverse(); - const failed = reversed.find((event) => event.status === "failed"); - if (failed !== undefined) return text(failed.stage); - const running = reversed.find((event) => event.status === "running"); - if (running !== undefined) return text(running.stage); - const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage))); - const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"]; - const next = order.find((stage) => !completed.has(stage)); - return next ?? result.phase; -} - -function sentinelRecentLogSummary(logsTail: string): string { - const lines = logsTail - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter((line) => line.length > 0 && !line.startsWith("{")) - .slice(-5) - .map((line) => short(line)); - return lines.length === 0 ? "-" : lines.join(" | "); -} - -function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] { - const remote = record(job); - if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return []; - const diagnostics = record(remote.diagnostics); - const commands = record(diagnostics.commands); - return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`]; -} - -function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] { - if (!observedReady) return []; - const warnings: string[] = []; - const publishRecord = record(publish); - if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) { - warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`); - } - const flushRecord = record(flush); - if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) { - warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout."); - } - return warnings; -} - -function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number { - return numberAt(state.cicd, "confirmWait.maxSeconds"); -} - -function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] { - const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; - const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; - if (elapsedMs === null || elapsedMs <= budgetMs) return []; - return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`]; -} - -function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] { - const sync = record(sourceMirrorSync); - if (sync.ok === true || state.sourceHead.ok !== true) return []; - return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`]; -} - -function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record { - return { - ok: true, - phase: "already-present", - jobName: null, - probe: record(probe), - payload: { - ok: true, - status: "already-present", - sourceCommit: state.sourceHead.commit, - mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit, - stageRef: state.sourceHead.stageRef, - sourceAuthority: state.sourceHead.sourceAuthority, - valuesRedacted: true, - }, - polls: 0, - elapsedMs: 0, - valuesRedacted: true, - }; -} - function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] { if (applyOnly) return []; const next = sentinelP5Next(state); @@ -3655,8 +2403,9 @@ function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: n "PY", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: manifestYaml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, result: compactCommand(result), valuesRedacted: true }; + const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-runtime-secrets-apply"); + const parsed = parsedResolution.parsed; + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), skippedKeyCount: skipped.length, skipped, stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true }; } function readSentinelSecretSourceValue(source: Record): Record { @@ -3772,8 +2521,9 @@ function applySentinelFrpcSecret(state: SentinelCicdState, frpcToml: string, tim "NODE", ].join("\n"); const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { input: frpcToml, timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; + const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-frpc-secret-apply"); + const parsed = parsedResolution.parsed; + return { ok: result.exitCode === 0 && parsed?.ok === true, ...record(parsed), stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true }; } function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: number): Record { @@ -3967,703 +2717,7 @@ function applySentinelCaddyBlock(state: SentinelCicdState, timeoutSeconds: numbe "PY", ].join("\n"); const result = runCommand(["trans", stringAt(state.publicExposure, "caddy.route"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); - const parsed = parseJsonObject(result.stdout); - return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), result: compactCommand(result), valuesRedacted: true }; -} - -export function secretSourcePaths(sourceRef: string): string[] { - if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef); - const paths = [join(repoRoot, ".state", "secrets", sourceRef)]; - const marker = "/.worktree/"; - const index = repoRoot.indexOf(marker); - if (index >= 0) paths.push(join(repoRoot.slice(0, index), ".state", "secrets", sourceRef)); - return [...new Set(paths)]; -} - -function ownerFileSourcePaths(sourceRef: string): string[] { - if (sourceRef.includes("..") || sourceRef.includes("\0")) return []; - const marker = "/.worktree/"; - const index = repoRoot.indexOf(marker); - const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot]; - return [...new Set(roots.map((root) => join(root, sourceRef)))]; -} - -export function parseEnvFile(textValue: string): Record { - const values: Record = {}; - for (const rawLine of textValue.split(/\r?\n/u)) { - const line = rawLine.trim(); - if (line.length === 0 || line.startsWith("#")) continue; - const index = line.indexOf("="); - if (index <= 0) continue; - const key = line.slice(0, index).trim(); - let value = line.slice(index + 1).trim(); - if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1); - values[key] = value; - } - return values; -} - -export function stringAtNullable(value: unknown, path: string): string | null { - const found = valueAtPath(value, path); - return typeof found === "string" && found.length > 0 ? found : null; -} - -export function numberAtNullable(value: unknown, path: string): number | null { - const found = valueAtPath(value, path); - return typeof found === "number" && Number.isFinite(found) ? found : null; -} - -function booleanAtNullable(value: unknown, path: string): boolean | null { - const found = valueAtPath(value, path); - return typeof found === "boolean" ? found : null; -} - -export function displayPath(pathValue: string): string { - if (pathValue.startsWith(`${repoRoot}/`)) return pathValue.slice(repoRoot.length + 1); - const marker = "/.worktree/"; - const index = repoRoot.indexOf(marker); - if (index >= 0) { - const mainRoot = repoRoot.slice(0, index); - if (pathValue.startsWith(`${mainRoot}/`)) return pathValue.slice(mainRoot.length + 1); - } - return pathValue; -} - -function sentinelPipelineRunName(state: SentinelCicdState, rerun = false): string { - const commit = state.sourceHead.commit ?? "source"; - const base = `hwlab-web-probe-sentinel-${safeKubernetesSegment(state.sentinelId, 24)}-${commit.slice(0, 12)}`; - if (!rerun) return base; - const suffix = `-r${Date.now().toString(36)}`; - return `${base.slice(0, Math.max(1, 63 - suffix.length)).replace(/-+$/u, "")}${suffix}`; -} - -export function sentinelCliSuffix(state: SentinelCicdState): string { - return ` --sentinel ${state.sentinelId}`; -} - -export function safeJobSegment(value: string): string { - return value.replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "").slice(0, 48) || "sentinel"; -} - -function safeKubernetesSegment(value: string, maxLength: number): string { - const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, ""); - return (normalized || "sentinel").slice(0, Math.max(1, maxLength)).replace(/-+$/u, "") || "sentinel"; -} - -function renderPublishResult(publish: Record): string { - const payload = record(publish.payload); - const diagnostics = record(publish.diagnostics); - const diagnosticEnvReuse = record(diagnostics.envReuse); - const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse; - const imageBuild = record(payload.imageBuild); - const imageBuildProxy = record(imageBuild.proxy); - const payloadStageTimings = record(payload.stageTimings); - const diagnosticStageTimings = record(diagnostics.stageTimings); - const timings = Object.keys(payloadStageTimings).length > 0 ? payloadStageTimings : diagnosticStageTimings; - const commands = record(diagnostics.commands); - const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none"; - const runColumn = diagnostics.resourceKind === "PipelineRun" || publish.resourceKind === "PipelineRun" ? "PIPELINERUN" : "JOB"; - const lines = [ - "PUBLISH", - table(["OK", "PHASE", runColumn, "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[ - publish.ok, - publish.phase, - publish.jobName, - publish.elapsedMs ?? "-", - diagnostics.pod ?? "-", - diagnostics.currentPhase ?? "-", - short(payload.digestRef), - short(payload.gitopsCommit), - ]]), - ]; - if (Object.keys(envReuse).length > 0) { - lines.push( - "", - "PUBLISH_ENV_REUSE", - table(["MODE", "NODE_DEPS", "PRESENT", "ENTRIES", "LINKED", "DEPENDENCY"], [[ - envReuse.mode, - envReuse.nodeDepsPath, - envReuse.nodeDepsPresent, - envReuse.nodeDepsEntries, - envReuse.linkedNodeDeps ?? "-", - envReuse.dependencyReuse, - ]]), - ); - } - if (Object.keys(imageBuild).length > 0 || Object.keys(timings).length > 0) { - lines.push( - "", - "PUBLISH_BUILD", - table(["BUILDER", "PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "TOTAL_MS"], [[ - imageBuild.builder ?? "-", - imageBuild.packageMode ?? "-", - imageBuild.networkMode ?? "-", - proxySummary, - imageBuild.contextIgnoreEntries ?? "-", - imageBuild.layerCache ?? "-", - imageBuild.cacheHitLines ?? "-", - imageBuild.stepLines ?? "-", - timings.sourceFetchMs ?? "-", - timings.monitorWebVerifyMs ?? "-", - timings.imageBuildMs ?? "-", - timings.gitopsMs ?? "-", - timings.totalMs ?? payload.elapsedMs ?? "-", - ]]), - ); - } - if (Object.keys(diagnostics).length > 0) { - lines.push( - "", - "PUBLISH_DIAGNOSTICS", - table(["TASKRUN", "POD_PHASE", "ACTIVE", "CONDITION", "COMPLETED", "RECENT_LOG"], [[ - diagnostics.taskRun ?? "-", - diagnostics.podPhase ?? "-", - diagnostics.active ?? "-", - diagnostics.conditionReason ?? diagnostics.conditionStatus ?? "-", - Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "-", - diagnostics.recentLogSummary ?? "-", - ]]), - ); - } - if (publish.ok !== true && Object.keys(commands).length > 0) { - lines.push( - "", - "PUBLISH_DRILLDOWN", - ` status: ${commands.cliStatus ?? "-"}`, - ` logs: ${commands.logs ?? "-"}`, - ` describe: ${commands.describe ?? "-"}`, - ` publish-current: ${commands.publishCurrent ?? "-"}`, - ` git-mirror: ${commands.gitMirrorStatus ?? "-"}`, - ` sync: ${commands.gitMirrorSync ?? "-"}`, - ` flush: ${commands.gitMirrorFlush ?? "-"}`, - ` apply: ${commands.controlPlaneApply ?? "-"}`, - ); - } - return lines.join("\n"); -} - -function renderPublishCurrentResult(result: Record): string { - const source = record(result.source); - const image = record(result.image); - const controlPlane = record(result.controlPlane); - const publish = record(controlPlane.publish); - const publishPayload = record(publish.payload); - const observed = record(controlPlane.observed); - const gitops = record(observed.gitops); - const argo = record(observed.argo); - const runtime = record(observed.runtime); - const runtimeDeployment = record(record(runtime.probe).deployment); - const health = record(result.health); - const healthBody = record(record(health.health).bodyJson); - const timings = record(result.timings); - const budget = record(result.budget); - const stageBudgets = record(result.stageBudgets); - const validationPlan = record(result.validationPlan); - const blocker = record(result.blocker); - const recoveryNext = record(controlPlane.recoveryNext); - const next = record(result.next); - const warnings = Array.isArray(result.warnings) ? result.warnings : []; - const slowStages = Array.isArray(result.slowStages) ? result.slowStages.map(record) : []; - const lines = [ - String(result.command), - "", - table(["NODE", "LANE", "SENTINEL", "STATUS", "MODE", "BUDGET_S", "ELAPSED_S"], [[ - result.node, - result.lane, - result.sentinelId, - result.ok === true ? "ok" : "blocked", - result.mode, - budget.maxSeconds ?? "-", - finiteNumberOrNull(result.elapsedMs) === null ? "-" : Math.round((finiteNumberOrNull(result.elapsedMs) ?? 0) / 1000), - ]]), - "", - table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE_REF", "DIGEST", "PIPELINERUN"], [[ - `${source.repository ?? "-"}@${source.branch ?? "-"}`, - short(source.commit), - source.sourceAuthority ?? "-", - short(source.stageRef), - image.ref ?? "-", - short(publishPayload.digestRef ?? record(record(observed.registry).probe).digest), - result.pipelineRun ?? publish.jobName ?? "-", - ]]), - "", - table(["GITOPS_REV", "ARGO_REV", "ARGO", "RUNTIME_IMAGE", "RUNTIME_READY", "HEALTH"], [[ - short(gitops.revision), - short(argo.revision), - `${argo.syncStatus ?? "-"}/${argo.healthStatus ?? "-"}`, - short(runtimeDeployment.image), - `${runtimeDeployment.readyReplicas ?? "-"}/${runtimeDeployment.desiredReplicas ?? "-"}`, - health.ok === true ? "pass" : health.skipped === true ? `skipped:${text(health.reason)}` : Object.keys(health).length === 0 ? "planned" : "blocked", - ]]), - "", - table(["SOURCE_SYNC_MS", "SOURCE_FETCH_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "ARGO_RUNTIME_MS", "VALIDATION_MS", "TOTAL_MS"], [[ - timings.sourceSyncMs ?? "-", - timings.sourceFetchMs ?? "-", - timings.monitorWebVerifyMs ?? "-", - timings.imageBuildMs ?? "-", - timings.gitopsMs ?? "-", - timings.argoRuntimeMs ?? "-", - timings.healthValidationMs ?? "-", - timings.totalMs ?? "-", - ]]), - "", - table(["BUDGET_SOURCE", "SOURCE_SYNC", "SOURCE_FETCH", "VERIFY", "IMAGE", "GITOPS", "ARGO_RUNTIME", "VALIDATION"], [[ - "YAML publishCurrent", - stageBudgets.sourceSyncSeconds ?? "-", - stageBudgets.sourceFetchSeconds ?? "-", - stageBudgets.monitorWebVerifySeconds ?? "-", - stageBudgets.imageBuildSeconds ?? "-", - stageBudgets.gitopsSeconds ?? "-", - stageBudgets.argoRuntimeSeconds ?? "-", - stageBudgets.dashboardVerifySeconds ?? "-", - ]]), - ]; - if (Object.keys(publish).length > 0) { - const payloadImageBuild = record(publishPayload.imageBuild); - const payloadEnvReuse = record(publishPayload.envReuse); - lines.push( - "", - table(["ENV_REUSE", "NODE_DEPS", "BUILD_PACKAGE", "BUILD_NETWORK", "CACHE", "CACHE_LINES"], [[ - payloadEnvReuse.dependencyReuse ?? "-", - payloadEnvReuse.nodeDepsPresent ?? "-", - payloadImageBuild.packageMode ?? "-", - payloadImageBuild.networkMode ?? "-", - payloadImageBuild.layerCache ?? "-", - payloadImageBuild.cacheHitLines ?? "-", - ]]), - ); - } - lines.push( - "", - Object.keys(health).length === 0 - ? "HEALTH_VALIDATION\n-" - : table(["ENDPOINT", "HTTP", "OK", "STATUS", "PUBLIC_URL", "INTERNAL_URL"], [[ - health.endpoint ?? validationPlan.endpoint ?? "-", - health.httpStatus ?? "-", - healthBody.ok ?? "-", - healthBody.status ?? "-", - health.publicUrl ?? "-", - health.internalUrl ?? "-", - ]]), - "", - slowStages.length === 0 ? "SLOW_STAGES\n-" : [ - "SLOW_STAGES", - table(["STAGE", "ELAPSED_MS", "BUDGET_S", "SUGGESTION"], slowStages.map((stage) => [stage.stage, stage.elapsedMs, stage.budgetSeconds, stage.suggestion])), - ].join("\n"), - "", - warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), - "", - Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), - "", - Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ - "RECOVERY_NEXT", - table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), - ` publish-current: ${recoveryNext.publishCurrent ?? "-"}`, - ` status: ${recoveryNext.nextStatus ?? "-"}`, - ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, - ` sync: ${recoveryNext.gitMirrorSync ?? "-"}`, - ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, - ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, - ].join("\n"), - "", - "NEXT", - ` publish-current: ${next.publishCurrent ?? "-"}`, - ` status: ${next.controlPlaneStatus ?? "-"}`, - ` post-deploy-dashboard: ${next.dashboardVerify ?? "-"}`, - ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, - ` sync: ${next.gitMirrorSync ?? "-"}`, - ` flush: ${next.gitMirrorFlush ?? "-"}`, - "", - "DISCLOSURE", - ` end-to-end and stage budgets are read from ${Object.keys(validationPlan).length > 0 ? "publishCurrent YAML and runtime.healthPath" : "YAML-required publishCurrent fields"}.`, - " CI/CD validation only checks the configured health endpoint; web-probe, Playwright and browser dashboard checks are post-deploy evidence, not this gate.", - " image build uses Tekton PipelineRun and BuildKit; this command does not require Docker daemon/socket/build.", - ); - return lines.join("\n"); -} - -function renderImageResult(result: Record): string { - const source = record(result.source); - const sourceMirror = record(result.sourceMirror); - const sourceMirrorSync = record(result.sourceMirrorSync); - const image = record(result.image); - const monitorWeb = record(image.monitorWeb); - const registry = record(result.registry); - const publish = record(result.publish); - const blocker = record(result.blocker); - const next = record(result.next); - const warnings = Array.isArray(result.warnings) ? result.warnings : []; - return [ - String(result.command), - "", - table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]), - "", - table(["SOURCE_REPO", "BRANCH", "COMMIT", "AUTHORITY", "STAGE_REF", "MIRROR"], [[source.repository, source.branch, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), short(source.mirrorCommit)]]), - "", - Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]), - "", - table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]), - "", - Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "BUILD_STATE", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, `${record(monitorWeb.imageBuildState).mode ?? "-"}:${record(monitorWeb.imageBuildState).path ?? record(monitorWeb.imageBuildState).claimName ?? record(monitorWeb.imageBuildState).sizeLimit ?? "-"}`, monitorWeb.imageBuildContextIgnore]]), - "", - Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]), - "", - Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), - "", - Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), - "", - warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), - "", - Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), - "", - "NEXT", - ` status: ${next.status ?? "-"}`, - ` dry-run: ${next.dryRun ?? "-"}`, - ` confirm: ${next.confirm ?? "-"}`, - ` trigger: ${next.controlPlaneTrigger ?? "-"}`, - ` control-plane: ${next.controlPlanePlan ?? "-"}`, - "", - "DISCLOSURE", - " valuesRedacted=true; image status shows refs, hashes and object names only.", - ].join("\n"); -} - -function renderControlPlaneResult(result: Record): string { - const source = record(result.source); - const image = record(result.image); - const gitops = record(result.gitops); - const argo = record(result.argo); - const validation = record(result.validation); - const observability = record(result.observability); - const observed = record(result.observed); - const sourceMirrorSync = record(result.sourceMirrorSync); - const publish = record(result.publish); - const flush = record(result.flush); - const runtimeSecretsApply = record(result.runtimeSecretsApply); - const publicExposureApply = record(result.publicExposureApply); - const publicExposureCaddy = record(publicExposureApply.caddy); - const argoApply = record(result.argoApply); - const blocker = record(result.blocker); - const statusDiagnosis = record(result.statusDiagnosis); - const targetValidation = record(result.targetValidation); - const targetValidationBusiness = record(targetValidation.businessStatus); - const recoveryNext = record(result.recoveryNext); - const next = record(result.next); - const warnings = Array.isArray(result.warnings) ? result.warnings : []; - return [ - String(result.command), - "", - table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]), - "", - table(["SOURCE", "COMMIT", "AUTHORITY", "STAGE_REF", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), source.sourceAuthority ?? "-", short(source.stageRef), image.ref, short(gitops.manifestSha256)]]), - "", - table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]), - "", - table(["SCENARIO", "MAX_SECONDS", "CI_WAIT", "QVERIFY", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.controlPlaneWaitMaxSeconds ?? "-", validation.quickVerifyMode ?? "-", validation.automaticSecondPath]]), - "", - Object.keys(observability).length === 0 ? "OTEL\n-" : table(["ENABLED", "ENDPOINT", "SERVICE", "COVERAGE"], [[observability.enabled, observability.endpointConfigured, observability.serviceName, observability.coverage]]), - "", - renderObservedStatus(observed), - "", - Object.keys(statusDiagnosis).length === 0 ? "STATUS_DIAGNOSIS\n-" : [ - "STATUS_DIAGNOSIS", - table(["CODE", "PHASE", "PIPELINERUN", "SOURCE", "REGISTRY", "GIT_MIRROR", "GITOPS", "ARGO", "RUNTIME"], [[ - statusDiagnosis.code, - statusDiagnosis.phase, - statusDiagnosis.pipelineRun, - statusDiagnosis.sourceMirror, - statusDiagnosis.registry, - statusDiagnosis.gitMirror, - statusDiagnosis.gitops, - statusDiagnosis.argo, - statusDiagnosis.runtime, - ]]), - ].join("\n"), - "", - Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), - "", - Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "BUSINESS", "ERROR_TITLE", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[ - targetValidation.ok, - targetValidation.status, - targetValidationBusiness.status ?? "-", - targetValidation.errorTitleZh ?? targetValidation.failureTitleZh ?? targetValidationBusiness.errorTitleZh ?? "-", - targetValidation.scenarioId, - targetValidation.runId, - targetValidation.observerId, - short(targetValidation.reportJsonSha256), - targetValidation.findingCount, - targetValidation.artifactCount, - ]]), - "", - Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), - "", - Object.keys(flush).length === 0 - ? "FLUSH\n-" - : flush.mode === "async-job" - ? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]]) - : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]), - "", - Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]), - "", - Object.keys(publicExposureApply).length === 0 ? "PUBLIC_EXPOSURE_APPLY\n-" : table(["OK", "SECRET", "CADDY", "HOST", "ROUTE_HTTP"], [[publicExposureApply.ok, record(publicExposureApply.secret).ok, record(publicExposureApply.caddy).ok, publicExposureApply.hostname, record(publicExposureApply.caddy).routeProbeHttpStatus ?? "-"]]), - "", - Object.keys(publicExposureCaddy).length === 0 || publicExposureCaddy.ok === true - ? "CADDY_APPLY_DETAIL\n-" - : table(["PY", "VALIDATE", "RELOAD", "PROBE", "HTTP", "BLOCK", "ACTIVE", "ERROR", "STDOUT", "STDERR"], [[publicExposureCaddy.pythonExitCode, publicExposureCaddy.validateExitCode, publicExposureCaddy.reloadExitCode, publicExposureCaddy.routeProbeExitCode, publicExposureCaddy.routeProbeHttpStatus, publicExposureCaddy.afterBlockPresent, publicExposureCaddy.active, short(publicExposureCaddy.errorPreview), short(record(publicExposureCaddy.result).stdoutPreview), short(record(publicExposureCaddy.result).stderrPreview)]]), - "", - Object.keys(argoApply).length === 0 ? "ARGO_APPLY\n-" : table(["OK", "EXIT", "PREVIEW"], [[argoApply.ok, record(argoApply.result).exitCode, record(argoApply.result).stdoutPreview]]), - "", - warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), - "", - Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), - "", - Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ - "RECOVERY_NEXT", - table(["REASON", "PIPELINERUN", "DIGEST", "GITOPS"], [[recoveryNext.reason, recoveryNext.pipelineRun ?? "-", short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), - ` publish-current: ${recoveryNext.publishCurrent ?? "-"}`, - ` status: ${recoveryNext.nextStatus ?? "-"}`, - ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, - ` sync: ${recoveryNext.gitMirrorSync ?? "-"}`, - ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, - ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, - ].join("\n"), - "", - "NEXT", - ` plan: ${next.plan ?? "-"}`, - ` status: ${next.status ?? "-"}`, - ` image: ${next.image ?? "-"}`, - ` trigger-current: ${next.triggerCurrent ?? "-"}`, - ` apply: ${next.apply ?? "-"}`, - ` validate: ${next.validate ?? "-"}`, - ` quick-verify: ${next.quickVerify ?? "-"}`, - ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, - ` sync: ${next.gitMirrorSync ?? "-"}`, - ` flush: ${next.gitMirrorFlush ?? "-"}`, - "", - "DISCLOSURE", - " default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.", - " sentinel unavailable policy is structured-failure; no automatic second execution path is rendered.", - ].join("\n"); -} - -function renderObservedStatus(observed: Record): string { - const rows = [ - observedStatusRow("source", observed.sourceMirror), - observedStatusRow("registry", observed.registry), - observedStatusRow("git-mirror", observed.gitMirror), - observedStatusRow("gitops", observed.gitops), - observedStatusRow("argo", observed.argo), - observedStatusRow("runtime", observed.runtime), - observedStatusRow("cadence", observed.cadence), - ].filter((row) => row !== null); - if (rows.length === 0) return "OBSERVED\n-"; - return table(["CHECK", "OK", "DETAIL", "EXIT", "TIMED_OUT", "PREVIEW"], rows); -} - -function observedStatusRow(name: string, value: unknown): unknown[] | null { - const item = record(value); - if (Object.keys(item).length === 0) return null; - const result = record(item.result); - return [name, item.ok, observedDetail(name, item), result.exitCode, result.timedOut, result.stdoutPreview]; -} - -function observedDetail(name: string, item: Record): string { - if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`; - if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`; - if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`; - if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`; - if (name === "argo") { - const diagnostics = record(item.diagnostics); - const problems = Array.isArray(diagnostics.problemResources) ? diagnostics.problemResources : []; - const first = problems.find((entry) => typeof entry === "object" && entry !== null && !Array.isArray(entry)) as Record | undefined; - const problemText = Number(diagnostics.problemResourceCount ?? 0) > 0 - ? ` degraded=${diagnostics.problemResourceCount}:${first?.kind ?? "-"} ${first?.namespace ?? "-"}/${first?.name ?? "-"} ${first?.healthStatus ?? first?.status ?? "-"}` - : ""; - return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}${problemText}`; - } - if (name === "runtime") { - const probe = record(item.probe); - const deployment = record(probe.deployment); - return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`; - } - if (name === "cadence") { - if (item.skipped === true) return `${item.reason ?? "skipped"}`; - const probe = record(item.probe); - return `${probe.code ?? "ok"} schedule=${probe.schedule ?? "-"}/${probe.expectedSchedule ?? "-"} last=${probe.lastScheduleTime ?? "-"} jobs=${probe.jobCount ?? "-"}`; - } - return "-"; -} - -export function renderAsyncJobResult(result: Record): string { - const job = record(result.job); - const next = record(result.next); - return [ - String(result.command), - "", - table(["NODE", "LANE", "MODE", "MUTATION", "JOB"], [[result.node, result.lane, result.mode, result.mutation, job.id]]), - "", - table(["STATUS", "NAME", "CREATED"], [[job.status, job.name, job.createdAt]]), - "", - "NEXT", - ` status: ${next.status ?? "-"}`, - ` wait: ${next.wait ?? "-"}`, - "", - "DISCLOSURE", - " confirmed operation is delegated to UniDesk job status to keep interactive calls bounded.", - ].join("\n"); -} - -export function rendered(ok: boolean, command: string, text: string): RenderedCliResult { - return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" }; -} - -function readConfigFile(file: string): unknown { - if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`); - const abs = rootPath(file); - if (!existsSync(abs)) throw new Error(`${file} does not exist`); - return Bun.YAML.parse(readFileSync(abs, "utf8")) as unknown; -} - -function configRefFile(ref: string): string { - const [file, path, extra] = ref.split("#"); - if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) throw new Error(`invalid configRef: ${ref}`); - return file; -} - -function valueAtPath(value: unknown, path: string): unknown { - let current: unknown = value; - for (const segment of path.split(".")) { - const match = /^([A-Za-z0-9_-]+)?(?:\[(\d+)\])?$/u.exec(segment); - if (match === null) return undefined; - if (match[1] !== undefined) { - if (!isRecord(current)) return undefined; - current = current[match[1]]; - } - if (match[2] !== undefined) { - if (!Array.isArray(current)) return undefined; - current = current[Number(match[2])]; - } - } - return current; -} - -export function stringAt(value: unknown, path: string): string { - const found = valueAtPath(value, path); - if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`); - return found; -} - -export function nonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null; -} - -function stringField(value: Record, path: string): string { - return stringAt(value, path); -} - -function stringTarget(value: unknown, label: string): string { - if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must resolve to a non-empty string`); - return value; -} - -export function numberAt(value: unknown, path: string): number { - const found = valueAtPath(value, path); - if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`); - return found; -} - -function booleanAt(value: unknown, path: string): boolean { - const found = valueAtPath(value, path); - if (typeof found !== "boolean") throw new Error(`${path} must be a boolean`); - return found; -} - -function finiteNumberOrNull(value: unknown): number | null { - return typeof value === "number" && Number.isFinite(value) ? value : null; -} - -export function arrayAt(value: unknown, path: string): unknown[] { - const found = valueAtPath(value, path); - if (!Array.isArray(found)) throw new Error(`${path} must be an array`); - return found; -} - -function arrayAtNullable(value: unknown, path: string): Record[] { - const found = valueAtPath(value, path); - return Array.isArray(found) ? found.map(record) : []; -} - -export function recordTarget(value: unknown, label: string): Record { - if (!isRecord(value)) throw new Error(`${label} must resolve to an object`); - return value; -} - -export function record(value: unknown): Record { - return isRecord(value) ? value : {}; -} - -export function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function manifestObjectSummary(items: readonly Record[]): readonly Record[] { - return items.map((item) => ({ - kind: item.kind ?? null, - name: record(item.metadata).name ?? null, - namespace: record(item.metadata).namespace ?? null, - })); -} - -export function compactCommand(result: CommandResult): CompactCommandResult { - return { - exitCode: result.exitCode, - timedOut: result.timedOut, - stdoutBytes: Buffer.byteLength(result.stdout), - stderrBytes: Buffer.byteLength(result.stderr), - stdoutPreview: result.stdout.trim().slice(0, 500), - stderrPreview: result.stderr.trim().slice(0, 500), - }; -} - -export function parseJsonObject(text: string): Record | null { - const trimmed = text.trim(); - if (trimmed.length === 0) return null; - try { - const parsed = JSON.parse(trimmed) as unknown; - return isRecord(parsed) ? parsed : null; - } catch { - return null; - } -} - -export function table(headers: string[], rows: unknown[][]): string { - const normalized = [headers, ...rows.map((row) => row.map(text))]; - const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length))); - return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n"); -} - -export function text(value: unknown): string { - if (value === undefined || value === null || value === "") return "-"; - if (typeof value === "boolean") return value ? "true" : "false"; - return String(value).replace(/\s+/gu, " ").trim(); -} - -export function short(value: unknown): string { - const raw = text(value); - if (raw === "-") return raw; - if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`; - if (/^[0-9a-f]{40}$/iu.test(raw)) return raw.slice(0, 12); - return raw.length > 42 ? `${raw.slice(0, 39)}...` : raw; -} - -function sha256(textValue: string): string { - return `sha256:${createHash("sha256").update(textValue).digest("hex")}`; -} - -export function shellQuote(value: string): string { - return `'${value.replace(/'/gu, "'\\''")}'`; -} - -function tomlEscape(value: string): string { - return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"'); + const parsedResolution = resolveSentinelChildJson(result, "web-probe-sentinel-caddy-apply"); + const parsed = parsedResolution.parsed; + return { ok: result.exitCode === 0 && parsed?.ok === true, routePrefix, rootOrder, ...record(parsed), stdoutRecovery: parsedResolution.diagnostics, result: compactCommand(result), valuesRedacted: true }; }