Codex
1127 lines
71 KiB
TypeScript
1127 lines
71 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p16-cicd-source-snapshot.
|
|
// Responsibility: source mirror and publish remote Job/PipelineRun orchestration for web-probe sentinel CI/CD.
|
|
import { runCommand } from "./command";
|
|
import { repoRoot, rootPath } from "./config";
|
|
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
|
|
import { emitWebProbeSentinelSpan } from "./hwlab-node-web-sentinel-otel";
|
|
import {
|
|
arrayAt,
|
|
compactCommand,
|
|
finiteNumberOrNull,
|
|
monitorWebBuildkitStatePlan,
|
|
nonEmptyString,
|
|
numberAt,
|
|
numberAtNullable,
|
|
parseJsonObject,
|
|
record,
|
|
resolveSentinelChildJson,
|
|
safeKubernetesSegment,
|
|
sentinelCliSuffix,
|
|
sentinelPipelineRunName,
|
|
sentinelProgressEvent,
|
|
sentinelSourceSnapshotStageRefPrefix,
|
|
shellQuote,
|
|
short,
|
|
stringAt,
|
|
stringAtNullable,
|
|
text,
|
|
valueAtPath,
|
|
type SentinelCicdState,
|
|
type SentinelRemoteJobResult,
|
|
} from "./hwlab-node-web-sentinel-cicd-shared";
|
|
|
|
export function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult {
|
|
const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`;
|
|
const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
|
const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName);
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) {
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane });
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror");
|
|
}
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
|
let slowWarningSent = false;
|
|
let polls = 0;
|
|
let lastProbe: Record<string, unknown> = {};
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-source-mirror-job-probe");
|
|
const probe = probeResolution.parsed ?? {};
|
|
lastProbe = { ...probe, stdoutRecovery: probeResolution.diagnostics, capture: compactCommand(probeCapture) };
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
sentinelProgressEvent("sentinel.source-mirror.progress", {
|
|
phase: "remote-job",
|
|
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
|
jobName,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
pod: probe.pod ?? null,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
});
|
|
if (probe.succeeded === true) {
|
|
const ok = payload.ok === true;
|
|
return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
|
|
}
|
|
if (probe.failed === true) {
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
|
|
}
|
|
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
|
slowWarningSent = true;
|
|
sentinelProgressEvent("sentinel.source-mirror.warning", { warning: `source mirror sync exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror");
|
|
}
|
|
|
|
export function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult {
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
jobName: "-",
|
|
payload: { ok: false, status: phase, reason, valuesRedacted: true },
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-source-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
};
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name: jobName, namespace, labels },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
|
|
ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
volumes: [
|
|
sentinelGitMirrorCacheVolume(state),
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
],
|
|
containers: [{
|
|
name: "sync",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)],
|
|
volumeMounts: [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
|
|
return sentinelSourceMirrorSyncShellFromConfig(state.cicd, state.controlPlaneNode, jobName, state.sourceHead.commit);
|
|
}
|
|
|
|
export function sentinelSourceMirrorSyncShellFromConfig(cicd: Record<string, unknown>, controlPlaneNode: Record<string, unknown>, jobName: string, selectedCommit: string | null): string {
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`source_repository=${shellQuote(stringAt(cicd, "source.repository"))}`,
|
|
`source_branch=${shellQuote(stringAt(cicd, "source.branch"))}`,
|
|
`source_git_url=${shellQuote(stringAt(cicd, "source.gitSshUrl"))}`,
|
|
`source_commit=${shellQuote(selectedCommit ?? "")}`,
|
|
`source_stage_ref_prefix=${shellQuote(sentinelSourceSnapshotStageRefPrefix(cicd))}`,
|
|
"started_ms=$(node -e 'console.log(Date.now())')",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
...sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode),
|
|
"repo=\"/cache/${source_repository}.git\"",
|
|
"mkdir -p \"$(dirname \"$repo\")\"",
|
|
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
|
|
" git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
|
"else",
|
|
" rm -rf \"$repo\"",
|
|
" git init --bare \"$repo\"",
|
|
" git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
|
"fi",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
|
"git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
|
|
"git --git-dir=\"$repo\" config http.uploadpack true",
|
|
"git --git-dir=\"$repo\" config http.receivepack true",
|
|
"fetch_ok=0",
|
|
"for attempt in 1 2 3; do",
|
|
" if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi",
|
|
" code=$?",
|
|
" printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2",
|
|
" sleep $((attempt * 5))",
|
|
"done",
|
|
"test \"$fetch_ok\" = 1",
|
|
"mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
|
|
"if [ -z \"$source_commit\" ]; then source_commit=\"$mirror_commit\"; fi",
|
|
"git --git-dir=\"$repo\" cat-file -e \"$source_commit^{commit}\"",
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
"stage_ref=\"${source_stage_ref_prefix%/}/${source_commit}\"",
|
|
"git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
|
|
"git --git-dir=\"$repo\" update-ref \"$stage_ref\" \"$source_commit\"",
|
|
"git --git-dir=\"$repo\" update-server-info",
|
|
"finished_ms=$(node -e 'console.log(Date.now())')",
|
|
"node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$stage_ref\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
|
|
"const [jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, startedMs, finishedMs] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, stageRef, sourceAuthority:'git-mirror-snapshot', elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
|
|
return sentinelSourceMirrorSshSetupShellLinesForNode(state.controlPlaneNode);
|
|
}
|
|
|
|
function sentinelSourceMirrorSshSetupShellLinesForNode(controlPlaneNode: Record<string, unknown>): string[] {
|
|
const proxy = record(valueAtPath(controlPlaneNode, "egressProxy"));
|
|
const serviceName = nonEmptyString(proxy.serviceName);
|
|
const namespace = nonEmptyString(proxy.namespace);
|
|
const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
|
|
const hostRouteProxyUrl = nonEmptyString(proxy.proxyUrl);
|
|
const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : "";
|
|
let proxyHost: string | null = null;
|
|
let proxyPort: number | null = null;
|
|
let proxyUrl: string | null = null;
|
|
if (serviceName !== null && namespace !== null && port !== null) {
|
|
proxyHost = `${serviceName}.${namespace}.svc.cluster.local`;
|
|
proxyPort = port;
|
|
proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
|
} else if (hostRouteProxyUrl !== null) {
|
|
try {
|
|
const parsed = new URL(hostRouteProxyUrl);
|
|
const parsedPort = Number.parseInt(parsed.port || "80", 10);
|
|
if ((parsed.protocol === "http:" || parsed.protocol === "https:") && parsed.hostname.length > 0 && Number.isInteger(parsedPort)) {
|
|
proxyHost = parsed.hostname;
|
|
proxyPort = parsedPort;
|
|
proxyUrl = hostRouteProxyUrl;
|
|
}
|
|
} catch {
|
|
proxyHost = null;
|
|
proxyPort = null;
|
|
proxyUrl = null;
|
|
}
|
|
}
|
|
const useProxy = proxyHost !== null && proxyPort !== null && proxyUrl !== null;
|
|
if (!useProxy) {
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2",
|
|
"unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy",
|
|
"export NO_PROXY='*'",
|
|
"export no_proxy='*'",
|
|
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
"exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"",
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
|
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
|
return [
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
`printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`,
|
|
`export HTTP_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export HTTPS_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export ALL_PROXY=${shellQuote(proxyUrl)}`,
|
|
`export http_proxy=${shellQuote(proxyUrl)}`,
|
|
`export https_proxy=${shellQuote(proxyUrl)}`,
|
|
`export all_proxy=${shellQuote(proxyUrl)}`,
|
|
`export NO_PROXY=${shellQuote(noProxy)}`,
|
|
`export no_proxy=${shellQuote(noProxy)}`,
|
|
"cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'",
|
|
"#!/usr/bin/env node",
|
|
"const net = require('node:net');",
|
|
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
|
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
|
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
|
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
|
|
" console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');",
|
|
" process.exit(64);",
|
|
"}",
|
|
"let settled = false;",
|
|
"let tunnelEstablished = false;",
|
|
"function finish(code, message) {",
|
|
" if (settled) return;",
|
|
" settled = true;",
|
|
" if (message) console.error('sentinel source-mirror proxy-connect: ' + message);",
|
|
" process.exit(code);",
|
|
"}",
|
|
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
|
"let buffer = Buffer.alloc(0);",
|
|
"socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
|
|
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
|
|
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
|
|
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
|
|
"function onData(chunk) {",
|
|
" buffer = Buffer.concat([buffer, chunk]);",
|
|
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
|
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
|
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
|
|
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
|
|
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
|
|
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
|
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
|
|
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
|
|
" socket.destroy();",
|
|
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
|
|
" return;",
|
|
" }",
|
|
" socket.off('data', onData);",
|
|
" socket.setTimeout(0);",
|
|
" tunnelEstablished = true;",
|
|
" const rest = buffer.slice(headerEnd + 4);",
|
|
" if (rest.length) process.stdout.write(rest);",
|
|
" process.stdin.on('error', () => {});",
|
|
" process.stdout.on('error', () => {});",
|
|
" process.stdin.pipe(socket);",
|
|
" socket.pipe(process.stdout);",
|
|
"}",
|
|
"socket.on('data', onData);",
|
|
"NODE_PROXY",
|
|
"chmod 0700 /tmp/sentinel-github-proxy-connect.cjs",
|
|
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
|
"#!/bin/sh",
|
|
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`,
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
|
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
];
|
|
}
|
|
|
|
export function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number, rerun: boolean): SentinelRemoteJobResult {
|
|
const pipelineRunName = sentinelPipelineRunName(state, rerun);
|
|
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRunName, publishGitops);
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "submitting", pipelineRun: pipelineRunName, publishGitops, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createTektonPipelineRunScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
if (created.exitCode !== 0) {
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "failed", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-pipelinerun", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish");
|
|
}
|
|
sentinelProgressEvent("sentinel.publish.progress", { phase: "create-pipelinerun", status: "succeeded", pipelineRun: pipelineRunName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane });
|
|
const startedAt = Date.now();
|
|
const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
|
|
const warningBudgetMs = Math.max(1, Math.trunc(controlPlaneWaitWarningSeconds(state))) * 1000;
|
|
let slowWarningSent = false;
|
|
let polls = 0;
|
|
let lastProbe: Record<string, unknown> = {};
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
polls += 1;
|
|
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeTektonPipelineRunScript(namespace, pipelineRunName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
const probeResolution = resolveSentinelChildJson(probeCapture, "web-probe-sentinel-publish-pipelinerun-probe");
|
|
const probe = probeResolution.parsed ?? {};
|
|
lastProbe = { ...probe, stdoutRecovery: probeResolution.diagnostics, capture: compactCommand(probeCapture) };
|
|
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
|
sentinelProgressEvent("sentinel.publish.progress", {
|
|
phase: "remote-pipelinerun",
|
|
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
|
pipelineRun: pipelineRunName,
|
|
publishGitops,
|
|
polls,
|
|
elapsedMs: Date.now() - startedAt,
|
|
taskRun: probe.taskRun ?? null,
|
|
pod: probe.pod ?? null,
|
|
sourceCommit: state.sourceHead.commit,
|
|
node: state.spec.nodeId,
|
|
lane: state.spec.lane,
|
|
});
|
|
if (probe.succeeded === true) {
|
|
const ok = payload.ok === true;
|
|
return withSentinelRemoteJobDiagnostics(state, { ok, phase: "pipelinerun-succeeded", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
|
}
|
|
if (probe.failed === true) {
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-failed", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
|
}
|
|
if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) {
|
|
slowWarningSent = true;
|
|
sentinelProgressEvent("sentinel.publish.warning", { warning: `remote publish PipelineRun exceeded configured ${Math.round(warningBudgetMs / 1000)}s timing budget; non-blocking timing alert`, pipelineRun: pipelineRunName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
|
}
|
|
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
|
}
|
|
return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "pipelinerun-timeout", resourceKind: "PipelineRun", jobName: pipelineRunName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish");
|
|
}
|
|
|
|
export function sentinelPublishPipelineRunManifest(state: SentinelCicdState, pipelineRunName: string, publishGitops: boolean): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const buildkitImage = requireSentinelBuildkitImage(state);
|
|
const proxyEnv = sentinelImageBuildProxyEnv(state);
|
|
const labels = {
|
|
"app.kubernetes.io/name": "web-probe-sentinel-publish",
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
|
"unidesk.ai/node": state.spec.nodeId,
|
|
"unidesk.ai/lane": state.spec.lane,
|
|
"unidesk.ai/ci-system": "tekton",
|
|
};
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRunName,
|
|
namespace,
|
|
labels,
|
|
annotations: {
|
|
"unidesk.ai/source-commit": state.sourceHead.commit ?? "",
|
|
"unidesk.ai/source-authority": state.sourceHead.sourceAuthority,
|
|
"unidesk.ai/source-stage-ref": state.sourceHead.stageRef ?? "",
|
|
"unidesk.ai/gitops-target-revision": stringAt(state.cicd, "argo.targetRevision"),
|
|
"unidesk.ai/publish-gitops": publishGitops ? "true" : "false",
|
|
},
|
|
},
|
|
spec: {
|
|
timeouts: { pipeline: `${numberAt(state.cicd, "builder.activeDeadlineSeconds")}s` },
|
|
taskRunTemplate: {
|
|
podTemplate: {
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
securityContext: { fsGroup: 1000 },
|
|
},
|
|
},
|
|
pipelineSpec: {
|
|
tasks: [{
|
|
name: "publish",
|
|
taskSpec: {
|
|
volumes: [
|
|
sentinelGitMirrorCacheVolume(state),
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
|
|
sentinelBuildkitStateVolume(state),
|
|
{ name: "tmp", emptyDir: {} },
|
|
],
|
|
steps: [
|
|
{
|
|
name: "source",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env: proxyEnv,
|
|
script: tektonShellScript(sentinelPublishSourceShell(state, pipelineRunName)),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
},
|
|
{
|
|
name: "prepare-buildkit-state",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
script: tektonShellScript("set -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit"),
|
|
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
|
volumeMounts: [
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
],
|
|
},
|
|
{
|
|
name: "image-build",
|
|
image: buildkitImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env: [
|
|
...proxyEnv,
|
|
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
|
],
|
|
script: tektonShellScript(sentinelPublishImageBuildShell(state, pipelineRunName)),
|
|
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
{ name: "tmp", mountPath: "/tmp" },
|
|
],
|
|
},
|
|
{
|
|
name: "publish",
|
|
image: state.image.baseImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env: proxyEnv,
|
|
script: tektonShellScript(sentinelPublishShell(state, pipelineRunName, publishGitops)),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
}],
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function tektonShellScript(body: string): string {
|
|
return `#!/bin/sh\n${body}`;
|
|
}
|
|
|
|
function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string, unknown> {
|
|
return sentinelGitMirrorCacheVolumeFromTarget(state.controlPlaneTarget);
|
|
}
|
|
|
|
export function sentinelGitMirrorCacheVolumeFromTarget(controlPlaneTarget: Record<string, unknown>): Record<string, unknown> {
|
|
const hostPath = nonEmptyString(valueAtPath(controlPlaneTarget, "gitMirror.cacheHostPath"));
|
|
if (hostPath !== null) return { name: "cache", hostPath: { path: hostPath, type: "DirectoryOrCreate" } };
|
|
return { name: "cache", persistentVolumeClaim: { claimName: stringAt(controlPlaneTarget, "gitMirror.cachePvcName") } };
|
|
}
|
|
|
|
function sentinelBuildkitStateVolume(state: SentinelCicdState): Record<string, unknown> {
|
|
const buildkitState = monitorWebBuildkitStatePlan(state.cicd);
|
|
const mode = stringAt(buildkitState, "mode");
|
|
if (mode === "hostPath") {
|
|
return {
|
|
name: "buildkit-state",
|
|
hostPath: {
|
|
path: stringAt(buildkitState, "path"),
|
|
type: stringAt(buildkitState, "type"),
|
|
},
|
|
};
|
|
}
|
|
if (mode === "persistentVolumeClaim") {
|
|
return { name: "buildkit-state", persistentVolumeClaim: { claimName: stringAt(buildkitState, "claimName") } };
|
|
}
|
|
if (mode === "emptyDir") {
|
|
return { name: "buildkit-state", emptyDir: { sizeLimit: stringAt(buildkitState, "sizeLimit") } };
|
|
}
|
|
throw new Error(`monitorWeb.imageBuild.buildkitState.mode must be hostPath, persistentVolumeClaim or emptyDir, got ${mode}`);
|
|
}
|
|
|
|
function requireSentinelBuildkitImage(state: SentinelCicdState): string {
|
|
const image = state.spec.buildkit?.sidecarImage;
|
|
if (typeof image !== "string" || image.length === 0) {
|
|
throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`);
|
|
}
|
|
return image;
|
|
}
|
|
|
|
function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> {
|
|
const proxy = state.spec.networkProfile.imageBuildProxy;
|
|
const noProxy = proxy.noProxy.join(",");
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxy.http },
|
|
{ name: "http_proxy", value: proxy.http },
|
|
{ name: "HTTPS_PROXY", value: proxy.https },
|
|
{ name: "https_proxy", value: proxy.https },
|
|
{ name: "ALL_PROXY", value: proxy.all },
|
|
{ name: "all_proxy", value: proxy.all },
|
|
{ name: "NO_PROXY", value: noProxy },
|
|
{ name: "no_proxy", value: noProxy },
|
|
];
|
|
}
|
|
|
|
function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string {
|
|
const monitorWeb = record(state.image.monitorWeb);
|
|
const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => {
|
|
if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths");
|
|
return item;
|
|
})), "utf8").toString("base64");
|
|
const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64");
|
|
const envReuseMode = stringAt(monitorWeb, "envReuseMode");
|
|
const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
|
|
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
|
|
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
|
|
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
|
`source_stage_ref=${shellQuote(state.sourceHead.stageRef ?? "")}`,
|
|
`checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
|
|
`dockerfile_b64=${shellQuote(dockerfileB64)}`,
|
|
`env_reuse_mode=${shellQuote(envReuseMode)}`,
|
|
`env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`,
|
|
"meta_dir=/workspace/meta",
|
|
"event_log=/workspace/publish-events.log",
|
|
"worktree=/workspace/source",
|
|
"rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json",
|
|
"mkdir -p \"$meta_dir\"",
|
|
": > \"$event_log\"",
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"started_ms=$(now_ms)",
|
|
"write_meta started_ms \"$started_ms\"",
|
|
"write_meta source_commit \"$source_commit\"",
|
|
"write_meta source_stage_ref \"$source_stage_ref\"",
|
|
"test -n \"$source_commit\"",
|
|
"test -n \"$source_stage_ref\"",
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
"git init \"$worktree\"",
|
|
"cd \"$worktree\"",
|
|
"git remote add origin \"$source_git_url\"",
|
|
"git config core.sparseCheckout true",
|
|
"git config remote.origin.promisor true",
|
|
"git config remote.origin.partialclonefilter blob:none",
|
|
"CHECKOUT_PATHS_B64=\"$checkout_paths_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const paths = JSON.parse(Buffer.from(process.env.CHECKOUT_PATHS_B64 || '', 'base64').toString('utf8'));",
|
|
"fs.mkdirSync('.git/info', { recursive: true });",
|
|
"fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');",
|
|
"NODE",
|
|
"source_fetch_started_ms=$(now_ms)",
|
|
"write_meta source_fetch_started_ms \"$source_fetch_started_ms\"",
|
|
"emit_stage source-fetch running \"$source_fetch_started_ms\"",
|
|
"git fetch --depth=1 --filter=blob:none origin \"+$source_stage_ref:refs/remotes/origin/unidesk-source-snapshot\"",
|
|
"git checkout --detach \"$source_commit\"",
|
|
"mirror_commit=$(git rev-parse HEAD)",
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
"write_meta mirror_commit \"$mirror_commit\"",
|
|
"source_fetch_finished_ms=$(now_ms)",
|
|
"write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"",
|
|
"emit_stage source-fetch succeeded \"$source_fetch_started_ms\"",
|
|
"env_reuse_node_deps_present=false",
|
|
"env_reuse_node_deps_entries=0",
|
|
"if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi",
|
|
"env_reuse_linked_node_deps=0",
|
|
"rm -rf node_modules",
|
|
"if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi",
|
|
"write_meta env_reuse_mode \"$env_reuse_mode\"",
|
|
"write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"",
|
|
"write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"",
|
|
"write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"",
|
|
"write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"",
|
|
"node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'",
|
|
"const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
|
|
"NODE",
|
|
"node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"",
|
|
"const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
|
|
"NODE",
|
|
"monitor_web_verify_started_ms=$(now_ms)",
|
|
"write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"",
|
|
"emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"",
|
|
"if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi",
|
|
"cat /tmp/web-probe-sentinel-monitor-web-verify.log",
|
|
"monitor_web_verify_finished_ms=$(now_ms)",
|
|
"write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"",
|
|
"emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"",
|
|
"mkdir -p .unidesk-sentinel-bin",
|
|
"cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'",
|
|
"#!/bin/sh",
|
|
"exec bun /app/scripts/ssh-cli.ts \"$@\"",
|
|
"SH_TRANS",
|
|
"chmod 0755 .unidesk-sentinel-bin/trans",
|
|
"DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));",
|
|
"NODE",
|
|
"cat > .dockerignore <<'EOF_DOCKERIGNORE'",
|
|
".git",
|
|
".git/**",
|
|
".state",
|
|
".state/**",
|
|
"logs",
|
|
"logs/**",
|
|
"node_modules",
|
|
"node_modules/**",
|
|
"**/node_modules",
|
|
"**/node_modules/**",
|
|
"**/dist",
|
|
"**/dist/**",
|
|
"**/target",
|
|
"**/target/**",
|
|
"**/coverage",
|
|
"**/coverage/**",
|
|
"npm-debug.log*",
|
|
".env",
|
|
".env.*",
|
|
"EOF_DOCKERIGNORE",
|
|
"context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')",
|
|
"write_meta context_ignore_entries \"$context_ignore_entries\"",
|
|
"chmod -R a+rwX /workspace",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string {
|
|
const monitorWeb = record(state.image.monitorWeb);
|
|
const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode");
|
|
const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode");
|
|
const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource");
|
|
const imageBuildProxy = state.spec.networkProfile.imageBuildProxy;
|
|
const imageBuildNoProxy = imageBuildProxy.noProxy.join(",");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
`image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`,
|
|
`image_build_package_mode=${shellQuote(imageBuildPackageMode)}`,
|
|
`image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`,
|
|
`image_build_proxy_source=${shellQuote(imageBuildProxySource)}`,
|
|
`image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`,
|
|
`image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`,
|
|
`image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`,
|
|
`image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`,
|
|
"meta_dir=/workspace/meta",
|
|
"event_log=/workspace/publish-events.log",
|
|
"build_log=/workspace/image-build.log",
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"cd /workspace/source",
|
|
"image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi",
|
|
"image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi",
|
|
"image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi",
|
|
"image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi",
|
|
"write_meta image_build_builder \"$image_build_builder\"",
|
|
"write_meta image_build_package_mode \"$image_build_package_mode\"",
|
|
"write_meta image_build_network_mode \"$image_build_network_mode\"",
|
|
"write_meta image_build_proxy_source \"$image_build_proxy_source\"",
|
|
"write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"",
|
|
"write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"",
|
|
"write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"",
|
|
"write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"",
|
|
"image_build_started_ms=$(now_ms)",
|
|
"write_meta image_build_started_ms \"$image_build_started_ms\"",
|
|
"emit_stage image-build running \"$image_build_started_ms\"",
|
|
"if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi",
|
|
"cat \"$build_log\"",
|
|
"image_build_finished_ms=$(now_ms)",
|
|
"write_meta image_build_finished_ms \"$image_build_finished_ms\"",
|
|
"metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)",
|
|
"digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)",
|
|
"test -n \"$digest\"",
|
|
"repo_no_tag=${image_ref%:*}",
|
|
"digest_ref=\"$repo_no_tag@$digest\"",
|
|
"write_meta digest \"$digest\"",
|
|
"write_meta digest_ref \"$digest_ref\"",
|
|
"emit_stage image-build succeeded \"$image_build_started_ms\"",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
|
|
const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
|
|
const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
|
|
path: file.path,
|
|
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
|
}))), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`job_name=${shellQuote(jobName)}`,
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
`image_repository=${shellQuote(state.image.repository)}`,
|
|
`gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`,
|
|
`gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`,
|
|
`files_b64=${shellQuote(filesB64)}`,
|
|
"meta_dir=/workspace/meta",
|
|
"event_log=/workspace/publish-events.log",
|
|
"build_log=/workspace/image-build.log",
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
"read_meta() { cat \"$meta_dir/$1\"; }",
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }",
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }",
|
|
"trap emit_failed EXIT",
|
|
"if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi",
|
|
"mkdir -p /root/.ssh",
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
"started_ms=$(read_meta started_ms)",
|
|
"source_commit=$(read_meta source_commit)",
|
|
"source_stage_ref=$(read_meta source_stage_ref)",
|
|
"mirror_commit=$(read_meta mirror_commit)",
|
|
"source_fetch_started_ms=$(read_meta source_fetch_started_ms)",
|
|
"source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)",
|
|
"monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)",
|
|
"monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)",
|
|
"image_build_started_ms=$(read_meta image_build_started_ms)",
|
|
"image_build_finished_ms=$(read_meta image_build_finished_ms)",
|
|
"digest_ref=$(read_meta digest_ref)",
|
|
"env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')",
|
|
"env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')",
|
|
"env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)",
|
|
"env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)",
|
|
"env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)",
|
|
"image_build_builder=$(read_meta image_build_builder)",
|
|
"image_build_package_mode=$(read_meta image_build_package_mode)",
|
|
"image_build_network_mode=$(read_meta image_build_network_mode)",
|
|
"image_build_proxy_source=$(read_meta image_build_proxy_source)",
|
|
"image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)",
|
|
"image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)",
|
|
"image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)",
|
|
"image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)",
|
|
"context_ignore_entries=$(read_meta context_ignore_entries)",
|
|
"image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)",
|
|
"image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)",
|
|
"image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')",
|
|
"gitops_commit=''",
|
|
"changed=false",
|
|
"file_count=0",
|
|
"gitops_started_ms=$(now_ms)",
|
|
"if [ \"$files_b64\" != \"W10=\" ]; then",
|
|
" emit_stage gitops running \"$gitops_started_ms\"",
|
|
" gitops_cache=\"/cache/${gitops_repository}.git\"",
|
|
" gitops_worktree=\"/tmp/$job_name/gitops\"",
|
|
" git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"",
|
|
" cd \"$gitops_worktree\"",
|
|
" git fetch origin \"$gitops_branch\" || true",
|
|
" if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"; else git checkout --orphan \"$gitops_branch\"; git rm -rf . >/dev/null 2>&1 || true; fi",
|
|
" FILES_B64=\"$files_b64\" IMAGE_REF=\"$image_ref\" DIGEST_REF=\"$digest_ref\" node <<'NODE'",
|
|
"const fs = require('node:fs');",
|
|
"const path = require('node:path');",
|
|
"const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
|
|
"for (const file of files) {",
|
|
" const target = path.resolve(process.cwd(), file.path);",
|
|
" if (!target.startsWith(process.cwd() + path.sep)) throw new Error(`refuse path outside workspace: ${file.path}`);",
|
|
" fs.mkdirSync(path.dirname(target), { recursive: true });",
|
|
" const text = Buffer.from(file.contentBase64, 'base64').toString('utf8').split(process.env.IMAGE_REF).join(process.env.DIGEST_REF);",
|
|
" fs.writeFileSync(target, text);",
|
|
"}",
|
|
"console.error(JSON.stringify({event:'web-probe-sentinel-gitops-files', fileCount: files.length, valuesRedacted:true}));",
|
|
"NODE",
|
|
" git add .",
|
|
" file_count=$(git diff --cached --name-only | wc -l | tr -d ' ')",
|
|
" if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi",
|
|
" git push origin \"HEAD:refs/heads/$gitops_branch\"",
|
|
" gitops_commit=$(git rev-parse HEAD)",
|
|
" emit_stage gitops succeeded \"$gitops_started_ms\"",
|
|
"else",
|
|
" emit_stage gitops skipped \"$gitops_started_ms\"",
|
|
"fi",
|
|
"gitops_finished_ms=$(now_ms)",
|
|
"finished_ms=$(now_ms)",
|
|
"node - \"$job_name\" \"$source_commit\" \"$source_stage_ref\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
|
|
"const [jobName, sourceCommit, sourceStageRef, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
|
|
"const elapsed = (start, finish) => Number(finish) - Number(start);",
|
|
"const cacheHits = Number(imageBuildCacheHits || 0);",
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, sourceStageRef, sourceAuthority:'git-mirror-snapshot', mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
|
|
"NODE",
|
|
"trap - EXIT",
|
|
].join("\n");
|
|
}
|
|
|
|
function sentinelGitopsFiles(state: SentinelCicdState): readonly { path: string; content: string }[] {
|
|
const runtimeManifests = state.manifests.filter((item) => item.kind !== "Application");
|
|
return [{
|
|
path: `${stringAt(state.cicd, "gitopsPath")}/web-probe-sentinel.yaml`,
|
|
content: `${runtimeManifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`,
|
|
}];
|
|
}
|
|
|
|
export function applySentinelArgoApplication(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
|
const app = state.manifests.find((item) => item.kind === "Application");
|
|
if (app === undefined) return { ok: false, reason: "application-manifest-missing", valuesRedacted: true };
|
|
const yaml = `${Bun.YAML.stringify(app).trim()}\n`;
|
|
const namespace = stringAt(state.cicd, "argo.namespace");
|
|
const applicationName = stringAt(state.cicd, "argo.applicationName");
|
|
const script = [
|
|
"set -eu",
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"kubectl apply -f \"$tmp\"",
|
|
`kubectl -n ${shellQuote(namespace)} annotate application ${shellQuote(applicationName)} argocd.argoproj.io/refresh=hard --overwrite`,
|
|
].join("\n");
|
|
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
|
return { ok: result.exitCode === 0, result: compactCommand(result), valuesRedacted: true };
|
|
}
|
|
|
|
export function createK8sJobScript(namespace: string, manifest: Record<string, unknown>): string {
|
|
const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
return [
|
|
"set -eu",
|
|
`kubectl -n ${shellQuote(namespace)} delete job ${shellQuote(stringAt(manifest, "metadata.name"))} --ignore-not-found=true >/dev/null 2>&1 || true`,
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"kubectl apply -f \"$tmp\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function createTektonPipelineRunScript(namespace: string, manifest: Record<string, unknown>): string {
|
|
const yaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
const pipelineRunName = stringAt(manifest, "metadata.name");
|
|
return [
|
|
"set -eu",
|
|
`pipeline_run=${shellQuote(pipelineRunName)}`,
|
|
`namespace=${shellQuote(namespace)}`,
|
|
"tmp=$(mktemp)",
|
|
`cat >"$tmp" <<'YAML'\n${yaml}YAML`,
|
|
"if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf '%s\\n' \"sentinel publish PipelineRun already exists; reusing $pipeline_run\"",
|
|
" exit 0",
|
|
"fi",
|
|
"if ! kubectl create -f \"$tmp\"; then",
|
|
" if kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" >/dev/null 2>&1; then",
|
|
" printf '%s\\n' \"sentinel publish PipelineRun appeared concurrently; reusing $pipeline_run\"",
|
|
" exit 0",
|
|
" fi",
|
|
" exit 1",
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function probeK8sJobScript(namespace: string, jobName: string): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
"succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)",
|
|
"failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)",
|
|
"active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)",
|
|
"pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)",
|
|
"pod_phase=''",
|
|
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
|
|
"logs_tail=''",
|
|
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
|
|
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
|
"const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
|
|
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string): string {
|
|
return [
|
|
"set +e",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`pipeline_run=${shellQuote(pipelineRunName)}`,
|
|
"condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].status}' 2>/dev/null || true)",
|
|
"condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].reason}' 2>/dev/null || true)",
|
|
"condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[?(@.type==\"Succeeded\")].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true)",
|
|
"if [ -z \"$condition_status\" ]; then condition_status=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].status}' 2>/dev/null || true); fi",
|
|
"if [ -z \"$condition_reason\" ]; then condition_reason=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].reason}' 2>/dev/null || true); fi",
|
|
"if [ -z \"$condition_message_b64\" ]; then condition_message_b64=$(kubectl -n \"$namespace\" get pipelinerun \"$pipeline_run\" -o jsonpath='{.status.conditions[0].message}' 2>/dev/null | head -c 1600 | base64 | tr -d '\\n' || true); fi",
|
|
"task_run=$(kubectl -n \"$namespace\" get taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
|
|
"pod=$(kubectl -n \"$namespace\" get pod -l tekton.dev/pipelineRun=\"$pipeline_run\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)",
|
|
"pod_phase=''",
|
|
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null || true); fi",
|
|
"logs_tail=''",
|
|
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; kubectl -n \"$namespace\" logs \"$pod\" -c step-publish --tail=100 2>/dev/null || true; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
|
|
"node - \"$condition_status\" \"$condition_reason\" \"$condition_message_b64\" \"$task_run\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
|
"const [conditionStatus, conditionReason, conditionMessageB64, taskRun, pod, podPhase, logsB64] = process.argv.slice(2);",
|
|
"const message = Buffer.from(conditionMessageB64 || '', 'base64').toString('utf8');",
|
|
"const active = conditionStatus === 'Unknown' || (!conditionStatus && (podPhase === 'Pending' || podPhase === 'Running'));",
|
|
"console.log(JSON.stringify({ succeeded: conditionStatus === 'True', failed: conditionStatus === 'False', active, conditionStatus: conditionStatus || null, conditionReason: conditionReason || null, conditionMessage: message || null, taskRun: taskRun || null, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
export function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
|
|
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
const line = lines[index];
|
|
if (!line.startsWith("{") || !line.endsWith("}")) continue;
|
|
const parsed = parseJsonObject(line);
|
|
if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed;
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult {
|
|
return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true };
|
|
}
|
|
|
|
function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record<string, unknown> {
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
const probe = record(result.probe);
|
|
const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : "";
|
|
const events = sentinelStageEventsFromLogs(logsTail, domain);
|
|
const envReuse = sentinelEnvReuseFromLogs(logsTail);
|
|
const completedStages = sentinelCompletedStages(events, record(result.payload));
|
|
const stageTimings = sentinelStageTimingSummary(events, record(result.payload), result.elapsedMs);
|
|
const currentPhase = sentinelCurrentRemotePhase(result, events, domain);
|
|
const isPipelineRun = result.resourceKind === "PipelineRun";
|
|
const commands = {
|
|
cliStatus: domain === "publish"
|
|
? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`
|
|
: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
logs: result.jobName === "-"
|
|
? "-"
|
|
: isPipelineRun
|
|
? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs -l tekton.dev/pipelineRun=${result.jobName} --all-containers=true --tail=120`
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`,
|
|
describe: result.jobName === "-"
|
|
? "-"
|
|
: isPipelineRun
|
|
? `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe pipelinerun/${result.jobName}`
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`,
|
|
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
|
gitMirrorSync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`,
|
|
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`,
|
|
controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
|
publishCurrent: `bun scripts/cli.ts web-probe sentinel publish-current --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`,
|
|
valuesRedacted: true,
|
|
};
|
|
return {
|
|
domain,
|
|
resourceKind: result.resourceKind ?? "Job",
|
|
pipelineRun: isPipelineRun ? result.jobName : null,
|
|
taskRun: probe.taskRun ?? null,
|
|
currentPhase,
|
|
completedStages,
|
|
stageTimings,
|
|
envReuse,
|
|
pod: probe.pod ?? null,
|
|
podPhase: probe.podPhase ?? null,
|
|
active: probe.active ?? null,
|
|
conditionStatus: probe.conditionStatus ?? null,
|
|
conditionReason: probe.conditionReason ?? null,
|
|
recentLogSummary: sentinelRecentLogSummary(logsTail),
|
|
commands,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function sentinelEnvReuseFromLogs(logsTail: string): Record<string, unknown> | null {
|
|
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
const parsed = parseJsonObject(lines[index]);
|
|
if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record<string, unknown>[] {
|
|
const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage";
|
|
return logsTail
|
|
.split(/\r?\n/u)
|
|
.map((line) => parseJsonObject(line.trim()))
|
|
.filter((item): item is Record<string, unknown> => item !== null && item.event === expectedEvent);
|
|
}
|
|
|
|
function sentinelCompletedStages(events: readonly Record<string, unknown>[], payload: Record<string, unknown>): string[] {
|
|
const completed = events
|
|
.filter((event) => event.status === "succeeded" || event.status === "skipped")
|
|
.map((event) => `${text(event.stage)}:${text(event.status)}`);
|
|
const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : [];
|
|
return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-");
|
|
}
|
|
|
|
function sentinelStageTimingSummary(events: readonly Record<string, unknown>[], payload: Record<string, unknown>, fallbackTotalMs: unknown): Record<string, unknown> {
|
|
const payloadTimings = record(payload.stageTimings);
|
|
const eventElapsed = (stage: string): number | null => {
|
|
const event = [...events].reverse().find((item) => item.stage === stage && (item.status === "succeeded" || item.status === "skipped" || item.status === "failed"));
|
|
return event === undefined ? null : finiteNumberOrNull(event.elapsedMs);
|
|
};
|
|
const sourceFetchMs = finiteNumberOrNull(payloadTimings.sourceFetchMs) ?? eventElapsed("source-fetch") ?? eventElapsed("source-mirror-fetch");
|
|
const monitorWebVerifyMs = finiteNumberOrNull(payloadTimings.monitorWebVerifyMs) ?? eventElapsed("monitor-web-verify");
|
|
const imageBuildMs = finiteNumberOrNull(payloadTimings.imageBuildMs) ?? eventElapsed("image-build");
|
|
const gitopsMs = finiteNumberOrNull(payloadTimings.gitopsMs) ?? eventElapsed("gitops");
|
|
const known = [sourceFetchMs, monitorWebVerifyMs, imageBuildMs, gitopsMs].filter((item): item is number => item !== null);
|
|
const summedTotalMs = known.length === 0 ? null : known.reduce((sum, item) => sum + item, 0);
|
|
const totalMs = finiteNumberOrNull(payloadTimings.totalMs)
|
|
?? finiteNumberOrNull(payload.elapsedMs)
|
|
?? finiteNumberOrNull(fallbackTotalMs)
|
|
?? summedTotalMs;
|
|
const result = {
|
|
sourceFetchMs,
|
|
monitorWebVerifyMs,
|
|
imageBuildMs,
|
|
gitopsMs,
|
|
totalMs,
|
|
valuesRedacted: true,
|
|
};
|
|
return Object.values(result).some((item) => item !== null && item !== true) ? result : {};
|
|
}
|
|
|
|
function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record<string, unknown>[], domain: "source-mirror" | "publish"): string {
|
|
if (result.phase === "job-succeeded" || result.phase === "pipelinerun-succeeded") return "completed";
|
|
if (result.phase === "create-job" || result.phase === "create-pipelinerun") return result.phase;
|
|
const reversed = [...events].reverse();
|
|
const failed = reversed.find((event) => event.status === "failed");
|
|
if (failed !== undefined) return text(failed.stage);
|
|
const running = reversed.find((event) => event.status === "running");
|
|
if (running !== undefined) return text(running.stage);
|
|
const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage)));
|
|
const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"];
|
|
const next = order.find((stage) => !completed.has(stage));
|
|
return next ?? result.phase;
|
|
}
|
|
|
|
function sentinelRecentLogSummary(logsTail: string): string {
|
|
const lines = logsTail
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter((line) => line.length > 0 && !line.startsWith("{"))
|
|
.slice(-5)
|
|
.map((line) => short(line));
|
|
return lines.length === 0 ? "-" : lines.join(" | ");
|
|
}
|
|
|
|
export function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] {
|
|
const remote = record(job);
|
|
if (remote.phase !== "job-timeout" && remote.phase !== "pipelinerun-timeout") return [];
|
|
const diagnostics = record(remote.diagnostics);
|
|
const commands = record(diagnostics.commands);
|
|
return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`];
|
|
}
|
|
|
|
export function publishSatisfiedByObservedWarnings(publish: unknown, flush: unknown, observedReady: boolean): string[] {
|
|
if (!observedReady) return [];
|
|
const warnings: string[] = [];
|
|
const publishRecord = record(publish);
|
|
if (Object.keys(publishRecord).length > 0 && publishRecord.ok !== true) {
|
|
warnings.push(`sentinel publish did not finish cleanly in the foreground (phase=${text(publishRecord.phase)}), but follow-up control-plane observation proves source, registry, GitOps, Argo and runtime are aligned; treating the publish wait result as visibility warning.`);
|
|
}
|
|
const flushRecord = record(flush);
|
|
if (Object.keys(flushRecord).length > 0 && flushRecord.ok !== true) {
|
|
warnings.push("sentinel git-mirror flush did not finish cleanly in the foreground, but runtime alignment is already proven; use git-mirror status/flush drill-down for GitHub mirror closeout.");
|
|
}
|
|
return warnings;
|
|
}
|
|
|
|
export function controlPlaneWaitWarningSeconds(state: SentinelCicdState): number {
|
|
return numberAt(state.cicd, "confirmWait.maxSeconds");
|
|
}
|
|
|
|
export function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeconds: number): string[] {
|
|
const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000;
|
|
if (elapsedMs === null || elapsedMs <= budgetMs) return [];
|
|
return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`];
|
|
}
|
|
|
|
export function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] {
|
|
const sync = record(sourceMirrorSync);
|
|
if (sync.ok === true || state.sourceHead.ok !== true) return [];
|
|
return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`];
|
|
}
|
|
|
|
export function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState, probe: unknown): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
phase: "already-present",
|
|
jobName: null,
|
|
probe: record(probe),
|
|
payload: {
|
|
ok: true,
|
|
status: "already-present",
|
|
sourceCommit: state.sourceHead.commit,
|
|
mirrorCommit: state.sourceHead.mirrorCommit ?? state.sourceHead.commit,
|
|
stageRef: state.sourceHead.stageRef,
|
|
sourceAuthority: state.sourceHead.sourceAuthority,
|
|
valuesRedacted: true,
|
|
},
|
|
polls: 0,
|
|
elapsedMs: 0,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|