Merge pull request #954 from pikasTech/chore/935-sentinel-dashboard-deploy-closeout
fix: gate sentinel publish on source mirror sync
This commit is contained in:
Lyon
@@ -177,10 +177,12 @@ function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSen
|
||||
if (!options.wait) return renderAsyncSentinelJob(state, "image", "build", options.timeoutSeconds);
|
||||
return runSentinelImageBuildConfirmed(state, options);
|
||||
}
|
||||
const sourceMirror = options.action === "status" ? probeSourceMirror(state, options.timeoutSeconds) : null;
|
||||
const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null;
|
||||
const sourceMirrorReady = options.action !== "status" || record(sourceMirror).ok === true;
|
||||
const registryReady = options.action !== "status" || record(registry?.probe).present === true;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && registryReady,
|
||||
ok: state.configReady && state.sourceHead.ok && sourceMirrorReady && registryReady,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
@@ -188,9 +190,12 @@ function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSen
|
||||
mutation: false,
|
||||
specRef: SPEC_REF,
|
||||
source: state.sourceHead,
|
||||
sourceMirror,
|
||||
image: state.image,
|
||||
registry,
|
||||
blocker: null,
|
||||
blocker: sourceMirrorReady
|
||||
? registryReady ? null : { code: "sentinel-image-missing", reason: "expected sentinel image tag is not present in the node-local registry" }
|
||||
: { code: "sentinel-source-mirror-not-ready", reason: "source.gitMirrorReadUrl does not expose the selected source commit yet" },
|
||||
next: {
|
||||
status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
||||
dryRun: `bun scripts/cli.ts web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane} --dry-run`,
|
||||
@@ -513,11 +518,15 @@ function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): R
|
||||
|
||||
function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
|
||||
const command = "web-probe sentinel image build";
|
||||
const publish = runSentinelPublishJob(state, false, options.timeoutSeconds);
|
||||
const sourceMirrorSync = runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
|
||||
const publish = sourceMirrorSync.ok === true
|
||||
? runSentinelPublishJob(state, false, options.timeoutSeconds)
|
||||
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
||||
const registry = probeImageRegistry(state, options.timeoutSeconds);
|
||||
const registryReady = record(registry.probe).present === true;
|
||||
const ok = state.configReady && state.sourceHead.ok && sourceMirrorSync.ok === true && publish.ok === true && registryReady;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && publish.ok === true && registryReady,
|
||||
ok,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
@@ -527,9 +536,19 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra
|
||||
source: state.sourceHead,
|
||||
image: state.image,
|
||||
registry,
|
||||
sourceMirrorSync,
|
||||
publish,
|
||||
warnings: sentinelElapsedWarnings(record(publish).elapsedMs),
|
||||
blocker: null,
|
||||
warnings: [
|
||||
...sentinelElapsedWarnings(record(sourceMirrorSync).elapsedMs),
|
||||
...sentinelElapsedWarnings(record(publish).elapsedMs),
|
||||
],
|
||||
blocker: ok
|
||||
? null
|
||||
: sourceMirrorSync.ok !== true
|
||||
? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" }
|
||||
: publish.ok !== true
|
||||
? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" }
|
||||
: { code: "sentinel-image-registry-missing", reason: "image publish completed but expected registry tag is not visible" },
|
||||
next: {
|
||||
status: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
||||
controlPlaneTrigger: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm`,
|
||||
@@ -542,7 +561,12 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra
|
||||
function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
||||
const command = `web-probe sentinel control-plane ${options.action}`;
|
||||
const applyOnly = options.action === "apply";
|
||||
const publish = applyOnly ? null : runSentinelPublishJob(state, true, options.timeoutSeconds);
|
||||
const sourceMirrorSync = applyOnly ? null : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds);
|
||||
const publish = applyOnly
|
||||
? null
|
||||
: record(sourceMirrorSync).ok === true
|
||||
? runSentinelPublishJob(state, true, options.timeoutSeconds)
|
||||
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
|
||||
const flush = !applyOnly && record(publish).ok === true
|
||||
? runChildCli(["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"], options.timeoutSeconds)
|
||||
: null;
|
||||
@@ -564,6 +588,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
const targetValidationOk = applyOnly || record(targetValidation).ok === true;
|
||||
const ok = state.configReady
|
||||
&& state.sourceHead.ok
|
||||
&& (applyOnly || record(sourceMirrorSync).ok === true)
|
||||
&& (applyOnly || record(publish).ok === true)
|
||||
&& (applyOnly || record(flush).ok === true)
|
||||
&& record(publicExposureApply).ok === true
|
||||
@@ -571,9 +596,13 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
&& observedReady
|
||||
&& targetValidationOk;
|
||||
const blocker = ok ? null : {
|
||||
code: targetValidationOk ? "sentinel-control-plane-not-ready" : "sentinel-target-validation-failed",
|
||||
code: targetValidationOk
|
||||
? record(sourceMirrorSync).ok === false ? "sentinel-source-mirror-sync-failed" : "sentinel-control-plane-not-ready"
|
||||
: "sentinel-target-validation-failed",
|
||||
reason: targetValidationOk
|
||||
? "one or more publish, publicExposure, Argo or runtime observation checks did not pass"
|
||||
? record(sourceMirrorSync).ok === false
|
||||
? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish"
|
||||
: "one or more publish, publicExposure, Argo or runtime observation checks did not pass"
|
||||
: text(record(targetValidation).failure ?? record(targetValidation).reason ?? "quick verify did not pass"),
|
||||
};
|
||||
const result = {
|
||||
@@ -607,6 +636,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
objects: manifestObjectSummary(state.manifests),
|
||||
sha256: state.manifestSha256,
|
||||
},
|
||||
sourceMirrorSync,
|
||||
publish,
|
||||
flush,
|
||||
publicExposureApply,
|
||||
@@ -614,6 +644,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
|
||||
observed,
|
||||
targetValidation,
|
||||
warnings: Array.from(new Set([
|
||||
...sentinelElapsedWarnings(record(sourceMirrorSync).elapsedMs),
|
||||
...sentinelElapsedWarnings(record(publish).elapsedMs),
|
||||
...sentinelElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs),
|
||||
...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []),
|
||||
@@ -688,21 +719,6 @@ function sentinelObservedReady(value: Record<string, unknown> | SentinelObserved
|
||||
}
|
||||
|
||||
function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
const sourceMode = stringAt(state.cicd, "builder.sourceMode");
|
||||
if (sourceMode === "sparse-git-checkout") {
|
||||
return {
|
||||
ok: state.sourceHead.ok,
|
||||
probe: {
|
||||
mode: sourceMode,
|
||||
commit: state.sourceHead.commit,
|
||||
expectedCommit: state.sourceHead.commit,
|
||||
persistentMirrorPresent: false,
|
||||
source: "commit-pinned sparse checkout declared in config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml#sentinel.cicd.source.checkoutPaths",
|
||||
valuesRedacted: true,
|
||||
},
|
||||
result: { exitCode: 0, timedOut: false, stdoutBytes: 0, stderrBytes: 0, stdoutPreview: "sourceMode=sparse-git-checkout", stderrPreview: "" },
|
||||
};
|
||||
}
|
||||
const namespace = stringAt(state.cicd, "builder.namespace");
|
||||
const repository = stringAt(state.cicd, "source.repository");
|
||||
const branch = stringAt(state.cicd, "source.branch");
|
||||
@@ -717,10 +733,10 @@ function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Re
|
||||
"node - \"$rc\" \"$commit\" \"$expected\" \"$repo_path\" \"$branch\" <<'NODE'",
|
||||
"const [rc, commit, expected, repoPath, branch] = process.argv.slice(2);",
|
||||
"const present = Number(rc) === 0 && /^[0-9a-f]{40}$/i.test(commit || '');",
|
||||
"console.log(JSON.stringify({ ok: present && (!expected || commit === expected), present, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, valuesRedacted: true }));",
|
||||
"console.log(JSON.stringify({ ok: present && (!expected || commit === expected), mode: 'internal-git-mirror', present, commit: present ? commit : null, expectedCommit: expected || null, branch, repoPath, persistentMirrorPresent: present, readUrl: process.env.SOURCE_GIT_MIRROR_READ_URL || null, valuesRedacted: true }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", `export SOURCE_GIT_MIRROR_READ_URL=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}\n${script}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
return { ok: result.exitCode === 0 && parseJsonObject(result.stdout)?.ok === true, probe: parseJsonObject(result.stdout), result: compactCommand(result) };
|
||||
}
|
||||
|
||||
@@ -835,6 +851,244 @@ function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Re
|
||||
return `${state.image.repository}@${digest}`;
|
||||
}
|
||||
|
||||
function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds: number): SentinelRemoteJobResult {
|
||||
const prefix = `${stringAt(state.cicd, "builder.jobPrefix")}-source-sync`;
|
||||
const jobName = `${prefix}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
||||
const manifest = sentinelSourceMirrorSyncJobManifest(state, jobName);
|
||||
const namespace = stringAt(state.cicd, "builder.namespace");
|
||||
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "submitting", jobName, sourceCommit: state.sourceHead.commit, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
if (created.exitCode !== 0) {
|
||||
sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
return { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true };
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = Math.max(30_000, Math.min(timeoutSeconds * 1000, 900_000));
|
||||
let polls = 0;
|
||||
let lastProbe: Record<string, unknown> = {};
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
polls += 1;
|
||||
const probeCapture = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const probe = parseJsonObject(probeCapture.stdout) ?? {};
|
||||
lastProbe = { ...probe, capture: compactCommand(probeCapture) };
|
||||
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
||||
sentinelProgressEvent("sentinel.source-mirror.progress", {
|
||||
phase: "remote-job",
|
||||
status: probe.succeeded === true ? "succeeded" : probe.failed === true ? "failed" : "running",
|
||||
jobName,
|
||||
polls,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
pod: probe.pod ?? null,
|
||||
sourceCommit: state.sourceHead.commit,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
});
|
||||
if (probe.succeeded === true) {
|
||||
const ok = payload.ok === true;
|
||||
return { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
||||
}
|
||||
if (probe.failed === true) {
|
||||
return { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
||||
}
|
||||
if (Date.now() - startedAt > 120_000) sentinelProgressEvent("sentinel.source-mirror.warning", { warning: "source mirror sync exceeded 120s; investigate env-reuse/git mirror/source build path", jobName, elapsedMs: Date.now() - startedAt, node: state.spec.nodeId, lane: state.spec.lane });
|
||||
runCommand(["sleep", "5"], repoRoot, { timeoutMs: 6_000 });
|
||||
}
|
||||
return { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult {
|
||||
return {
|
||||
ok: false,
|
||||
phase,
|
||||
jobName: "-",
|
||||
payload: { ok: false, status: phase, reason, valuesRedacted: true },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelSourceMirrorSyncJobManifest(state: SentinelCicdState, jobName: string): Record<string, unknown> {
|
||||
const namespace = stringAt(state.cicd, "builder.namespace");
|
||||
const labels = {
|
||||
"app.kubernetes.io/name": "web-probe-sentinel-source-mirror",
|
||||
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
||||
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
||||
"unidesk.ai/node": state.spec.nodeId,
|
||||
"unidesk.ai/lane": state.spec.lane,
|
||||
};
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: { name: jobName, namespace, labels },
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: numberAt(state.cicd, "builder.activeDeadlineSeconds"),
|
||||
ttlSecondsAfterFinished: numberAt(state.cicd, "builder.ttlSecondsAfterFinished"),
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
restartPolicy: "Never",
|
||||
volumes: [
|
||||
{ name: "cache", hostPath: { path: stringAt(state.controlPlaneTarget, "gitMirror.cacheHostPath"), type: "DirectoryOrCreate" } },
|
||||
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
||||
],
|
||||
containers: [{
|
||||
name: "sync",
|
||||
image: state.image.baseImage,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/bin/sh", "-ec", sentinelSourceMirrorSyncShell(state, jobName)],
|
||||
volumeMounts: [
|
||||
{ name: "cache", mountPath: "/cache" },
|
||||
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
||||
],
|
||||
}],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string): string {
|
||||
return [
|
||||
"set -eu",
|
||||
`job_name=${shellQuote(jobName)}`,
|
||||
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
|
||||
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
|
||||
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`,
|
||||
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
||||
"started_ms=$(node -e 'console.log(Date.now())')",
|
||||
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
|
||||
"trap emit_failed EXIT",
|
||||
"test -n \"$source_commit\"",
|
||||
...sentinelSourceMirrorSshSetupShellLines(state),
|
||||
"repo=\"/cache/${source_repository}.git\"",
|
||||
"mkdir -p \"$(dirname \"$repo\")\"",
|
||||
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
|
||||
" git --git-dir=\"$repo\" remote set-url origin \"$source_git_url\" || git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
||||
"else",
|
||||
" rm -rf \"$repo\"",
|
||||
" git init --bare \"$repo\"",
|
||||
" git --git-dir=\"$repo\" remote add origin \"$source_git_url\"",
|
||||
"fi",
|
||||
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
|
||||
"git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
|
||||
"git --git-dir=\"$repo\" config http.uploadpack true",
|
||||
"git --git-dir=\"$repo\" config http.receivepack true",
|
||||
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"",
|
||||
"mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")",
|
||||
"test \"$mirror_commit\" = \"$source_commit\"",
|
||||
"git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"",
|
||||
"git --git-dir=\"$repo\" update-server-info",
|
||||
"finished_ms=$(node -e 'console.log(Date.now())')",
|
||||
"node - \"$job_name\" \"$source_repository\" \"$source_branch\" \"$source_commit\" \"$mirror_commit\" \"$started_ms\" \"$finished_ms\" <<'NODE'",
|
||||
"const [jobName, repository, branch, sourceCommit, mirrorCommit, startedMs, finishedMs] = process.argv.slice(2);",
|
||||
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, repository, branch, sourceCommit, mirrorCommit, elapsedMs:Number(finishedMs)-Number(startedMs), valuesRedacted:true }));",
|
||||
"NODE",
|
||||
"trap - EXIT",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): string[] {
|
||||
const proxy = record(valueAtPath(state.controlPlaneNode, "egressProxy"));
|
||||
const serviceName = nonEmptyString(proxy.serviceName);
|
||||
const namespace = nonEmptyString(proxy.namespace);
|
||||
const port = typeof proxy.port === "number" && Number.isFinite(proxy.port) ? proxy.port : null;
|
||||
const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.filter((item): item is string => typeof item === "string" && item.length > 0).join(",") : "";
|
||||
const useProxy = serviceName !== null && namespace !== null && port !== null;
|
||||
if (!useProxy) {
|
||||
return [
|
||||
"mkdir -p /root/.ssh",
|
||||
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
||||
"chmod 0400 /root/.ssh/id_rsa",
|
||||
"printf '%s\\n' 'sentinel source-mirror-egress-proxy mode=direct transport=ssh source=yaml' >&2",
|
||||
"unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy",
|
||||
"export NO_PROXY='*'",
|
||||
"export no_proxy='*'",
|
||||
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
||||
"#!/bin/sh",
|
||||
"exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 \"$@\"",
|
||||
"SH_PROXY",
|
||||
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
||||
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
||||
"unset GIT_SSH_COMMAND",
|
||||
];
|
||||
}
|
||||
const proxyHost = `${serviceName}.${namespace}.svc.cluster.local`;
|
||||
const proxyUrl = `http://${proxyHost}:${port}`;
|
||||
const proxyCommand = `ProxyCommand=node /tmp/sentinel-github-proxy-connect.cjs ${proxyHost} ${port} %h %p`;
|
||||
return [
|
||||
"mkdir -p /root/.ssh",
|
||||
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
||||
"chmod 0400 /root/.ssh/id_rsa",
|
||||
`printf '%s\\n' ${shellQuote(`sentinel source-mirror-egress-proxy host=${proxyHost} port=${port} transport=ssh ssh=GIT_SSH-wrapper source=yaml`)} >&2`,
|
||||
`export HTTP_PROXY=${shellQuote(proxyUrl)}`,
|
||||
`export HTTPS_PROXY=${shellQuote(proxyUrl)}`,
|
||||
`export ALL_PROXY=${shellQuote(proxyUrl)}`,
|
||||
`export http_proxy=${shellQuote(proxyUrl)}`,
|
||||
`export https_proxy=${shellQuote(proxyUrl)}`,
|
||||
`export all_proxy=${shellQuote(proxyUrl)}`,
|
||||
`export NO_PROXY=${shellQuote(noProxy)}`,
|
||||
`export no_proxy=${shellQuote(noProxy)}`,
|
||||
"cat > /tmp/sentinel-github-proxy-connect.cjs <<'NODE_PROXY'",
|
||||
"#!/usr/bin/env node",
|
||||
"const net = require('node:net');",
|
||||
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
||||
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
||||
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
||||
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
|
||||
" console.error('sentinel source-mirror proxy-connect: invalid ProxyCommand arguments');",
|
||||
" process.exit(64);",
|
||||
"}",
|
||||
"let settled = false;",
|
||||
"let tunnelEstablished = false;",
|
||||
"function finish(code, message) {",
|
||||
" if (settled) return;",
|
||||
" settled = true;",
|
||||
" if (message) console.error('sentinel source-mirror proxy-connect: ' + message);",
|
||||
" process.exit(code);",
|
||||
"}",
|
||||
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
||||
"let buffer = Buffer.alloc(0);",
|
||||
"socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
|
||||
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
|
||||
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
|
||||
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
|
||||
"function onData(chunk) {",
|
||||
" buffer = Buffer.concat([buffer, chunk]);",
|
||||
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
||||
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
||||
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
|
||||
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
|
||||
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
|
||||
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
||||
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
|
||||
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
|
||||
" socket.destroy();",
|
||||
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
|
||||
" return;",
|
||||
" }",
|
||||
" socket.off('data', onData);",
|
||||
" socket.setTimeout(0);",
|
||||
" tunnelEstablished = true;",
|
||||
" const rest = buffer.slice(headerEnd + 4);",
|
||||
" if (rest.length) process.stdout.write(rest);",
|
||||
" process.stdin.on('error', () => {});",
|
||||
" process.stdout.on('error', () => {});",
|
||||
" process.stdin.pipe(socket);",
|
||||
" socket.pipe(process.stdout);",
|
||||
"}",
|
||||
"socket.on('data', onData);",
|
||||
"NODE_PROXY",
|
||||
"chmod 0700 /tmp/sentinel-github-proxy-connect.cjs",
|
||||
"cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'",
|
||||
"#!/bin/sh",
|
||||
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shellQuote(proxyCommand)} "$@"`,
|
||||
"SH_PROXY",
|
||||
"chmod 0700 /tmp/sentinel-git-ssh-proxy.sh",
|
||||
"export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",
|
||||
"unset GIT_SSH_COMMAND",
|
||||
];
|
||||
}
|
||||
|
||||
function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, timeoutSeconds: number): SentinelRemoteJobResult {
|
||||
const jobName = `${stringAt(state.cicd, "builder.jobPrefix")}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 63);
|
||||
const manifest = sentinelPublishJobManifest(state, jobName, publishGitops);
|
||||
@@ -940,7 +1194,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
||||
`job_name=${shellQuote(jobName)}`,
|
||||
`source_repository=${shellQuote(stringAt(state.cicd, "source.repository"))}`,
|
||||
`source_branch=${shellQuote(stringAt(state.cicd, "source.branch"))}`,
|
||||
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitSshUrl"))}`,
|
||||
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
|
||||
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
||||
`checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
|
||||
`image_ref=${shellQuote(state.image.ref)}`,
|
||||
@@ -2324,6 +2578,8 @@ function sentinelPipelineRunName(state: SentinelCicdState): string {
|
||||
|
||||
function renderImageResult(result: Record<string, unknown>): string {
|
||||
const source = record(result.source);
|
||||
const sourceMirror = record(result.sourceMirror);
|
||||
const sourceMirrorSync = record(result.sourceMirrorSync);
|
||||
const image = record(result.image);
|
||||
const registry = record(result.registry);
|
||||
const publish = record(result.publish);
|
||||
@@ -2337,10 +2593,14 @@ function renderImageResult(result: Record<string, unknown>): string {
|
||||
"",
|
||||
table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]),
|
||||
"",
|
||||
Object.keys(sourceMirror).length === 0 ? "SOURCE_MIRROR\n-" : table(["OK", "MODE", "COMMIT", "EXPECTED", "READ_URL"], [[sourceMirror.ok, record(sourceMirror.probe).mode, short(record(sourceMirror.probe).commit), short(record(sourceMirror.probe).expectedCommit), record(sourceMirror.probe).readUrl ?? "-"]]),
|
||||
"",
|
||||
table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]),
|
||||
"",
|
||||
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
|
||||
"",
|
||||
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
|
||||
"",
|
||||
Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]),
|
||||
"",
|
||||
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
|
||||
@@ -2366,6 +2626,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
const argo = record(result.argo);
|
||||
const validation = record(result.validation);
|
||||
const observed = record(result.observed);
|
||||
const sourceMirrorSync = record(result.sourceMirrorSync);
|
||||
const publish = record(result.publish);
|
||||
const flush = record(result.flush);
|
||||
const publicExposureApply = record(result.publicExposureApply);
|
||||
@@ -2388,6 +2649,8 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
"",
|
||||
renderObservedStatus(observed),
|
||||
"",
|
||||
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
|
||||
"",
|
||||
Object.keys(targetValidation).length === 0 ? "TARGET_VALIDATION\n-" : table(["OK", "STATUS", "SCENARIO", "RUN", "OBSERVER", "REPORT", "FINDINGS", "ARTIFACTS"], [[
|
||||
targetValidation.ok,
|
||||
targetValidation.status,
|
||||
|
||||
Reference in New Issue
Block a user