|
|
|
@@ -370,7 +370,7 @@ function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unkn
|
|
|
|
|
const baseImageRef = stringAt(cicd, "image.baseImageRef");
|
|
|
|
|
const baseImage = stringTarget(readWebProbeSentinelConfigRefTarget(spec, baseImageRef), baseImageRef);
|
|
|
|
|
const entrypoint = stringAt(cicd, "source.entrypoint");
|
|
|
|
|
const monitorWeb = monitorWebCicdPlan(cicd);
|
|
|
|
|
const monitorWeb = monitorWebCicdPlan(spec, cicd);
|
|
|
|
|
const dockerfile = sentinelDockerfile(baseImage, entrypoint);
|
|
|
|
|
return {
|
|
|
|
|
repository,
|
|
|
|
@@ -398,7 +398,7 @@ function sentinelDockerfile(baseImage: string, entrypoint: string): string {
|
|
|
|
|
].join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function monitorWebCicdPlan(cicd: Record<string, unknown>): Record<string, unknown> {
|
|
|
|
|
function monitorWebCicdPlan(spec: HwlabRuntimeLaneSpec, cicd: Record<string, unknown>): Record<string, unknown> {
|
|
|
|
|
return {
|
|
|
|
|
stack: stringAtNullable(cicd, "monitorWeb.frontendStack") ?? "vue3-vendored-browser-build",
|
|
|
|
|
runtimeMode: stringAtNullable(cicd, "monitorWeb.runtimeMode") ?? "runner-served-bridge",
|
|
|
|
@@ -406,21 +406,22 @@ function monitorWebCicdPlan(cicd: Record<string, unknown>): Record<string, unkno
|
|
|
|
|
verifyCommand: "bun scripts/verify-web-probe-sentinel-monitor-web.ts",
|
|
|
|
|
gitMirrorReadUrl: stringAt(cicd, "source.gitMirrorReadUrl"),
|
|
|
|
|
sourceMode: stringAt(cicd, "builder.sourceMode"),
|
|
|
|
|
envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "docker-layer-and-ci-node-deps",
|
|
|
|
|
envReuseMode: stringAtNullable(cicd, "monitorWeb.envReuse.mode") ?? "k8s-buildkit-and-ci-node-deps",
|
|
|
|
|
envReuseNodeDepsPath: stringAtNullable(cicd, "monitorWeb.envReuse.nodeDepsPath") ?? "/opt/hwlab-ci-node-deps/node_modules",
|
|
|
|
|
verifyPhase: stringAtNullable(cicd, "monitorWeb.dockerBuild.verifyPhase") ?? "pre-docker-build",
|
|
|
|
|
dockerBuildPackageMode: stringAtNullable(cicd, "monitorWeb.dockerBuild.packageMode") ?? "copy-only-dockerfile",
|
|
|
|
|
dockerBuildNetworkMode: monitorWebDockerBuildNetworkMode(cicd),
|
|
|
|
|
dockerBuildProxySource: stringAtNullable(cicd, "monitorWeb.dockerBuild.proxySource") ?? "node.networkProfile.dockerBuildProxy",
|
|
|
|
|
dockerBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.dockerBuild.contextIgnore") ?? "generated",
|
|
|
|
|
verifyPhase: stringAtNullable(cicd, "monitorWeb.imageBuild.verifyPhase") ?? "pre-image-build",
|
|
|
|
|
imageBuildBuilder: spec.buildkit?.sidecarImage ?? null,
|
|
|
|
|
imageBuildPackageMode: stringAtNullable(cicd, "monitorWeb.imageBuild.packageMode") ?? "copy-only-containerfile",
|
|
|
|
|
imageBuildNetworkMode: monitorWebImageBuildNetworkMode(cicd),
|
|
|
|
|
imageBuildProxySource: stringAtNullable(cicd, "monitorWeb.imageBuild.proxySource") ?? "node.networkProfile.imageBuildProxy",
|
|
|
|
|
imageBuildContextIgnore: stringAtNullable(cicd, "monitorWeb.imageBuild.contextIgnore") ?? "generated",
|
|
|
|
|
ciBudgetSeconds: numberAtNullable(cicd, "monitorWeb.ciBudget.maxSeconds") ?? numberAt(cicd, "confirmWait.maxSeconds"),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function monitorWebDockerBuildNetworkMode(cicd: Record<string, unknown>): "default" | "host" {
|
|
|
|
|
const value = stringAtNullable(cicd, "monitorWeb.dockerBuild.networkMode") ?? "host";
|
|
|
|
|
if (value !== "default" && value !== "host") throw new Error(`monitorWeb.dockerBuild.networkMode must be default or host, got ${value}`);
|
|
|
|
|
function monitorWebImageBuildNetworkMode(cicd: Record<string, unknown>): "default" | "host" {
|
|
|
|
|
const value = stringAtNullable(cicd, "monitorWeb.imageBuild.networkMode") ?? "host";
|
|
|
|
|
if (value !== "default" && value !== "host") throw new Error(`monitorWeb.imageBuild.networkMode must be default or host, got ${value}`);
|
|
|
|
|
return value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -1613,6 +1614,8 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean,
|
|
|
|
|
|
|
|
|
|
function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, publishGitops: boolean): Record<string, unknown> {
|
|
|
|
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
|
|
|
|
const buildkitImage = requireSentinelBuildkitImage(state);
|
|
|
|
|
const proxyEnv = sentinelImageBuildProxyEnv(state);
|
|
|
|
|
const labels = {
|
|
|
|
|
"app.kubernetes.io/name": "web-probe-sentinel-publish",
|
|
|
|
|
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
|
|
|
@@ -1632,20 +1635,55 @@ function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, p
|
|
|
|
|
metadata: { labels },
|
|
|
|
|
spec: {
|
|
|
|
|
restartPolicy: "Never",
|
|
|
|
|
hostNetwork: true,
|
|
|
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
|
|
|
securityContext: { fsGroup: 1000 },
|
|
|
|
|
volumes: [
|
|
|
|
|
sentinelGitMirrorCacheVolume(state),
|
|
|
|
|
{ name: "git-ssh", secret: { secretName: stringAt(state.cicd, "builder.gitSshSecretName"), defaultMode: 256 } },
|
|
|
|
|
{ name: "docker-sock", hostPath: { path: stringAt(state.cicd, "builder.dockerSocketPath"), type: "Socket" } },
|
|
|
|
|
{ name: "workspace", emptyDir: { sizeLimit: "8Gi" } },
|
|
|
|
|
{ name: "buildkit-state", emptyDir: { sizeLimit: "8Gi" } },
|
|
|
|
|
{ name: "tmp", emptyDir: {} },
|
|
|
|
|
],
|
|
|
|
|
initContainers: [
|
|
|
|
|
{
|
|
|
|
|
name: "source",
|
|
|
|
|
image: state.image.baseImage,
|
|
|
|
|
imagePullPolicy: "IfNotPresent",
|
|
|
|
|
env: proxyEnv,
|
|
|
|
|
command: ["/bin/sh", "-ec", sentinelPublishSourceShell(state, jobName)],
|
|
|
|
|
volumeMounts: [
|
|
|
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
|
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "image-build",
|
|
|
|
|
image: buildkitImage,
|
|
|
|
|
imagePullPolicy: "IfNotPresent",
|
|
|
|
|
env: [
|
|
|
|
|
...proxyEnv,
|
|
|
|
|
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
|
|
|
|
],
|
|
|
|
|
command: ["/bin/sh", "-ec", sentinelPublishImageBuildShell(state, jobName)],
|
|
|
|
|
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
|
|
|
|
volumeMounts: [
|
|
|
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
|
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
|
|
|
{ name: "tmp", mountPath: "/tmp" },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
],
|
|
|
|
|
containers: [{
|
|
|
|
|
name: "publish",
|
|
|
|
|
image: state.image.baseImage,
|
|
|
|
|
imagePullPolicy: "IfNotPresent",
|
|
|
|
|
env: proxyEnv,
|
|
|
|
|
command: ["/bin/sh", "-ec", sentinelPublishShell(state, jobName, publishGitops)],
|
|
|
|
|
volumeMounts: [
|
|
|
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
|
|
|
{ name: "cache", mountPath: "/cache" },
|
|
|
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
|
|
|
{ name: "docker-sock", mountPath: stringAt(state.cicd, "builder.dockerSocketPath") },
|
|
|
|
|
],
|
|
|
|
|
}],
|
|
|
|
|
},
|
|
|
|
@@ -1660,13 +1698,31 @@ function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record<string,
|
|
|
|
|
return { name: "cache", persistentVolumeClaim: { claimName: stringAt(state.controlPlaneTarget, "gitMirror.cachePvcName") } };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
|
|
|
|
|
const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
|
|
|
|
|
function requireSentinelBuildkitImage(state: SentinelCicdState): string {
|
|
|
|
|
const image = state.spec.buildkit?.sidecarImage;
|
|
|
|
|
if (typeof image !== "string" || image.length === 0) {
|
|
|
|
|
throw new Error(`config/hwlab-node-lanes.yaml ${state.spec.nodeId}/${state.spec.lane} buildkit.sidecarImage is required for pure k8s web-probe sentinel image publish`);
|
|
|
|
|
}
|
|
|
|
|
return image;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> {
|
|
|
|
|
const proxy = state.spec.networkProfile.imageBuildProxy;
|
|
|
|
|
const noProxy = proxy.noProxy.join(",");
|
|
|
|
|
return [
|
|
|
|
|
{ name: "HTTP_PROXY", value: proxy.http },
|
|
|
|
|
{ name: "http_proxy", value: proxy.http },
|
|
|
|
|
{ name: "HTTPS_PROXY", value: proxy.https },
|
|
|
|
|
{ name: "https_proxy", value: proxy.https },
|
|
|
|
|
{ name: "ALL_PROXY", value: proxy.all },
|
|
|
|
|
{ name: "all_proxy", value: proxy.all },
|
|
|
|
|
{ name: "NO_PROXY", value: noProxy },
|
|
|
|
|
{ name: "no_proxy", value: noProxy },
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string {
|
|
|
|
|
const monitorWeb = record(state.image.monitorWeb);
|
|
|
|
|
const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
|
|
|
|
|
path: file.path,
|
|
|
|
|
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
|
|
|
|
}))), "utf8").toString("base64");
|
|
|
|
|
const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => {
|
|
|
|
|
if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths");
|
|
|
|
|
return item;
|
|
|
|
@@ -1674,11 +1730,6 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64");
|
|
|
|
|
const envReuseMode = stringAt(monitorWeb, "envReuseMode");
|
|
|
|
|
const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath");
|
|
|
|
|
const dockerBuildPackageMode = stringAt(monitorWeb, "dockerBuildPackageMode");
|
|
|
|
|
const dockerBuildNetworkMode = stringAt(monitorWeb, "dockerBuildNetworkMode");
|
|
|
|
|
const dockerBuildProxySource = stringAt(monitorWeb, "dockerBuildProxySource");
|
|
|
|
|
const dockerBuildProxy = state.spec.networkProfile.dockerBuildProxy;
|
|
|
|
|
const dockerBuildNoProxy = dockerBuildProxy.noProxy.join(",");
|
|
|
|
|
return [
|
|
|
|
|
"set -eu",
|
|
|
|
|
`job_name=${shellQuote(jobName)}`,
|
|
|
|
@@ -1687,32 +1738,27 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
`source_git_url=${shellQuote(stringAt(state.cicd, "source.gitMirrorReadUrl"))}`,
|
|
|
|
|
`source_commit=${shellQuote(state.sourceHead.commit ?? "")}`,
|
|
|
|
|
`checkout_paths_b64=${shellQuote(checkoutPathsB64)}`,
|
|
|
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
|
|
|
`image_repository=${shellQuote(state.image.repository)}`,
|
|
|
|
|
`dockerfile_b64=${shellQuote(dockerfileB64)}`,
|
|
|
|
|
`gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`,
|
|
|
|
|
`gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`,
|
|
|
|
|
`files_b64=${shellQuote(filesB64)}`,
|
|
|
|
|
`env_reuse_mode=${shellQuote(envReuseMode)}`,
|
|
|
|
|
`env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`,
|
|
|
|
|
`docker_build_package_mode=${shellQuote(dockerBuildPackageMode)}`,
|
|
|
|
|
`docker_build_network_mode=${shellQuote(dockerBuildNetworkMode)}`,
|
|
|
|
|
`docker_build_proxy_source=${shellQuote(dockerBuildProxySource)}`,
|
|
|
|
|
`docker_build_http_proxy=${shellQuote(dockerBuildProxy.http)}`,
|
|
|
|
|
`docker_build_https_proxy=${shellQuote(dockerBuildProxy.https)}`,
|
|
|
|
|
`docker_build_all_proxy=${shellQuote(dockerBuildProxy.all)}`,
|
|
|
|
|
`docker_build_no_proxy=${shellQuote(dockerBuildNoProxy)}`,
|
|
|
|
|
"started_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(node -e 'console.log(Date.now())'); node - \"$stage\" \"$status\" \"$started\" \"$finished\" <<'NODE'\nconst [stage, status, started, finished] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-stage', stage, status, elapsedMs:Number(finished)-Number(started), valuesRedacted:true }));\nNODE\n}",
|
|
|
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }",
|
|
|
|
|
"meta_dir=/workspace/meta",
|
|
|
|
|
"event_log=/workspace/publish-events.log",
|
|
|
|
|
"worktree=/workspace/source",
|
|
|
|
|
"rm -rf \"$worktree\" \"$meta_dir\" /workspace/image-build.log /workspace/build-metadata.json",
|
|
|
|
|
"mkdir -p \"$meta_dir\"",
|
|
|
|
|
": > \"$event_log\"",
|
|
|
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
|
|
|
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
|
|
|
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
|
|
|
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
|
|
|
|
|
"trap emit_failed EXIT",
|
|
|
|
|
"started_ms=$(now_ms)",
|
|
|
|
|
"write_meta started_ms \"$started_ms\"",
|
|
|
|
|
"write_meta source_commit \"$source_commit\"",
|
|
|
|
|
"mkdir -p /root/.ssh",
|
|
|
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
|
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
|
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
|
|
|
"worktree=\"/tmp/$job_name/source\"",
|
|
|
|
|
"rm -rf \"/tmp/$job_name\"",
|
|
|
|
|
"mkdir -p \"/tmp/$job_name\"",
|
|
|
|
|
"git init \"$worktree\"",
|
|
|
|
|
"cd \"$worktree\"",
|
|
|
|
|
"git remote add origin \"$source_git_url\"",
|
|
|
|
@@ -1725,13 +1771,16 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
"fs.mkdirSync('.git/info', { recursive: true });",
|
|
|
|
|
"fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');",
|
|
|
|
|
"NODE",
|
|
|
|
|
"source_fetch_started_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"source_fetch_started_ms=$(now_ms)",
|
|
|
|
|
"write_meta source_fetch_started_ms \"$source_fetch_started_ms\"",
|
|
|
|
|
"emit_stage source-fetch running \"$source_fetch_started_ms\"",
|
|
|
|
|
"git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"",
|
|
|
|
|
"git checkout --detach \"$source_commit\"",
|
|
|
|
|
"mirror_commit=$(git rev-parse HEAD)",
|
|
|
|
|
"test \"$mirror_commit\" = \"$source_commit\"",
|
|
|
|
|
"source_fetch_finished_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"write_meta mirror_commit \"$mirror_commit\"",
|
|
|
|
|
"source_fetch_finished_ms=$(now_ms)",
|
|
|
|
|
"write_meta source_fetch_finished_ms \"$source_fetch_finished_ms\"",
|
|
|
|
|
"emit_stage source-fetch succeeded \"$source_fetch_started_ms\"",
|
|
|
|
|
"env_reuse_node_deps_present=false",
|
|
|
|
|
"env_reuse_node_deps_entries=0",
|
|
|
|
@@ -1739,14 +1788,24 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
"env_reuse_linked_node_deps=0",
|
|
|
|
|
"rm -rf node_modules",
|
|
|
|
|
"if [ \"$env_reuse_node_deps_present\" = true ]; then mkdir -p node_modules; for dep in \"$env_reuse_node_deps_path\"/*; do [ -e \"$dep\" ] || continue; ln -sf \"$dep\" \"node_modules/$(basename \"$dep\")\"; env_reuse_linked_node_deps=$((env_reuse_linked_node_deps + 1)); done; fi",
|
|
|
|
|
"write_meta env_reuse_mode \"$env_reuse_mode\"",
|
|
|
|
|
"write_meta env_reuse_node_deps_path \"$env_reuse_node_deps_path\"",
|
|
|
|
|
"write_meta env_reuse_node_deps_present \"$env_reuse_node_deps_present\"",
|
|
|
|
|
"write_meta env_reuse_node_deps_entries \"$env_reuse_node_deps_entries\"",
|
|
|
|
|
"write_meta env_reuse_linked_node_deps \"$env_reuse_linked_node_deps\"",
|
|
|
|
|
"node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE'",
|
|
|
|
|
"const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
|
|
|
|
|
"NODE",
|
|
|
|
|
"monitor_web_verify_started_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" <<'NODE' >> \"$event_log\"",
|
|
|
|
|
"const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries, linkedNodeDeps] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), linkedNodeDeps: Number(linkedNodeDeps || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));",
|
|
|
|
|
"NODE",
|
|
|
|
|
"monitor_web_verify_started_ms=$(now_ms)",
|
|
|
|
|
"write_meta monitor_web_verify_started_ms \"$monitor_web_verify_started_ms\"",
|
|
|
|
|
"emit_stage monitor-web-verify running \"$monitor_web_verify_started_ms\"",
|
|
|
|
|
"if ! bun scripts/verify-web-probe-sentinel-monitor-web.ts > /tmp/web-probe-sentinel-monitor-web-verify.log 2>&1; then cat /tmp/web-probe-sentinel-monitor-web-verify.log; emit_stage monitor-web-verify failed \"$monitor_web_verify_started_ms\"; exit 1; fi",
|
|
|
|
|
"cat /tmp/web-probe-sentinel-monitor-web-verify.log",
|
|
|
|
|
"monitor_web_verify_finished_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"monitor_web_verify_finished_ms=$(now_ms)",
|
|
|
|
|
"write_meta monitor_web_verify_finished_ms \"$monitor_web_verify_finished_ms\"",
|
|
|
|
|
"emit_stage monitor-web-verify succeeded \"$monitor_web_verify_started_ms\"",
|
|
|
|
|
"mkdir -p .unidesk-sentinel-bin",
|
|
|
|
|
"cat > .unidesk-sentinel-bin/trans <<'SH_TRANS'",
|
|
|
|
@@ -1756,7 +1815,7 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
"chmod 0755 .unidesk-sentinel-bin/trans",
|
|
|
|
|
"DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'",
|
|
|
|
|
"const fs = require('node:fs');",
|
|
|
|
|
"fs.writeFileSync('Dockerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));",
|
|
|
|
|
"fs.writeFileSync('Containerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));",
|
|
|
|
|
"NODE",
|
|
|
|
|
"cat > .dockerignore <<'EOF_DOCKERIGNORE'",
|
|
|
|
|
".git",
|
|
|
|
@@ -1779,37 +1838,131 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
".env",
|
|
|
|
|
".env.*",
|
|
|
|
|
"EOF_DOCKERIGNORE",
|
|
|
|
|
"docker_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')",
|
|
|
|
|
"docker_build_http_proxy_present=false; if [ -n \"$docker_build_http_proxy\" ]; then docker_build_http_proxy_present=true; fi",
|
|
|
|
|
"docker_build_https_proxy_present=false; if [ -n \"$docker_build_https_proxy\" ]; then docker_build_https_proxy_present=true; fi",
|
|
|
|
|
"docker_build_all_proxy_present=false; if [ -n \"$docker_build_all_proxy\" ]; then docker_build_all_proxy_present=true; fi",
|
|
|
|
|
"docker_build_no_proxy_present=false; if [ -n \"$docker_build_no_proxy\" ]; then docker_build_no_proxy_present=true; fi",
|
|
|
|
|
"docker_build_started_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"emit_stage docker-build running \"$docker_build_started_ms\"",
|
|
|
|
|
"if ! env HTTP_PROXY=\"$docker_build_http_proxy\" HTTPS_PROXY=\"$docker_build_https_proxy\" ALL_PROXY=\"$docker_build_all_proxy\" NO_PROXY=\"$docker_build_no_proxy\" http_proxy=\"$docker_build_http_proxy\" https_proxy=\"$docker_build_https_proxy\" all_proxy=\"$docker_build_all_proxy\" no_proxy=\"$docker_build_no_proxy\" docker build --network \"$docker_build_network_mode\" --build-arg HTTP_PROXY=\"$docker_build_http_proxy\" --build-arg HTTPS_PROXY=\"$docker_build_https_proxy\" --build-arg ALL_PROXY=\"$docker_build_all_proxy\" --build-arg NO_PROXY=\"$docker_build_no_proxy\" --build-arg http_proxy=\"$docker_build_http_proxy\" --build-arg https_proxy=\"$docker_build_https_proxy\" --build-arg all_proxy=\"$docker_build_all_proxy\" --build-arg no_proxy=\"$docker_build_no_proxy\" -f Dockerfile.web-probe-sentinel -t \"$image_ref\" . > /tmp/web-probe-sentinel-docker-build.log 2>&1; then cat /tmp/web-probe-sentinel-docker-build.log; emit_stage docker-build failed \"$docker_build_started_ms\"; exit 1; fi",
|
|
|
|
|
"cat /tmp/web-probe-sentinel-docker-build.log",
|
|
|
|
|
"docker_build_finished_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"emit_stage docker-build succeeded \"$docker_build_started_ms\"",
|
|
|
|
|
"docker_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' /tmp/web-probe-sentinel-docker-build.log 2>/dev/null || true)",
|
|
|
|
|
"docker_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' /tmp/web-probe-sentinel-docker-build.log 2>/dev/null || true)",
|
|
|
|
|
"docker_build_log_tail_b64=$(tail -n 30 /tmp/web-probe-sentinel-docker-build.log 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')",
|
|
|
|
|
"docker_push_started_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"emit_stage docker-push running \"$docker_push_started_ms\"",
|
|
|
|
|
"if ! docker push \"$image_ref\" > /tmp/web-probe-sentinel-docker-push.log 2>&1; then cat /tmp/web-probe-sentinel-docker-push.log; emit_stage docker-push failed \"$docker_push_started_ms\"; exit 1; fi",
|
|
|
|
|
"cat /tmp/web-probe-sentinel-docker-push.log",
|
|
|
|
|
"docker_push_finished_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"emit_stage docker-push succeeded \"$docker_push_started_ms\"",
|
|
|
|
|
"tag=${image_ref##*:}",
|
|
|
|
|
"repo_no_tag=${image_ref%:*}",
|
|
|
|
|
"registry_path=${repo_no_tag#127.0.0.1:5000/}",
|
|
|
|
|
"digest=$(awk '/digest: sha256:/ {print $3; exit}' /tmp/web-probe-sentinel-docker-push.log)",
|
|
|
|
|
"if [ -z \"$digest\" ]; then digest=$(curl -fsSI --max-time 10 \"http://127.0.0.1:5000/v2/$registry_path/manifests/$tag\" 2>/dev/null | awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}'); fi",
|
|
|
|
|
"context_ignore_entries=$(wc -l < .dockerignore | tr -d ' ')",
|
|
|
|
|
"write_meta context_ignore_entries \"$context_ignore_entries\"",
|
|
|
|
|
"chmod -R a+rwX /workspace",
|
|
|
|
|
"trap - EXIT",
|
|
|
|
|
].join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string {
|
|
|
|
|
const monitorWeb = record(state.image.monitorWeb);
|
|
|
|
|
const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode");
|
|
|
|
|
const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode");
|
|
|
|
|
const imageBuildProxySource = stringAt(monitorWeb, "imageBuildProxySource");
|
|
|
|
|
const imageBuildProxy = state.spec.networkProfile.imageBuildProxy;
|
|
|
|
|
const imageBuildNoProxy = imageBuildProxy.noProxy.join(",");
|
|
|
|
|
return [
|
|
|
|
|
"set -eu",
|
|
|
|
|
`job_name=${shellQuote(jobName)}`,
|
|
|
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
|
|
|
`image_build_builder=${shellQuote(requireSentinelBuildkitImage(state))}`,
|
|
|
|
|
`image_build_package_mode=${shellQuote(imageBuildPackageMode)}`,
|
|
|
|
|
`image_build_network_mode=${shellQuote(imageBuildNetworkMode)}`,
|
|
|
|
|
`image_build_proxy_source=${shellQuote(imageBuildProxySource)}`,
|
|
|
|
|
`image_build_http_proxy=${shellQuote(imageBuildProxy.http)}`,
|
|
|
|
|
`image_build_https_proxy=${shellQuote(imageBuildProxy.https)}`,
|
|
|
|
|
`image_build_all_proxy=${shellQuote(imageBuildProxy.all)}`,
|
|
|
|
|
`image_build_no_proxy=${shellQuote(imageBuildNoProxy)}`,
|
|
|
|
|
"meta_dir=/workspace/meta",
|
|
|
|
|
"event_log=/workspace/publish-events.log",
|
|
|
|
|
"build_log=/workspace/image-build.log",
|
|
|
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
|
|
|
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
|
|
|
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
|
|
|
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
|
|
|
|
|
"trap emit_failed EXIT",
|
|
|
|
|
"cd /workspace/source",
|
|
|
|
|
"image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi",
|
|
|
|
|
"image_build_https_proxy_present=false; if [ -n \"$image_build_https_proxy\" ]; then image_build_https_proxy_present=true; fi",
|
|
|
|
|
"image_build_all_proxy_present=false; if [ -n \"$image_build_all_proxy\" ]; then image_build_all_proxy_present=true; fi",
|
|
|
|
|
"image_build_no_proxy_present=false; if [ -n \"$image_build_no_proxy\" ]; then image_build_no_proxy_present=true; fi",
|
|
|
|
|
"write_meta image_build_builder \"$image_build_builder\"",
|
|
|
|
|
"write_meta image_build_package_mode \"$image_build_package_mode\"",
|
|
|
|
|
"write_meta image_build_network_mode \"$image_build_network_mode\"",
|
|
|
|
|
"write_meta image_build_proxy_source \"$image_build_proxy_source\"",
|
|
|
|
|
"write_meta image_build_http_proxy_present \"$image_build_http_proxy_present\"",
|
|
|
|
|
"write_meta image_build_https_proxy_present \"$image_build_https_proxy_present\"",
|
|
|
|
|
"write_meta image_build_all_proxy_present \"$image_build_all_proxy_present\"",
|
|
|
|
|
"write_meta image_build_no_proxy_present \"$image_build_no_proxy_present\"",
|
|
|
|
|
"image_build_started_ms=$(now_ms)",
|
|
|
|
|
"write_meta image_build_started_ms \"$image_build_started_ms\"",
|
|
|
|
|
"emit_stage image-build running \"$image_build_started_ms\"",
|
|
|
|
|
"if ! env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl-daemonless.sh build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1; then cat \"$build_log\"; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi",
|
|
|
|
|
"cat \"$build_log\"",
|
|
|
|
|
"image_build_finished_ms=$(now_ms)",
|
|
|
|
|
"write_meta image_build_finished_ms \"$image_build_finished_ms\"",
|
|
|
|
|
"metadata_compact=$(tr -d '\\n' < /workspace/build-metadata.json)",
|
|
|
|
|
"digest=$(printf '%s' \"$metadata_compact\" | sed -n 's/.*\"containerimage.digest\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' | head -n 1)",
|
|
|
|
|
"test -n \"$digest\"",
|
|
|
|
|
"repo_no_tag=${image_ref%:*}",
|
|
|
|
|
"digest_ref=\"$repo_no_tag@$digest\"",
|
|
|
|
|
"write_meta digest \"$digest\"",
|
|
|
|
|
"write_meta digest_ref \"$digest_ref\"",
|
|
|
|
|
"emit_stage image-build succeeded \"$image_build_started_ms\"",
|
|
|
|
|
"chmod -R a+rwX /workspace",
|
|
|
|
|
"trap - EXIT",
|
|
|
|
|
].join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
|
|
|
|
|
const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
|
|
|
|
|
const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
|
|
|
|
|
path: file.path,
|
|
|
|
|
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
|
|
|
|
}))), "utf8").toString("base64");
|
|
|
|
|
return [
|
|
|
|
|
"set -eu",
|
|
|
|
|
`job_name=${shellQuote(jobName)}`,
|
|
|
|
|
`image_ref=${shellQuote(state.image.ref)}`,
|
|
|
|
|
`image_repository=${shellQuote(state.image.repository)}`,
|
|
|
|
|
`gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`,
|
|
|
|
|
`gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`,
|
|
|
|
|
`files_b64=${shellQuote(filesB64)}`,
|
|
|
|
|
"meta_dir=/workspace/meta",
|
|
|
|
|
"event_log=/workspace/publish-events.log",
|
|
|
|
|
"build_log=/workspace/image-build.log",
|
|
|
|
|
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
|
|
|
|
|
"read_meta() { cat \"$meta_dir/$1\"; }",
|
|
|
|
|
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }",
|
|
|
|
|
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }",
|
|
|
|
|
"trap emit_failed EXIT",
|
|
|
|
|
"if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi",
|
|
|
|
|
"mkdir -p /root/.ssh",
|
|
|
|
|
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
|
|
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
|
|
|
"export GIT_SSH_COMMAND='ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
|
|
|
|
"started_ms=$(read_meta started_ms)",
|
|
|
|
|
"source_commit=$(read_meta source_commit)",
|
|
|
|
|
"mirror_commit=$(read_meta mirror_commit)",
|
|
|
|
|
"source_fetch_started_ms=$(read_meta source_fetch_started_ms)",
|
|
|
|
|
"source_fetch_finished_ms=$(read_meta source_fetch_finished_ms)",
|
|
|
|
|
"monitor_web_verify_started_ms=$(read_meta monitor_web_verify_started_ms)",
|
|
|
|
|
"monitor_web_verify_finished_ms=$(read_meta monitor_web_verify_finished_ms)",
|
|
|
|
|
"image_build_started_ms=$(read_meta image_build_started_ms)",
|
|
|
|
|
"image_build_finished_ms=$(read_meta image_build_finished_ms)",
|
|
|
|
|
"digest_ref=$(read_meta digest_ref)",
|
|
|
|
|
"env_reuse_mode=$(read_meta env_reuse_mode 2>/dev/null || printf 'k8s-buildkit-and-ci-node-deps')",
|
|
|
|
|
"env_reuse_node_deps_path=$(read_meta env_reuse_node_deps_path 2>/dev/null || printf '-')",
|
|
|
|
|
"env_reuse_node_deps_present=$(read_meta env_reuse_node_deps_present)",
|
|
|
|
|
"env_reuse_node_deps_entries=$(read_meta env_reuse_node_deps_entries)",
|
|
|
|
|
"env_reuse_linked_node_deps=$(read_meta env_reuse_linked_node_deps)",
|
|
|
|
|
"image_build_builder=$(read_meta image_build_builder)",
|
|
|
|
|
"image_build_package_mode=$(read_meta image_build_package_mode)",
|
|
|
|
|
"image_build_network_mode=$(read_meta image_build_network_mode)",
|
|
|
|
|
"image_build_proxy_source=$(read_meta image_build_proxy_source)",
|
|
|
|
|
"image_build_http_proxy_present=$(read_meta image_build_http_proxy_present)",
|
|
|
|
|
"image_build_https_proxy_present=$(read_meta image_build_https_proxy_present)",
|
|
|
|
|
"image_build_all_proxy_present=$(read_meta image_build_all_proxy_present)",
|
|
|
|
|
"image_build_no_proxy_present=$(read_meta image_build_no_proxy_present)",
|
|
|
|
|
"context_ignore_entries=$(read_meta context_ignore_entries)",
|
|
|
|
|
"image_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' \"$build_log\" 2>/dev/null || true)",
|
|
|
|
|
"image_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' \"$build_log\" 2>/dev/null || true)",
|
|
|
|
|
"image_build_log_tail_b64=$(tail -n 30 \"$build_log\" 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')",
|
|
|
|
|
"gitops_commit=''",
|
|
|
|
|
"changed=false",
|
|
|
|
|
"file_count=0",
|
|
|
|
|
"gitops_started_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"gitops_started_ms=$(now_ms)",
|
|
|
|
|
"if [ \"$files_b64\" != \"W10=\" ]; then",
|
|
|
|
|
" emit_stage gitops running \"$gitops_started_ms\"",
|
|
|
|
|
" gitops_cache=\"/cache/${gitops_repository}.git\"",
|
|
|
|
@@ -1840,13 +1993,13 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
|
|
|
|
|
"else",
|
|
|
|
|
" emit_stage gitops skipped \"$gitops_started_ms\"",
|
|
|
|
|
"fi",
|
|
|
|
|
"gitops_finished_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"finished_ms=$(node -e 'console.log(Date.now())')",
|
|
|
|
|
"node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$docker_build_started_ms\" \"$docker_build_finished_ms\" \"$docker_push_started_ms\" \"$docker_push_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$docker_build_cache_hits\" \"$docker_build_step_lines\" \"$docker_build_log_tail_b64\" \"$docker_build_package_mode\" \"$docker_build_network_mode\" \"$docker_build_proxy_source\" \"$docker_build_http_proxy_present\" \"$docker_build_https_proxy_present\" \"$docker_build_all_proxy_present\" \"$docker_build_no_proxy_present\" \"$docker_ignore_entries\" <<'NODE'",
|
|
|
|
|
"const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, dockerBuildStartedMs, dockerBuildFinishedMs, dockerPushStartedMs, dockerPushFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, dockerBuildCacheHits, dockerBuildStepLines, dockerBuildLogTailB64, dockerBuildPackageMode, dockerBuildNetworkMode, dockerBuildProxySource, dockerBuildHttpProxyPresent, dockerBuildHttpsProxyPresent, dockerBuildAllProxyPresent, dockerBuildNoProxyPresent, dockerIgnoreEntries] = process.argv.slice(2);",
|
|
|
|
|
"gitops_finished_ms=$(now_ms)",
|
|
|
|
|
"finished_ms=$(now_ms)",
|
|
|
|
|
"node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$monitor_web_verify_started_ms\" \"$monitor_web_verify_finished_ms\" \"$image_build_started_ms\" \"$image_build_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$env_reuse_linked_node_deps\" \"$image_build_cache_hits\" \"$image_build_step_lines\" \"$image_build_log_tail_b64\" \"$image_build_builder\" \"$image_build_package_mode\" \"$image_build_network_mode\" \"$image_build_proxy_source\" \"$image_build_http_proxy_present\" \"$image_build_https_proxy_present\" \"$image_build_all_proxy_present\" \"$image_build_no_proxy_present\" \"$context_ignore_entries\" <<'NODE'",
|
|
|
|
|
"const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs, imageBuildStartedMs, imageBuildFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, envReuseLinkedNodeDeps, imageBuildCacheHits, imageBuildStepLines, imageBuildLogTailB64, imageBuildBuilder, imageBuildPackageMode, imageBuildNetworkMode, imageBuildProxySource, imageBuildHttpProxyPresent, imageBuildHttpsProxyPresent, imageBuildAllProxyPresent, imageBuildNoProxyPresent, contextIgnoreEntries] = process.argv.slice(2);",
|
|
|
|
|
"const elapsed = (start, finish) => Number(finish) - Number(start);",
|
|
|
|
|
"const cacheHits = Number(dockerBuildCacheHits || 0);",
|
|
|
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), dockerBuildMs: elapsed(dockerBuildStartedMs, dockerBuildFinishedMs), dockerPushMs: elapsed(dockerPushStartedMs, dockerPushFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, dockerBuild: { cacheHitLines: cacheHits, stepLines: Number(dockerBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: dockerBuildPackageMode, networkMode: dockerBuildNetworkMode, proxySource: dockerBuildProxySource, proxy: { httpProxyPresent: dockerBuildHttpProxyPresent === 'true', httpsProxyPresent: dockerBuildHttpsProxyPresent === 'true', allProxyPresent: dockerBuildAllProxyPresent === 'true', noProxyPresent: dockerBuildNoProxyPresent === 'true', valuesRedacted:true }, dockerIgnoreEntries: Number(dockerIgnoreEntries || 0), verifyLocation: 'pre-docker-build', logTail: Buffer.from(dockerBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'docker-build', 'docker-push', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
|
|
|
|
|
"const cacheHits = Number(imageBuildCacheHits || 0);",
|
|
|
|
|
"console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), monitorWebVerifyMs: elapsed(monitorWebVerifyStartedMs, monitorWebVerifyFinishedMs), imageBuildMs: elapsed(imageBuildStartedMs, imageBuildFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), linkedNodeDeps: Number(envReuseLinkedNodeDeps || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, imageBuild: { builder: 'k8s-buildkit-rootless', builderImage: imageBuildBuilder, cacheHitLines: cacheHits, stepLines: Number(imageBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', packageMode: imageBuildPackageMode, networkMode: imageBuildNetworkMode, proxySource: imageBuildProxySource, proxy: { httpProxyPresent: imageBuildHttpProxyPresent === 'true', httpsProxyPresent: imageBuildHttpsProxyPresent === 'true', allProxyPresent: imageBuildAllProxyPresent === 'true', noProxyPresent: imageBuildNoProxyPresent === 'true', valuesRedacted:true }, contextIgnoreEntries: Number(contextIgnoreEntries || 0), verifyLocation: 'pre-image-build', logTail: Buffer.from(imageBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'monitor-web-verify', 'image-build', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));",
|
|
|
|
|
"NODE",
|
|
|
|
|
"trap - EXIT",
|
|
|
|
|
].join("\n");
|
|
|
|
@@ -1900,7 +2053,7 @@ function probeK8sJobScript(namespace: string, jobName: string): string {
|
|
|
|
|
"pod_phase=''",
|
|
|
|
|
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
|
|
|
|
|
"logs_tail=''",
|
|
|
|
|
"if [ -n \"$pod\" ]; then logs_tail=$(kubectl -n \"$namespace\" logs \"$pod\" --tail=120 2>/dev/null | tail -c 12000 | base64 | tr -d '\\n'); fi",
|
|
|
|
|
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=120 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null || true; done; } | tail -c 16000 | base64 | tr -d '\\n'); fi",
|
|
|
|
|
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
|
|
|
|
"const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
|
|
|
|
|
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
|
|
|
@@ -1937,7 +2090,7 @@ function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: Sentinel
|
|
|
|
|
: `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`,
|
|
|
|
|
logs: result.jobName === "-"
|
|
|
|
|
? "-"
|
|
|
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --tail=120`,
|
|
|
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --all-containers=true --tail=120`,
|
|
|
|
|
describe: result.jobName === "-"
|
|
|
|
|
? "-"
|
|
|
|
|
: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`,
|
|
|
|
@@ -1994,7 +2147,7 @@ function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: rea
|
|
|
|
|
const running = reversed.find((event) => event.status === "running");
|
|
|
|
|
if (running !== undefined) return text(running.stage);
|
|
|
|
|
const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage)));
|
|
|
|
|
const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "docker-build", "docker-push", "gitops"] : ["source-mirror-fetch"];
|
|
|
|
|
const order = domain === "publish" ? ["source-fetch", "monitor-web-verify", "image-build", "gitops"] : ["source-mirror-fetch"];
|
|
|
|
|
const next = order.find((stage) => !completed.has(stage));
|
|
|
|
|
return next ?? result.phase;
|
|
|
|
|
}
|
|
|
|
@@ -2640,11 +2793,11 @@ function renderPublishResult(publish: Record<string, unknown>): string {
|
|
|
|
|
const diagnostics = record(publish.diagnostics);
|
|
|
|
|
const diagnosticEnvReuse = record(diagnostics.envReuse);
|
|
|
|
|
const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse;
|
|
|
|
|
const dockerBuild = record(payload.dockerBuild);
|
|
|
|
|
const dockerBuildProxy = record(dockerBuild.proxy);
|
|
|
|
|
const imageBuild = record(payload.imageBuild);
|
|
|
|
|
const imageBuildProxy = record(imageBuild.proxy);
|
|
|
|
|
const timings = record(payload.stageTimings);
|
|
|
|
|
const commands = record(diagnostics.commands);
|
|
|
|
|
const proxySummary = [dockerBuildProxy.httpProxyPresent, dockerBuildProxy.httpsProxyPresent, dockerBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none";
|
|
|
|
|
const proxySummary = [imageBuildProxy.httpProxyPresent, imageBuildProxy.httpsProxyPresent, imageBuildProxy.allProxyPresent].some((item) => item === true) ? "present" : "none";
|
|
|
|
|
const lines = [
|
|
|
|
|
"PUBLISH",
|
|
|
|
|
table(["OK", "PHASE", "JOB", "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[
|
|
|
|
@@ -2672,22 +2825,22 @@ function renderPublishResult(publish: Record<string, unknown>): string {
|
|
|
|
|
]]),
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (Object.keys(dockerBuild).length > 0 || Object.keys(timings).length > 0) {
|
|
|
|
|
if (Object.keys(imageBuild).length > 0 || Object.keys(timings).length > 0) {
|
|
|
|
|
lines.push(
|
|
|
|
|
"",
|
|
|
|
|
"PUBLISH_BUILD",
|
|
|
|
|
table(["PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "BUILD_MS", "PUSH_MS", "GITOPS_MS", "TOTAL_MS"], [[
|
|
|
|
|
dockerBuild.packageMode ?? "-",
|
|
|
|
|
dockerBuild.networkMode ?? "-",
|
|
|
|
|
table(["BUILDER", "PACKAGE", "NETWORK", "PROXY", "IGNORE", "CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "VERIFY_MS", "IMAGE_MS", "GITOPS_MS", "TOTAL_MS"], [[
|
|
|
|
|
imageBuild.builder ?? "-",
|
|
|
|
|
imageBuild.packageMode ?? "-",
|
|
|
|
|
imageBuild.networkMode ?? "-",
|
|
|
|
|
proxySummary,
|
|
|
|
|
dockerBuild.dockerIgnoreEntries ?? "-",
|
|
|
|
|
dockerBuild.layerCache ?? "-",
|
|
|
|
|
dockerBuild.cacheHitLines ?? "-",
|
|
|
|
|
dockerBuild.stepLines ?? "-",
|
|
|
|
|
imageBuild.contextIgnoreEntries ?? "-",
|
|
|
|
|
imageBuild.layerCache ?? "-",
|
|
|
|
|
imageBuild.cacheHitLines ?? "-",
|
|
|
|
|
imageBuild.stepLines ?? "-",
|
|
|
|
|
timings.sourceFetchMs ?? "-",
|
|
|
|
|
timings.monitorWebVerifyMs ?? "-",
|
|
|
|
|
timings.dockerBuildMs ?? "-",
|
|
|
|
|
timings.dockerPushMs ?? "-",
|
|
|
|
|
timings.imageBuildMs ?? "-",
|
|
|
|
|
timings.gitopsMs ?? "-",
|
|
|
|
|
timings.totalMs ?? payload.elapsedMs ?? "-",
|
|
|
|
|
]]),
|
|
|
|
@@ -2742,7 +2895,7 @@ function renderImageResult(result: Record<string, unknown>): string {
|
|
|
|
|
"",
|
|
|
|
|
table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]),
|
|
|
|
|
"",
|
|
|
|
|
Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "BUILD_PKG", "BUILD_NET", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.dockerBuildPackageMode, monitorWeb.dockerBuildNetworkMode, monitorWeb.dockerBuildContextIgnore]]),
|
|
|
|
|
Object.keys(monitorWeb).length === 0 ? "MONITOR_WEB\n-" : table(["STACK", "MODE", "ASSETS", "VERIFY", "ENV_REUSE", "IMAGE_BUILDER", "BUILD_PKG", "BUILD_NET", "CTX_IGNORE"], [[monitorWeb.stack, monitorWeb.runtimeMode, monitorWeb.assetRoot, monitorWeb.verifyCommand, `${monitorWeb.envReuseMode}:${monitorWeb.envReuseNodeDepsPath}`, monitorWeb.imageBuildBuilder ?? "-", monitorWeb.imageBuildPackageMode, monitorWeb.imageBuildNetworkMode, monitorWeb.imageBuildContextIgnore]]),
|
|
|
|
|
"",
|
|
|
|
|
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
|
|
|
|
|
"",
|
|
|
|
|
Codex