From 244e46b71db51757cd1635a8f513c4372c163f5c Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 4 Jul 2026 00:29:37 +0000 Subject: [PATCH] feat(cicd): add branch follower closeout drilldown --- scripts/native/cicd/k8s-job-drilldown.mjs | 355 +++++++++++++++++++++ scripts/native/cicd/runtime-drilldown.mjs | 356 ++++++++++++++++++++++ scripts/src/cicd-branch-follower.ts | 49 ++- scripts/src/cicd-drilldown-render.ts | 122 ++++++++ scripts/src/cicd-job-runtime-drilldown.ts | 237 ++++++++++++++ scripts/src/cicd-render.ts | 2 +- scripts/src/cicd-types.ts | 5 +- 7 files changed, 1121 insertions(+), 5 deletions(-) create mode 100644 scripts/native/cicd/k8s-job-drilldown.mjs create mode 100644 scripts/native/cicd/runtime-drilldown.mjs create mode 100644 scripts/src/cicd-job-runtime-drilldown.ts diff --git a/scripts/native/cicd/k8s-job-drilldown.mjs b/scripts/native/cicd/k8s-job-drilldown.mjs new file mode 100644 index 00000000..90c5ae3a --- /dev/null +++ b/scripts/native/cicd/k8s-job-drilldown.mjs @@ -0,0 +1,355 @@ +import { existsSync, readFileSync } from "node:fs"; +import https from "node:https"; +import { spawnSync } from "node:child_process"; + +const namespace = requiredEnv("JOB_NAMESPACE"); +const jobName = requiredEnv("JOB_NAME"); +const stageName = process.env.STAGE_NAME || "job"; +const sourceCommit = process.env.SOURCE_COMMIT || ""; +const logsTailLines = positiveInt(process.env.LOGS_TAIL_LINES, "LOGS_TAIL_LINES"); +const maxLogBytes = positiveInt(process.env.MAX_LOG_BYTES, "MAX_LOG_BYTES"); +const maxMessageBytes = positiveInt(process.env.MAX_MESSAGE_BYTES, "MAX_MESSAGE_BYTES"); +const maxContainers = positiveInt(process.env.MAX_CONTAINERS, "MAX_CONTAINERS"); + +const useServiceAccount = Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) + && existsSync("/var/run/secrets/kubernetes.io/serviceaccount/token") + && existsSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +class KubeReadError extends Error { + constructor(reason, message, details = {}) { + super(shortText(message) || reason); + this.name = "KubeReadError"; + this.reason = reason; + this.statusCode = details.statusCode ?? null; + this.notFound = details.notFound === true; + this.path = details.path || null; + } +} + +main().catch((error) => { + process.stderr.write(error?.message || String(error)); + process.exit(1); +}); + +async function main() { + const job = await getJson(`/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs/${encodeURIComponent(jobName)}`, false); + if (job === null) { + console.log(JSON.stringify({ + ok: false, + degradedReason: "job-not-found", + message: `Job ${namespace}/${jobName} was not found; it may have expired by ttlSecondsAfterFinished`, + query: { namespace, jobName, stage: stageName, sourceCommit: sourceCommit || null }, + job: null, + pods: [], + logs: [], + statusAuthority: useServiceAccount ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + })); + return; + } + const pods = await listJobPods(job); + const podSummaries = pods.slice(0, Math.max(1, maxContainers)).map(podSummary); + const logTargets = selectLogTargets(pods).slice(0, maxContainers); + const logs = []; + const perContainerBytes = Math.max(1, Math.floor(maxLogBytes / Math.max(1, logTargets.length))); + for (const target of logTargets) { + const read = await readPodLog(target.podName, target.container, logsTailLines, perContainerBytes); + const text = read.tail || ""; + logs.push({ + ok: read.ok, + degradedReason: read.degradedReason, + message: read.message, + pod: target.podName, + container: target.container, + lineCount: text.length === 0 ? 0 : text.split(/\r?\n/u).filter((line) => line.length > 0).length, + bytes: Buffer.byteLength(text, "utf8"), + tail: text, + nodeCicdTiming: lastNodeCicdTiming(text), + }); + } + const logFailures = logs.filter((item) => item.ok === false); + const status = job.status || {}; + const metadata = job.metadata || {}; + console.log(JSON.stringify({ + ok: logFailures.length === 0, + degradedReason: logFailures.length === 0 ? null : "log-read-failed", + errors: logFailures.map((item) => ({ pod: item.pod, container: item.container, degradedReason: item.degradedReason, message: item.message })), + query: { namespace, jobName, stage: stageName, sourceCommit: sourceCommit || null }, + job: { + name: metadata.name || jobName, + namespace: metadata.namespace || namespace, + stage: stageName, + sourceCommit: shortSha(sourceCommit), + createdAt: metadata.creationTimestamp || null, + startTime: status.startTime || null, + completionTime: status.completionTime || null, + durationSeconds: durationSeconds(status.startTime, status.completionTime), + activeDeadlineSeconds: job.spec?.activeDeadlineSeconds ?? null, + ttlSecondsAfterFinished: job.spec?.ttlSecondsAfterFinished ?? null, + active: integerOrNull(status.active), + succeeded: integerOrNull(status.succeeded), + failed: integerOrNull(status.failed), + condition: compactCondition(latestCondition(status.conditions)), + completed: Boolean(status.succeeded && status.succeeded > 0), + failedState: Boolean(status.failed && status.failed > 0), + }, + pods: podSummaries, + logs, + nodeCicdTiming: lastTiming(logs), + statusAuthority: useServiceAccount ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + })); +} + +async function listJobPods(job) { + const uid = job?.metadata?.uid || ""; + const selectors = [ + `batch.kubernetes.io/job-name=${jobName}`, + `job-name=${jobName}`, + uid ? `controller-uid=${uid}` : null, + ].filter(Boolean); + for (const selector of selectors) { + const list = await getJson(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${encodeURIComponent(selector)}`, false); + const items = Array.isArray(list?.items) ? list.items : []; + if (items.length > 0) return items.sort((left, right) => String(right.metadata?.creationTimestamp || "").localeCompare(String(left.metadata?.creationTimestamp || ""))); + } + return []; +} + +function podSummary(pod) { + const statuses = allContainerStatuses(pod); + return { + name: pod.metadata?.name || null, + phase: pod.status?.phase || null, + createdAt: pod.metadata?.creationTimestamp || null, + startTime: pod.status?.startTime || null, + ready: conditionByType(pod.status, "Ready")?.status || null, + containers: statuses.slice(0, maxContainers).map(containerSummary), + }; +} + +function selectLogTargets(pods) { + const targets = []; + for (const pod of pods) { + for (const status of allContainerStatuses(pod)) { + if (status.name) targets.push({ podName: pod.metadata?.name, container: status.name, status }); + } + } + targets.sort((left, right) => scoreContainer(right.status) - scoreContainer(left.status)); + return targets.filter((item) => item.podName && item.container); +} + +function scoreContainer(status) { + if (status.state?.waiting) return 5; + const terminated = status.state?.terminated || status.lastState?.terminated; + if (terminated && terminated.exitCode !== 0) return 4; + if (terminated) return 3; + if (status.state?.running) return 2; + return 1; +} + +function allContainerStatuses(pod) { + const status = pod?.status || {}; + return [ + ...arrayItems(status.initContainerStatuses), + ...arrayItems(status.containerStatuses), + ...arrayItems(status.ephemeralContainerStatuses), + ]; +} + +function containerSummary(item) { + return { + name: item.name || null, + ready: item.ready === true, + restartCount: integerOrNull(item.restartCount), + waiting: compactWaiting(item.state?.waiting), + running: item.state?.running ? { startedAt: item.state.running.startedAt || null } : null, + terminated: compactTerminated(item.state?.terminated || item.lastState?.terminated), + imageID: shortImageId(item.imageID || null), + }; +} + +async function readPodLog(podName, container, tailLines, limitBytes) { + const path = `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods/${encodeURIComponent(podName)}/log?container=${encodeURIComponent(container)}&tailLines=${tailLines}&limitBytes=${limitBytes}`; + try { + const text = tailBytes(await getText(path, false), limitBytes); + return { ok: true, degradedReason: null, message: null, tail: text }; + } catch (error) { + return { + ok: false, + degradedReason: isNotFoundError(error) ? "log-not-found" : error instanceof KubeReadError ? error.reason : "log-read-failed", + message: shortText(error?.message || String(error)), + tail: "", + }; + } +} + +async function getJson(path, required) { + let text = ""; + try { + text = await getText(path, required); + } catch (error) { + if (!required && isNotFoundError(error)) return null; + throw error; + } + try { + return JSON.parse(text); + } catch (error) { + throw new KubeReadError("invalid-json", `kube api returned non-JSON for ${path}: ${error?.message || String(error)}`, { path }); + } +} + +async function getText(path, required) { + if (useServiceAccount) return kubeApiGet(path, required); + const result = spawnSync("kubectl", ["get", "--raw", path], { encoding: "utf8", maxBuffer: Math.max(maxLogBytes * 2, 1024 * 1024) }); + if (result.error) throw new KubeReadError("transport-error", `kubectl get --raw transport error for ${path}: ${result.error.message}`, { path }); + if (result.status === 0) return result.stdout; + const body = result.stderr || result.stdout || `kubectl get --raw failed with exit ${result.status}`; + throw new KubeReadError(isNotFoundText(body) ? "not-found" : "kube-api-error", `kubectl get --raw failed for ${path}: ${body}`, { + path, + notFound: isNotFoundText(body), + }); +} + +function kubeApiGet(path, required) { + const host = process.env.KUBERNETES_SERVICE_HOST; + const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); + const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); + const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + return new Promise((resolve, reject) => { + const req = https.request({ host, port, path, method: "GET", ca, headers: { authorization: `Bearer ${token}` } }, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => { + const code = res.statusCode || 0; + if (code >= 200 && code < 300) resolve(body); + else reject(new KubeReadError(code === 404 || isNotFoundText(body) ? "not-found" : "kube-api-error", `kube api GET ${path} status ${code}: ${body || "-"}`, { + path, + statusCode: code, + notFound: code === 404 || isNotFoundText(body), + })); + }); + }); + req.on("error", (error) => reject(new KubeReadError("transport-error", `kube api GET ${path} transport error: ${error?.message || String(error)}`, { path }))); + req.end(); + }); +} + +function latestCondition(conditions) { + return arrayItems(conditions).slice().sort((left, right) => String(right.lastTransitionTime || "").localeCompare(String(left.lastTransitionTime || "")))[0] || null; +} + +function conditionByType(status, type) { + return arrayItems(status?.conditions).find((item) => item?.type === type) || null; +} + +function compactCondition(value) { + if (!value) return null; + return { type: value.type || null, status: value.status || null, reason: value.reason || null, message: shortText(value.message || null), lastTransitionTime: value.lastTransitionTime || null }; +} + +function compactTerminated(value) { + if (!value) return null; + return { reason: value.reason || null, exitCode: integerOrNull(value.exitCode), message: shortText(value.message || null), startedAt: value.startedAt || null, finishedAt: value.finishedAt || null }; +} + +function compactWaiting(value) { + if (!value) return null; + return { reason: value.reason || null, message: shortText(value.message || null) }; +} + +function lastNodeCicdTiming(text) { + const lines = text.split(/\r?\n/u).filter((line) => line.includes("node-cicd-timing")); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const parsed = parseTimingLine(lines[index]); + if (parsed !== null) return parsed; + } + return null; +} + +function parseTimingLine(line) { + const candidates = [line.trim()]; + const brace = line.indexOf("{"); + if (brace >= 0) candidates.push(line.slice(brace).trim()); + for (const candidate of candidates) { + try { + const parsed = JSON.parse(candidate); + if (parsed && typeof parsed === "object" && JSON.stringify(parsed).includes("node-cicd-timing")) return parsed; + } catch { + // Try the next bounded candidate. + } + } + return { rawTail: shortText(line) }; +} + +function lastTiming(logs) { + for (let index = logs.length - 1; index >= 0; index -= 1) { + if (logs[index].nodeCicdTiming !== null) return logs[index].nodeCicdTiming; + } + return null; +} + +function durationSeconds(start, end) { + const s = timestampMs(start); + const e = timestampMs(end); + return s === null || e === null || e < s ? null : Math.round((e - s) / 1000); +} + +function timestampMs(value) { + const parsed = Date.parse(String(value || "")); + return Number.isFinite(parsed) ? parsed : null; +} + +function shortImageId(value) { + if (!value) return null; + const text = String(value); + const at = text.lastIndexOf("@"); + return at >= 0 ? text.slice(at + 1, at + 25) : shortText(text); +} + +function shortSha(value) { + if (!value) return null; + const text = String(value); + return text.length > 12 ? text.slice(0, 12) : text; +} + +function shortText(value) { + if (value === null || value === undefined) return null; + const text = String(value).replace(/\s+/gu, " ").trim(); + return text.length <= maxMessageBytes ? text : `${text.slice(0, Math.max(0, maxMessageBytes - 3))}...`; +} + +function tailBytes(value, maxBytes) { + const buffer = Buffer.from(value, "utf8"); + if (buffer.length <= maxBytes) return value; + return buffer.subarray(buffer.length - maxBytes).toString("utf8"); +} + +function arrayItems(value) { + return Array.isArray(value) ? value : []; +} + +function integerOrNull(value) { + return Number.isInteger(value) ? value : null; +} + +function isNotFoundError(error) { + return error instanceof KubeReadError && error.notFound === true; +} + +function isNotFoundText(value) { + return /\b404\b|not found|NotFound/u.test(String(value || "")); +} + +function positiveInt(value, name) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); + return parsed; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} diff --git a/scripts/native/cicd/runtime-drilldown.mjs b/scripts/native/cicd/runtime-drilldown.mjs new file mode 100644 index 00000000..df6b7235 --- /dev/null +++ b/scripts/native/cicd/runtime-drilldown.mjs @@ -0,0 +1,356 @@ +import { existsSync, readFileSync } from "node:fs"; +import https from "node:https"; +import { spawnSync } from "node:child_process"; + +const namespace = requiredEnv("RUNTIME_NAMESPACE"); +const expectedSha = process.env.EXPECTED_SHA || ""; +const workloads = parseWorkloads(requiredEnv("WORKLOADS_B64")); +const maxMessageBytes = positiveInt(process.env.MAX_MESSAGE_BYTES, "MAX_MESSAGE_BYTES"); +const maxContainers = positiveInt(process.env.MAX_CONTAINERS, "MAX_CONTAINERS"); + +const useServiceAccount = Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) + && existsSync("/var/run/secrets/kubernetes.io/serviceaccount/token") + && existsSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +class KubeReadError extends Error { + constructor(reason, message, details = {}) { + super(shortText(message) || reason); + this.name = "KubeReadError"; + this.reason = reason; + this.statusCode = details.statusCode ?? null; + this.notFound = details.notFound === true; + this.path = details.path || null; + } +} + +main().catch((error) => { + process.stderr.write(error?.message || String(error)); + process.exit(1); +}); + +async function main() { + const items = []; + for (const spec of workloads) { + items.push(await workloadSummary(spec)); + } + const targetShas = items.map((item) => item.sourceCommit?.value).filter(Boolean); + const targetSha = uniqueOrNull(targetShas); + const ready = items.length > 0 && items.every((item) => item.ready === true); + const aligned = expectedSha ? targetShas.length > 0 && targetShas.every((value) => value === expectedSha) : null; + console.log(JSON.stringify({ + ok: items.every((item) => item.ok !== false), + degradedReason: items.some((item) => item.ok === false) ? "runtime-read-degraded" : null, + namespace, + expectedSha: shortSha(expectedSha), + targetSha: shortSha(targetSha), + ready, + aligned, + firstSeenReadyAt: null, + firstSeenExpectedShaAt: null, + lastObservedAt: new Date().toISOString(), + blockingReason: blockingReason(items, expectedSha), + workloads: items, + statusAuthority: useServiceAccount ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + })); +} + +async function workloadSummary(spec) { + const resource = await getJson(apiPathForWorkload(spec), false); + if (resource === null) { + return { + ok: false, + degradedReason: "workload-not-found", + kind: spec.kind, + name: spec.name, + namespace, + message: `${spec.kind} ${namespace}/${spec.name} was not found`, + ready: false, + aligned: expectedSha ? false : null, + sourceCommit: null, + pods: [], + }; + } + const selector = matchLabels(resource.spec?.selector?.matchLabels); + const pods = selector === null ? [] : await listPods(selector); + const sourceCommit = sourceCommitSummary(spec, resource, pods); + const ready = workloadReady(spec.kind, resource); + return { + ok: true, + kind: spec.kind, + name: spec.name, + namespace, + generation: integerOrNull(resource.metadata?.generation), + observedGeneration: integerOrNull(resource.status?.observedGeneration), + replicas: integerOrNull(resource.status?.replicas), + updatedReplicas: integerOrNull(resource.status?.updatedReplicas), + readyReplicas: integerOrNull(resource.status?.readyReplicas), + availableReplicas: integerOrNull(resource.status?.availableReplicas), + unavailableReplicas: integerOrNull(resource.status?.unavailableReplicas), + ready, + aligned: expectedSha && sourceCommit.value ? sourceCommit.value === expectedSha : null, + selector, + conditions: arrayItems(resource.status?.conditions).slice(-4).map(compactCondition), + sourceCommit, + pods: pods.slice(0, 4).map((pod) => podSummary(spec, pod)), + }; +} + +function apiPathForWorkload(spec) { + const plural = spec.kind === "StatefulSet" ? "statefulsets" : "deployments"; + return `/apis/apps/v1/namespaces/${encodeURIComponent(namespace)}/${plural}/${encodeURIComponent(spec.name)}`; +} + +async function listPods(selector) { + const labelSelector = Object.entries(selector).map(([key, value]) => `${key}=${value}`).join(","); + const list = await getJson(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${encodeURIComponent(labelSelector)}`, false); + return arrayItems(list?.items).sort((left, right) => String(right.metadata?.creationTimestamp || "").localeCompare(String(left.metadata?.creationTimestamp || ""))); +} + +function podSummary(spec, pod) { + const readyCondition = conditionByType(pod.status, "Ready"); + const sourceCommit = podSourceCommitSummary(spec, pod); + return { + name: pod.metadata?.name || null, + phase: pod.status?.phase || null, + createdAt: pod.metadata?.creationTimestamp || null, + startTime: pod.status?.startTime || null, + ready: readyCondition?.status || null, + readyReason: readyCondition?.reason || null, + readyMessage: shortText(readyCondition?.message || null), + readyLastTransitionTime: readyCondition?.lastTransitionTime || null, + sourceCommit, + containers: allContainerStatuses(pod).slice(0, maxContainers).map(containerSummary), + }; +} + +function sourceCommitSummary(spec, resource, pods) { + const checks = []; + for (const key of arrayItems(spec.sourceCommit?.labels)) checks.push({ source: "workload-label", key, value: resource.metadata?.labels?.[key] || null }); + for (const key of arrayItems(spec.sourceCommit?.annotations)) checks.push({ source: "workload-annotation", key, value: resource.metadata?.annotations?.[key] || null }); + const template = resource.spec?.template?.metadata || {}; + for (const key of arrayItems(spec.sourceCommit?.podLabels)) checks.push({ source: "template-label", key, value: template.labels?.[key] || null }); + for (const key of arrayItems(spec.sourceCommit?.podAnnotations)) checks.push({ source: "template-annotation", key, value: template.annotations?.[key] || null }); + const podCommit = pods.map((pod) => podSourceCommitSummary(spec, pod).value).find(Boolean) || null; + if (podCommit) checks.push({ source: "selected-pod", key: "pod", value: podCommit }); + const found = checks.find((item) => item.value); + return { + value: found?.value || null, + short: shortSha(found?.value || null), + source: found?.source || null, + key: found?.key || null, + checks: checks.map((item) => ({ source: item.source, key: item.key, short: shortSha(item.value) })), + }; +} + +function podSourceCommitSummary(spec, pod) { + const checks = []; + for (const key of arrayItems(spec.sourceCommit?.podLabels)) checks.push({ source: "pod-label", key, value: pod.metadata?.labels?.[key] || null }); + for (const key of arrayItems(spec.sourceCommit?.podAnnotations)) checks.push({ source: "pod-annotation", key, value: pod.metadata?.annotations?.[key] || null }); + for (const envName of arrayItems(spec.sourceCommit?.env)) { + const value = envValue(pod, envName); + checks.push({ source: "pod-env", key: envName, value }); + } + const found = checks.find((item) => item.value); + return { + value: found?.value || null, + short: shortSha(found?.value || null), + source: found?.source || null, + key: found?.key || null, + checks: checks.map((item) => ({ source: item.source, key: item.key, short: shortSha(item.value) })), + }; +} + +function envValue(pod, envName) { + for (const container of arrayItems(pod.spec?.containers)) { + for (const env of arrayItems(container.env)) { + if (env.name === envName && typeof env.value === "string") return env.value; + if (env.name === envName && env.valueFrom) return null; + } + } + return null; +} + +function workloadReady(kind, resource) { + const status = resource.status || {}; + if (kind === "Deployment") { + const desired = integerOrNull(resource.spec?.replicas) ?? 1; + const available = integerOrNull(status.availableReplicas) ?? 0; + const updated = integerOrNull(status.updatedReplicas) ?? 0; + return available >= desired && updated >= desired; + } + const desired = integerOrNull(resource.spec?.replicas) ?? 1; + const ready = integerOrNull(status.readyReplicas) ?? 0; + return ready >= desired; +} + +function blockingReason(items, expected) { + const missing = items.find((item) => item.ok === false); + if (missing) return `${missing.kind}/${missing.name} not-found`; + const notReady = items.find((item) => item.ready !== true); + if (notReady) return `${notReady.kind}/${notReady.name} not-ready`; + if (expected) { + const stale = items.find((item) => item.sourceCommit?.value && item.sourceCommit.value !== expected); + if (stale) return `${stale.kind}/${stale.name} source-mismatch`; + } + return null; +} + +async function getJson(path, required) { + let text = ""; + try { + text = await getText(path, required); + } catch (error) { + if (!required && isNotFoundError(error)) return null; + throw error; + } + try { + return JSON.parse(text); + } catch (error) { + throw new KubeReadError("invalid-json", `kube api returned non-JSON for ${path}: ${error?.message || String(error)}`, { path }); + } +} + +async function getText(path, required) { + if (useServiceAccount) return kubeApiGet(path, required); + const result = spawnSync("kubectl", ["get", "--raw", path], { encoding: "utf8", maxBuffer: 1024 * 1024 }); + if (result.error) throw new KubeReadError("transport-error", `kubectl get --raw transport error for ${path}: ${result.error.message}`, { path }); + if (result.status === 0) return result.stdout; + const body = result.stderr || result.stdout || `kubectl get --raw failed with exit ${result.status}`; + throw new KubeReadError(isNotFoundText(body) ? "not-found" : "kube-api-error", `kubectl get --raw failed for ${path}: ${body}`, { + path, + notFound: isNotFoundText(body), + }); +} + +function kubeApiGet(path, required) { + const host = process.env.KUBERNETES_SERVICE_HOST; + const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); + const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); + const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + return new Promise((resolve, reject) => { + const req = https.request({ host, port, path, method: "GET", ca, headers: { authorization: `Bearer ${token}` } }, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => { + const code = res.statusCode || 0; + if (code >= 200 && code < 300) resolve(body); + else reject(new KubeReadError(code === 404 || isNotFoundText(body) ? "not-found" : "kube-api-error", `kube api GET ${path} status ${code}: ${body || "-"}`, { + path, + statusCode: code, + notFound: code === 404 || isNotFoundText(body), + })); + }); + }); + req.on("error", (error) => reject(new KubeReadError("transport-error", `kube api GET ${path} transport error: ${error?.message || String(error)}`, { path }))); + req.end(); + }); +} + +function parseWorkloads(value) { + try { + const parsed = JSON.parse(Buffer.from(value, "base64").toString("utf8")); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +function matchLabels(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const entries = Object.entries(value).filter((entry) => typeof entry[1] === "string" && entry[1].length > 0); + return entries.length === 0 ? null : Object.fromEntries(entries); +} + +function allContainerStatuses(pod) { + const status = pod?.status || {}; + return [ + ...arrayItems(status.initContainerStatuses), + ...arrayItems(status.containerStatuses), + ...arrayItems(status.ephemeralContainerStatuses), + ]; +} + +function containerSummary(item) { + return { + name: item.name || null, + ready: item.ready === true, + restartCount: integerOrNull(item.restartCount), + waiting: compactWaiting(item.state?.waiting), + running: item.state?.running ? { startedAt: item.state.running.startedAt || null } : null, + terminated: compactTerminated(item.state?.terminated || item.lastState?.terminated), + imageID: shortImageId(item.imageID || null), + }; +} + +function conditionByType(status, type) { + return arrayItems(status?.conditions).find((item) => item?.type === type) || null; +} + +function compactCondition(value) { + if (!value) return null; + return { type: value.type || null, status: value.status || null, reason: value.reason || null, message: shortText(value.message || null), lastTransitionTime: value.lastTransitionTime || null }; +} + +function compactTerminated(value) { + if (!value) return null; + return { reason: value.reason || null, exitCode: integerOrNull(value.exitCode), message: shortText(value.message || null), startedAt: value.startedAt || null, finishedAt: value.finishedAt || null }; +} + +function compactWaiting(value) { + if (!value) return null; + return { reason: value.reason || null, message: shortText(value.message || null) }; +} + +function uniqueOrNull(values) { + const unique = [...new Set(values.filter(Boolean))]; + return unique.length === 1 ? unique[0] : null; +} + +function shortImageId(value) { + if (!value) return null; + const text = String(value); + const at = text.lastIndexOf("@"); + return at >= 0 ? text.slice(at + 1, at + 25) : shortText(text); +} + +function shortSha(value) { + if (!value) return null; + const text = String(value); + return text.length > 12 ? text.slice(0, 12) : text; +} + +function shortText(value) { + if (value === null || value === undefined) return null; + const text = String(value).replace(/\s+/gu, " ").trim(); + return text.length <= maxMessageBytes ? text : `${text.slice(0, Math.max(0, maxMessageBytes - 3))}...`; +} + +function arrayItems(value) { + return Array.isArray(value) ? value : []; +} + +function integerOrNull(value) { + return Number.isInteger(value) ? value : null; +} + +function isNotFoundError(error) { + return error instanceof KubeReadError && error.notFound === true; +} + +function isNotFoundText(value) { + return /\b404\b|not found|NotFound/u.test(String(value || "")); +} + +function positiveInt(value, name) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); + return parsed; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} diff --git a/scripts/src/cicd-branch-follower.ts b/scripts/src/cicd-branch-follower.ts index 9e997505..61855184 100644 --- a/scripts/src/cicd-branch-follower.ts +++ b/scripts/src/cicd-branch-follower.ts @@ -30,6 +30,7 @@ import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGi import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuseConfig, RUNTIME_REUSE_CONFIG_PATH, runtimeReuseService, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config"; import { prioritizedTaskRunItems } from "./cicd-taskruns"; import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown"; +import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown"; import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; import { arrayField, @@ -50,7 +51,7 @@ const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; export function cicdHelp(): unknown { return { - command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun", + command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun|job|runtime", output: "text by default; use --json, --raw, or -o json|yaml for machine output", usage: [ "bun scripts/cli.ts cicd branch-follower plan", @@ -67,6 +68,8 @@ export function cicdHelp(): unknown { "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master", "bun scripts/cli.ts cicd branch-follower status --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json", "bun scripts/cli.ts cicd branch-follower taskrun --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json", + "bun scripts/cli.ts cicd branch-follower job --follower agentrun-jd01-v02 --source-commit --job image-build --json", + "bun scripts/cli.ts cicd branch-follower runtime --follower agentrun-jd01-v02 --workload agentrun-mgr --source-commit --json", ], config: DEFAULT_CONFIG_PATH, spec: `${SPEC_REF} ${SPEC_VERSION}`, @@ -78,7 +81,7 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string const top = args[0]; if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json"); if (top !== "branch-follower") { - throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun"); + throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun|job|runtime"); } const options = parseOptions(args.slice(1)); const command = commandLabel(options); @@ -103,6 +106,10 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string return renderResult(command, await runFollowerDrillDown(registry, options), options); case "taskrun": return renderResult(command, await runTaskRunDrillDown(registry, options), options); + case "job": + return renderResult(command, await runJobDrillDown(registry, options), options); + case "runtime": + return renderResult(command, await runRuntimeDrillDown(registry, options), options); case "help": return renderMachine(command, cicdHelp(), "json"); } @@ -113,7 +120,7 @@ function parseOptions(args: string[]): ParsedOptions { if (actionToken === undefined || isHelpToken(actionToken)) { return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1)); } - if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs", "taskrun"].includes(actionToken)) { + if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs", "taskrun", "job", "runtime"].includes(actionToken)) { throw new Error(`cicd branch-follower unknown action: ${actionToken}`); } const action = actionToken as BranchFollowerAction; @@ -157,6 +164,12 @@ function parseOptions(args: string[]): ParsedOptions { options.taskRunName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg); } else if (arg === "--pipelinerun" || arg === "--pipeline-run") { options.pipelineRunName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg); + } else if (arg === "--job") { + options.jobName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg); + } else if (arg === "--source-commit" || arg === "--source-sha") { + options.sourceCommit = simpleGitSha(valueOption(rest, ++index, arg), arg); + } else if (arg === "--workload") { + options.workloadName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg); } else if (arg === "--logs-tail") { options.logsTailLines = positiveInt(valueOption(rest, ++index, arg), arg); } else if (arg === "--max-log-bytes") { @@ -195,9 +208,15 @@ function parseOptions(args: string[]): ParsedOptions { if (options.action === "taskrun" && options.taskRunName === null) { throw new Error("taskrun requires --taskrun "); } + if (options.action === "job" && options.jobName === null) { + throw new Error("job requires --job "); + } if (options.taskRunName !== null && options.followerId === null) { throw new Error("--taskrun requires --follower "); } + if ((options.action === "job" || options.action === "runtime" || options.jobName !== null || options.workloadName !== null) && options.followerId === null) { + throw new Error(`${options.action} requires --follower `); + } return options; } @@ -228,6 +247,9 @@ function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOp debugStep: null, taskRunName: null, pipelineRunName: null, + jobName: null, + sourceCommit: null, + workloadName: null, logsTailLines: null, maxLogBytes: null, output: "human", @@ -253,6 +275,11 @@ function simpleK8sObjectName(value: string, option: string): string { return value; } +function simpleGitSha(value: string, option: string): string { + if (!/^[A-Fa-f0-9]{7,64}$/u.test(value)) throw new Error(`${option} must be a git sha`); + return value; +} + function positiveInt(value: string, option: string): number { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`); @@ -814,6 +841,20 @@ async function runTaskRunDrillDown(registry: BranchFollowerRegistry, options: Pa return runBranchFollowerTaskRunDrillDown(registry, follower, options, runKubeScript); } +async function runJobDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + if (options.followerId === null) throw new Error("job requires --follower "); + const follower = registry.followers.find((item) => item.id === options.followerId); + if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`); + return runBranchFollowerJobDrillDown(registry, follower, options, runKubeScript); +} + +async function runRuntimeDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + if (options.followerId === null) throw new Error("runtime requires --follower "); + const follower = registry.followers.find((item) => item.id === options.followerId); + if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`); + return runBranchFollowerRuntimeDrillDown(registry, follower, options, runKubeScript); +} + async function decideAndMaybeTrigger( registry: BranchFollowerRegistry, follower: FollowerSpec, @@ -2299,9 +2340,11 @@ function buildFollowerTimings( const nativePayload = asOptionalRecord(live.payload); const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live); const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha); + const storedStages = live.observedSha !== null && stringOrNull(storedTimings?.sourceCommit) === live.observedSha ? storedStageTimings(storedTimings ?? null) : []; const stages = dedupeTimingStages([ ...stageTimingsFromCommand(triggerCommand), ...stageTimingsFromNativePayload(nativePayload), + ...storedStages, ]).slice(0, 24); const stageSourceCommit = stages.length > 0 ? live.observedSha : null; return { diff --git a/scripts/src/cicd-drilldown-render.ts b/scripts/src/cicd-drilldown-render.ts index 977ca738..75b2a589 100644 --- a/scripts/src/cicd-drilldown-render.ts +++ b/scripts/src/cicd-drilldown-render.ts @@ -3,6 +3,8 @@ export function renderDrillDownHuman(payload: Record): string { if (payload.action === "taskrun") return renderTaskRunHuman(payload); + if (payload.action === "job") return renderJobHuman(payload); + if (payload.action === "runtime") return renderRuntimeHuman(payload); if (payload.follower === undefined) { const followers = arrayRecords(payload.followers); return [ @@ -27,6 +29,80 @@ export function renderDrillDownHuman(payload: Record): string { ].filter((line) => line !== "").join("\n"); } +function renderJobHuman(payload: Record): string { + const result = asOptionalRecord(payload.result); + const job = asOptionalRecord(result?.job); + const query = asOptionalRecord(payload.query); + const policy = asOptionalRecord(payload.policy); + const pods = arrayRecords(result?.pods); + const logs = arrayRecords(result?.logs); + const errors = arrayRecords(result?.errors); + const command = asOptionalRecord(payload.command); + const identity = asOptionalRecord(command?.identity); + return [ + `CI/CD BRANCH-FOLLOWER JOB (${payload.ok === false ? "failed" : "ok"})`, + "", + table( + ["FOLLOWER", "ADAPTER", "STAGE", "NAMESPACE", "JOB", "STATUS", "REASON", "DURATION", "PODS"], + [[ + payload.follower, + payload.adapter ?? "-", + job?.stage ?? query?.stage ?? "-", + job?.namespace ?? query?.namespace ?? "-", + job?.name ?? query?.jobName ?? "-", + jobStatus(job), + asOptionalRecord(job?.condition)?.reason ?? result?.degradedReason ?? "-", + job?.durationSeconds ?? "-", + pods.length, + ]], + ), + pods.length === 0 ? "" : `\nPODS\n${table(["POD", "PHASE", "READY", "START", "CONTAINERS", "REASON"], pods.map(jobPodRow))}`, + logs.length === 0 ? "" : `\nLOG TAILS\n${table(["POD", "CONTAINER", "STATUS", "REASON", "LINES", "BYTES", "TIMING", "MESSAGE"], logs.map(logRow))}`, + errors.length === 0 ? "" : `\nERRORS\n${table(["POD", "CONTAINER", "REASON", "MESSAGE"], errors.map((item) => [item.pod, item.container, item.degradedReason, item.message]))}`, + command === null ? "" : `\nTARGET COMMAND\n${table(["ROUTE", "SCRIPT", "EXIT", "PARSE_ERROR"], [[identity?.route ?? "-", identity?.script ?? "-", command.exitCode ?? "-", command.parseError ?? "-"]])}`, + command?.stdoutTail ? `\nSTDOUT_TAIL\n${command.stdoutTail}` : "", + command?.stderrTail ? `\nSTDERR_TAIL\n${command.stderrTail}` : "", + "", + `policy: tailLines=${policy?.logsTailLines ?? "-"} maxLogBytes=${policy?.maxLogBytes ?? "-"} timeoutSeconds=${policy?.timeoutSeconds ?? "-"} maxContainers=${policy?.maxContainers ?? "-"}`, + "", + ].filter((line) => line !== "").join("\n"); +} + +function renderRuntimeHuman(payload: Record): string { + const result = asOptionalRecord(payload.result); + const query = asOptionalRecord(payload.query); + const policy = asOptionalRecord(payload.policy); + const workloads = arrayRecords(result?.workloads); + const pods = workloads.flatMap((workload) => arrayRecords(workload.pods).map((pod) => ({ ...pod, workload: `${workload.kind ?? "-"}/${workload.name ?? "-"}` }))).slice(0, 12); + const command = asOptionalRecord(payload.command); + const identity = asOptionalRecord(command?.identity); + return [ + `CI/CD BRANCH-FOLLOWER RUNTIME (${payload.ok === false ? "failed" : "ok"})`, + "", + table( + ["FOLLOWER", "ADAPTER", "NAMESPACE", "EXPECTED", "TARGET", "READY", "ALIGNED", "BLOCKING"], + [[ + payload.follower, + payload.adapter ?? "-", + result?.namespace ?? query?.namespace ?? "-", + shortSha(stringOrNull(result?.expectedSha)), + shortSha(stringOrNull(result?.targetSha)), + result?.ready ?? "-", + result?.aligned ?? "-", + result?.blockingReason ?? "-", + ]], + ), + workloads.length === 0 ? "" : `\nWORKLOADS\n${table(["KIND", "NAME", "READY", "ALIGNED", "REPLICAS", "UPDATED", "SOURCE", "BLOCKING"], workloads.map(runtimeWorkloadRow))}`, + pods.length === 0 ? "" : `\nPODS\n${table(["WORKLOAD", "POD", "PHASE", "READY", "START", "SOURCE", "CONTAINERS"], pods.map(runtimePodRow))}`, + command === null ? "" : `\nTARGET COMMAND\n${table(["ROUTE", "SCRIPT", "EXIT", "PARSE_ERROR"], [[identity?.route ?? "-", identity?.script ?? "-", command.exitCode ?? "-", command.parseError ?? "-"]])}`, + command?.stdoutTail ? `\nSTDOUT_TAIL\n${command.stdoutTail}` : "", + command?.stderrTail ? `\nSTDERR_TAIL\n${command.stderrTail}` : "", + "", + `policy: timeoutSeconds=${policy?.timeoutSeconds ?? "-"} maxContainers=${policy?.maxContainers ?? "-"}`, + "", + ].filter((line) => line !== "").join("\n"); +} + function renderTaskRunHuman(payload: Record): string { const result = asOptionalRecord(payload.result); const taskRun = asOptionalRecord(result?.taskRun); @@ -69,6 +145,52 @@ function renderTaskRunHuman(payload: Record): string { ].filter((line) => line !== "").join("\n"); } +function jobStatus(job: Record | null): string { + if (job === null) return "-"; + if (job.completed === true) return "completed"; + if (job.failedState === true) return "failed"; + if (numberOrNull(job.active) !== null && numberOrNull(job.active)! > 0) return "active"; + return stringOrNull(asOptionalRecord(job.condition)?.type) ?? "-"; +} + +function jobPodRow(item: Record): unknown[] { + return [ + item.name, + item.phase, + item.ready, + item.startTime ?? item.createdAt ?? "-", + item.containerCount, + item.reason ?? "-", + ]; +} + +function runtimeWorkloadRow(item: Record): unknown[] { + const sourceCommit = asOptionalRecord(item.sourceCommit); + return [ + item.kind, + item.name, + item.ready, + item.aligned, + `${item.readyReplicas ?? "-"}/${item.replicas ?? "-"}`, + item.updatedReplicas ?? "-", + shortSha(stringOrNull(sourceCommit?.value)), + item.blockingReason ?? "-", + ]; +} + +function runtimePodRow(item: Record): unknown[] { + const sourceCommit = asOptionalRecord(item.sourceCommit); + return [ + item.workload, + item.name, + item.phase, + item.ready, + item.startTime ?? item.createdAt ?? "-", + shortSha(stringOrNull(sourceCommit?.value)), + arrayRecords(item.containers).length, + ]; +} + function logRow(item: Record): unknown[] { return [ item.pod, diff --git a/scripts/src/cicd-job-runtime-drilldown.ts b/scripts/src/cicd-job-runtime-drilldown.ts new file mode 100644 index 00000000..58d5f4d8 --- /dev/null +++ b/scripts/src/cicd-job-runtime-drilldown.ts @@ -0,0 +1,237 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower job/runtime drill-down. +// Responsibility: follower-scoped read-only K8s Job and runtime workload visibility. +import type { CommandResult } from "./command"; +import { resolveAgentRunLaneTarget } from "./agentrun-lanes"; +import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types"; +import { nativeCicdScriptLoadShell } from "./cicd-native-bundle"; +import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; +import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe"; +import { redactText, shQuote } from "./platform-infra-ops-library"; + +type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult; + +export async function runBranchFollowerJobDrillDown( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + options: ParsedOptions, + runKubeScript: KubeScriptRunner, +): Promise> { + const query = options.jobName; + if (query === null) throw new Error("job drill-down requires --job "); + if (follower.drillDown === null) return missingPolicyPayload("job", follower, registry, options, query); + const sourceCommit = options.sourceCommit ?? null; + const target = resolveJobTarget(registry, follower, query, sourceCommit); + const policy = drillDownPolicy(follower, options); + if (target === null) { + return { + ok: false, + action: "job", + follower: follower.id, + adapter: follower.adapter, + degradedReason: "job-target-unresolved", + message: `job stage ${query} cannot be resolved for ${follower.id}; pass a concrete Kubernetes Job name`, + query: { job: query, sourceCommit }, + policy, + result: null, + statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` }, + }; + } + const script = [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["k8s-job-drilldown.mjs"]), + `JOB_NAMESPACE=${shQuote(target.namespace)}`, + `JOB_NAME=${shQuote(target.jobName)}`, + `STAGE_NAME=${shQuote(target.stage)}`, + `SOURCE_COMMIT=${shQuote(sourceCommit ?? "")}`, + `LOGS_TAIL_LINES=${policy.logsTailLines}`, + `MAX_LOG_BYTES=${policy.maxLogBytes}`, + `MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`, + `MAX_CONTAINERS=${policy.maxContainers}`, + "export JOB_NAMESPACE JOB_NAME STAGE_NAME SOURCE_COMMIT LOGS_TAIL_LINES MAX_LOG_BYTES MAX_MESSAGE_BYTES MAX_CONTAINERS", + "node \"$tmpdir/k8s-job-drilldown.mjs\"", + ].join("\n"); + const startedAt = Date.now(); + const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000); + return drillDownPayload("job", follower, registry, options, policy, target, result, Date.now() - startedAt); +} + +export async function runBranchFollowerRuntimeDrillDown( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + options: ParsedOptions, + runKubeScript: KubeScriptRunner, +): Promise> { + if (follower.nativeStatus.runtime === null) throw new Error(`follower ${follower.id} has no runtime native status config`); + if (follower.drillDown === null) return missingPolicyPayload("runtime", follower, registry, options, options.workloadName); + const policy = drillDownPolicy(follower, options); + const workloads = follower.nativeStatus.runtime.workloads + .filter((item) => options.workloadName === null || item.name === options.workloadName) + .map((item) => ({ + kind: item.kind, + name: item.name, + sourceCommit: item.sourceCommit, + })); + if (workloads.length === 0) throw new Error(`unknown runtime workload ${options.workloadName ?? "-"}`); + const workloadsB64 = Buffer.from(JSON.stringify(workloads), "utf8").toString("base64"); + const script = [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["runtime-drilldown.mjs"]), + `RUNTIME_NAMESPACE=${shQuote(follower.nativeStatus.runtime.namespace)}`, + `EXPECTED_SHA=${shQuote(options.sourceCommit ?? "")}`, + `WORKLOADS_B64=${shQuote(workloadsB64)}`, + `MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`, + `MAX_CONTAINERS=${policy.maxContainers}`, + "export RUNTIME_NAMESPACE EXPECTED_SHA WORKLOADS_B64 MAX_MESSAGE_BYTES MAX_CONTAINERS", + "node \"$tmpdir/runtime-drilldown.mjs\"", + ].join("\n"); + const startedAt = Date.now(); + const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000); + return drillDownPayload("runtime", follower, registry, options, policy, { + namespace: follower.nativeStatus.runtime.namespace, + workload: options.workloadName, + sourceCommit: options.sourceCommit ?? null, + }, result, Date.now() - startedAt); +} + +function drillDownPolicy(follower: FollowerSpec, options: ParsedOptions): Record { + const drillDown = follower.drillDown; + if (drillDown === null) throw new Error(`follower ${follower.id} registry is missing drillDown policy`); + return { + timeoutSeconds: options.timeoutSeconds ?? drillDown.taskRunTimeoutSeconds, + logsTailLines: options.logsTailLines ?? drillDown.logsTailLines, + maxLogBytes: options.maxLogBytes ?? drillDown.maxLogBytes, + maxMessageBytes: drillDown.maxMessageBytes, + maxContainers: drillDown.maxContainers, + }; +} + +function drillDownPayload( + action: "job" | "runtime", + follower: FollowerSpec, + registry: BranchFollowerRegistry, + options: ParsedOptions, + policy: Record, + target: Record, + command: CommandResult, + elapsedMs: number, +): Record { + const parsedResult = command.exitCode === 0 ? parseJsonObject(command.stdout) : { value: null, error: "target-command-failed" }; + const parsed = parsedResult.value; + const includeTails = command.exitCode !== 0 || parsed === null; + return { + ok: command.exitCode === 0 && parsed !== null && parsed.ok !== false, + action, + follower: follower.id, + adapter: follower.adapter, + statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + query: target, + policy, + result: parsed, + command: { + identity: { + route: options.inCluster ? "in-cluster" : registry.controller.kubeRoute, + script: action === "job" ? "scripts/native/cicd/k8s-job-drilldown.mjs" : "scripts/native/cicd/runtime-drilldown.mjs", + }, + exitCode: command.exitCode, + timedOut: command.timedOut, + elapsedMs, + parseError: parsedResult.error, + stdoutTail: includeTails ? redactText(tailText(command.stdout, 1600)) : "", + stderrTail: includeTails ? redactText(tailText(command.stderr, 1200)) : "", + }, + next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` }, + }; +} + +function missingPolicyPayload(action: string, follower: FollowerSpec, registry: BranchFollowerRegistry, options: ParsedOptions, query: string | null): Record { + return { + ok: false, + action, + follower: follower.id, + adapter: follower.adapter, + degradedReason: "drilldown-policy-missing", + message: `follower ${follower.id} registry is missing drillDown policy; apply the current config before drill-down`, + query, + policy: null, + result: null, + statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + next: { + apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", + status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, + }, + }; +} + +function resolveJobTarget(registry: BranchFollowerRegistry, follower: FollowerSpec, query: string, sourceCommit: string | null): { namespace: string; jobName: string; stage: string } | null { + if (!isStageAlias(query)) return { namespace: follower.target.namespace, jobName: query, stage: "explicit-job" }; + if (sourceCommit === null) return null; + if (query === "git-mirror-sync" || query === "git-mirror-flush") { + const action = query === "git-mirror-sync" ? "sync" : "flush"; + const target = gitMirrorJobTarget(follower, sourceCommit, action); + return target === null ? null : { ...target, stage: query }; + } + if (query === "control-plane-refresh" && follower.adapter === "hwlab-node-runtime") { + return { namespace: registry.controller.namespace, jobName: nativeCapabilityJobName(follower.id, "control-plane-refresh", sourceCommit), stage: query }; + } + if (follower.adapter === "agentrun-yaml-lane" && (query === "image-build" || query === "gitops-publish")) { + const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); + const prefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; + if (query === "image-build") return { namespace: spec.ci.namespace, jobName: `${prefix}-build-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query }; + return { namespace: spec.gitMirror.namespace, jobName: `${prefix}-gitops-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query }; + } + return null; +} + +function isStageAlias(value: string): boolean { + return ["git-mirror-sync", "git-mirror-flush", "control-plane-refresh", "image-build", "gitops-publish"].includes(value); +} + +function gitMirrorJobTarget(follower: FollowerSpec, sourceCommit: string, action: "sync" | "flush"): { namespace: string; jobName: string } | null { + const jobName = nativeCapabilityJobName(follower.id, action, sourceCommit); + if (follower.adapter === "hwlab-node-runtime") { + const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); + return { namespace: nodeRuntimeGitMirrorTarget(spec).namespace, jobName }; + } + if (follower.adapter === "agentrun-yaml-lane") { + const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); + return { namespace: spec.gitMirror.namespace, jobName }; + } + return null; +} + +function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { + const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`; + return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); +} + +function safeK8sNameSegment(value: string): string { + const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, ""); + return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, ""); +} + +function parseJsonObject(text: string): { value: Record | null; error: string | null } { + const trimmed = text.trim(); + if (trimmed.length === 0) return { value: null, error: "empty-stdout" }; + try { + const parsed = JSON.parse(trimmed) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? { value: parsed as Record, error: null } + : { value: null, error: "stdout-json-not-object" }; + } catch (error) { + return { value: null, error: `stdout-json-parse-failed: ${error instanceof Error ? error.message : String(error)}` }; + } +} + +function tailText(text: string, maxChars: number): string { + return text.length <= maxChars ? text : text.slice(text.length - maxChars); +} diff --git a/scripts/src/cicd-render.ts b/scripts/src/cicd-render.ts index 43fae256..fb5831a4 100644 --- a/scripts/src/cicd-render.ts +++ b/scripts/src/cicd-render.ts @@ -27,7 +27,7 @@ function renderHuman(command: string, payload: Record, options: if (command.endsWith(" run-once")) return renderRunOnceHuman(payload); if (command.endsWith(" debug-step")) return renderDebugStepHuman(payload); if (command.endsWith(" cleanup-state")) return renderCleanupStateHuman(payload); - if (command.endsWith(" events") || command.endsWith(" logs") || command.endsWith(" taskrun")) return renderDrillDownHuman(payload); + if (command.endsWith(" events") || command.endsWith(" logs") || command.endsWith(" taskrun") || command.endsWith(" job") || command.endsWith(" runtime")) return renderDrillDownHuman(payload); return `${JSON.stringify(payload, null, 2)}\n`; } diff --git a/scripts/src/cicd-types.ts b/scripts/src/cicd-types.ts index b26492c1..b23ccdea 100644 --- a/scripts/src/cicd-types.ts +++ b/scripts/src/cicd-types.ts @@ -2,7 +2,7 @@ // Responsibility: type contracts shared by branch follower entry, controller render, and native K8s helpers. export type OutputMode = "human" | "json" | "yaml"; -export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs" | "taskrun"; +export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs" | "taskrun" | "job" | "runtime"; export type BranchFollowerDebugStep = "state-read" | "controller-source" | "status-read" | "decide" | "state-write"; export type BranchFollowerPhase = | "Observed" @@ -33,6 +33,9 @@ export interface ParsedOptions { debugStep: BranchFollowerDebugStep | null; taskRunName: string | null; pipelineRunName: string | null; + jobName: string | null; + sourceCommit: string | null; + workloadName: string | null; logsTailLines: number | null; maxLogBytes: number | null; output: OutputMode;