diff --git a/config/cicd-branch-followers.yaml b/config/cicd-branch-followers.yaml index cbe5c4e9..6d648ebd 100644 --- a/config/cicd-branch-followers.yaml +++ b/config/cicd-branch-followers.yaml @@ -126,6 +126,12 @@ followers: podLabels: ["hwlab.pikastech.local/source-commit", "hwlab.pikastech.local/gitops-render-source-commit"] podAnnotations: ["hwlab.pikastech.local/source-commit", "hwlab.pikastech.local/gitops-render-source-commit", "hwlab.pikastech.local/boot-commit"] env: ["HWLAB_COMMIT_ID"] + drillDown: + taskRunTimeoutSeconds: 20 + logsTailLines: 120 + maxLogBytes: 24000 + maxMessageBytes: 1200 + maxContainers: 6 closeout: checks: ["sourceSnapshot", "pipelineRun", "gitMirrorPostFlush", "gitops", "argo", "runtime", "publicHealth"] @@ -196,6 +202,12 @@ followers: annotations: ["agentrun.pikastech.local/source-commit"] podAnnotations: ["agentrun.pikastech.local/source-commit"] env: ["AGENTRUN_SOURCE_COMMIT", "AGENTRUN_BOOT_COMMIT"] + drillDown: + taskRunTimeoutSeconds: 20 + logsTailLines: 120 + maxLogBytes: 24000 + maxMessageBytes: 1200 + maxContainers: 6 closeout: checks: ["sourceSnapshot", "pipelineRun", "gitops", "argo", "manager", "runtimeHealth"] @@ -264,5 +276,11 @@ followers: annotations: ["unidesk.ai/source-commit", "hwlab.pikastech.local/source-commit"] podAnnotations: ["unidesk.ai/source-commit", "hwlab.pikastech.local/source-commit"] env: ["UNIDESK_SOURCE_COMMIT", "WEB_PROBE_SENTINEL_SOURCE_COMMIT"] + drillDown: + taskRunTimeoutSeconds: 20 + logsTailLines: 120 + maxLogBytes: 24000 + maxMessageBytes: 1200 + maxContainers: 6 closeout: checks: ["sourceMirror", "imageRegistry", "gitops", "argo", "runtimeHealthEndpoint", "dashboard", "report"] diff --git a/scripts/native/cicd/taskrun-drilldown.mjs b/scripts/native/cicd/taskrun-drilldown.mjs new file mode 100644 index 00000000..6d8c7786 --- /dev/null +++ b/scripts/native/cicd/taskrun-drilldown.mjs @@ -0,0 +1,399 @@ +import { existsSync, readFileSync } from "node:fs"; +import https from "node:https"; +import { spawnSync } from "node:child_process"; + +const namespace = requiredEnv("TEKTON_NAMESPACE"); +const query = requiredEnv("TASKRUN_QUERY"); +const pipelineRunName = process.env.PIPELINE_RUN_NAME || ""; +const pipelineRunPrefix = process.env.PIPELINE_RUN_PREFIX || ""; +const logsTailLines = positiveInt(process.env.LOGS_TAIL_LINES, "LOGS_TAIL_LINES"); +const maxLogBytes = positiveInt(process.env.MAX_LOG_BYTES, "MAX_LOG_BYTES"); +const maxMessageBytes = positiveInt(process.env.MAX_MESSAGE_BYTES, "MAX_MESSAGE_BYTES"); +const maxContainers = positiveInt(process.env.MAX_CONTAINERS, "MAX_CONTAINERS"); + +const useServiceAccount = Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) + && existsSync("/var/run/secrets/kubernetes.io/serviceaccount/token") + && existsSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + +class KubeReadError extends Error { + constructor(reason, message, details = {}) { + super(shortText(message) || reason); + this.name = "KubeReadError"; + this.reason = reason; + this.statusCode = details.statusCode ?? null; + this.notFound = details.notFound === true; + this.path = details.path || null; + } +} + +main().catch((error) => { + process.stderr.write(error?.message || String(error)); + process.exit(1); +}); + +async function main() { + const resolution = await resolveTaskRun(); + if (resolution.taskRun === null) { + console.log(JSON.stringify({ + ok: false, + degradedReason: "taskrun-not-found", + query: { namespace, taskRun: query, pipelineRun: pipelineRunName || null, pipelineRunPrefix: pipelineRunPrefix || null }, + candidates: resolution.candidates, + statusAuthority: useServiceAccount ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + })); + return; + } + const taskRun = resolution.taskRun; + const metadata = taskRun.metadata || {}; + const status = taskRun.status || {}; + const condition = conditionByType(status, "Succeeded"); + const podName = status.podName || await resolvePodName(taskRun); + const pod = podName ? await getJson(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods/${encodeURIComponent(podName)}`, false) : null; + const podStatuses = podStatusRows(pod); + const taskRunSteps = stepRows(status); + const containers = mergeContainerRows(taskRunSteps, podStatuses).slice(0, maxContainers); + const logContainers = selectLogContainers(containers); + const logs = []; + const perContainerBytes = Math.max(1, Math.floor(maxLogBytes / Math.max(1, logContainers.length))); + for (const container of logContainers) { + const name = container.containerName || container.name; + if (!podName || !name) continue; + const read = await readPodLog(podName, name, logsTailLines, perContainerBytes); + const text = read.tail || ""; + logs.push({ + ok: read.ok, + degradedReason: read.degradedReason, + message: read.message, + pod: podName, + container: name, + lineCount: text.length === 0 ? 0 : text.split(/\r?\n/u).filter((line) => line.length > 0).length, + bytes: Buffer.byteLength(text, "utf8"), + tail: text, + nodeCicdTiming: lastNodeCicdTiming(text), + }); + } + const timing = lastTiming(logs); + const logFailures = logs.filter((item) => item.ok === false); + console.log(JSON.stringify({ + ok: logFailures.length === 0, + degradedReason: logFailures.length === 0 ? null : "log-read-failed", + errors: logFailures.map((item) => ({ + pod: item.pod, + container: item.container, + degradedReason: item.degradedReason, + message: item.message, + })), + taskRun: { + name: metadata.name || null, + namespace: metadata.namespace || namespace, + pipelineRun: metadata.labels?.["tekton.dev/pipelineRun"] || null, + pipelineTask: metadata.labels?.["tekton.dev/pipelineTask"] || metadata.labels?.["tekton.dev/task"] || null, + podName, + condition: { + status: condition?.status || null, + reason: condition?.reason || null, + message: shortText(condition?.message || null), + }, + startTime: status.startTime || null, + completionTime: status.completionTime || null, + durationSeconds: durationSeconds(status.startTime, status.completionTime), + }, + pod: pod === null ? null : { + name: pod.metadata?.name || podName, + phase: pod.status?.phase || null, + containerCount: podStatuses.length, + initContainerCount: Array.isArray(pod.status?.initContainerStatuses) ? pod.status.initContainerStatuses.length : 0, + startTime: pod.status?.startTime || null, + }, + containers, + logs, + nodeCicdTiming: timing, + resolution: { + mode: resolution.mode, + candidates: resolution.candidates, + }, + statusAuthority: useServiceAccount ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + })); +} + +async function resolveTaskRun() { + const direct = await getJson(`/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/taskruns/${encodeURIComponent(query)}`, false); + if (direct !== null) return { mode: "direct-name", taskRun: direct, candidates: [] }; + const selectors = []; + if (pipelineRunName) selectors.push(`tekton.dev/pipelineRun=${pipelineRunName},tekton.dev/pipelineTask=${query}`); + selectors.push(`tekton.dev/pipelineTask=${query}`); + selectors.push(`tekton.dev/task=${query}`); + for (const selector of selectors) { + const list = await getJson(`/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/taskruns?labelSelector=${encodeURIComponent(selector)}`, false); + const candidates = taskRunCandidates(list); + const filtered = candidates.filter((item) => pipelineRunPrefix.length === 0 || String(item.pipelineRun || "").startsWith(pipelineRunPrefix)); + const picked = filtered[0] || null; + if (picked !== null) { + const item = await getJson(`/apis/tekton.dev/v1/namespaces/${encodeURIComponent(namespace)}/taskruns/${encodeURIComponent(picked.name)}`, true); + return { mode: "pipeline-task-label", taskRun: item, candidates: filtered.slice(0, 8) }; + } + if (candidates.length > 0) return { mode: "pipeline-task-label-no-prefix-match", taskRun: null, candidates: candidates.slice(0, 8) }; + } + return { mode: "not-found", taskRun: null, candidates: [] }; +} + +async function resolvePodName(taskRun) { + const uid = taskRun?.metadata?.uid || ""; + const name = taskRun?.metadata?.name || query; + const selectors = uid ? [`tekton.dev/taskRunUID=${uid}`, `tekton.dev/taskRun=${name}`] : [`tekton.dev/taskRun=${name}`]; + for (const selector of selectors) { + const list = await getJson(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${encodeURIComponent(selector)}`, false); + const items = Array.isArray(list?.items) ? list.items : []; + const pod = items[0]; + if (pod?.metadata?.name) return pod.metadata.name; + } + return null; +} + +function taskRunCandidates(list) { + const items = Array.isArray(list?.items) ? list.items : []; + return items.map((item) => ({ + name: item?.metadata?.name || null, + pipelineRun: item?.metadata?.labels?.["tekton.dev/pipelineRun"] || null, + pipelineTask: item?.metadata?.labels?.["tekton.dev/pipelineTask"] || item?.metadata?.labels?.["tekton.dev/task"] || null, + startTime: item?.status?.startTime || item?.metadata?.creationTimestamp || null, + status: conditionByType(item?.status || {}, "Succeeded")?.status || null, + reason: conditionByType(item?.status || {}, "Succeeded")?.reason || null, + })).filter((item) => item.name !== null).sort((left, right) => String(right.startTime || "").localeCompare(String(left.startTime || ""))); +} + +function stepRows(status) { + const steps = Array.isArray(status?.steps) ? status.steps : []; + return steps.map((step) => ({ + source: "taskrun-step", + name: step.name || null, + containerName: step.container || (step.name ? `step-${step.name}` : null), + terminated: compactTerminated(step.terminated), + waiting: compactWaiting(step.waiting), + running: step.running ? { startedAt: step.running.startedAt || null } : null, + })); +} + +function podStatusRows(pod) { + const status = pod?.status || {}; + return [ + ...statusArray(status.initContainerStatuses, "pod-init-container"), + ...statusArray(status.containerStatuses, "pod-container"), + ...statusArray(status.ephemeralContainerStatuses, "pod-ephemeral-container"), + ]; +} + +function statusArray(value, source) { + return (Array.isArray(value) ? value : []).map((item) => ({ + source, + name: item.name || null, + containerName: item.name || null, + ready: item.ready === true, + restartCount: Number.isInteger(item.restartCount) ? item.restartCount : null, + terminated: compactTerminated(item.state?.terminated || item.lastState?.terminated), + waiting: compactWaiting(item.state?.waiting), + running: item.state?.running ? { startedAt: item.state.running.startedAt || null } : null, + })); +} + +function mergeContainerRows(steps, podStatuses) { + const rows = []; + const byName = new Map(podStatuses.map((item) => [item.containerName, item])); + for (const step of steps) rows.push({ ...step, ...(byName.get(step.containerName) || {}) }); + const seen = new Set(rows.map((item) => item.containerName)); + for (const item of podStatuses) { + if (!seen.has(item.containerName)) rows.push(item); + } + return rows.map((item) => ({ + source: item.source || null, + name: item.name || null, + containerName: item.containerName || null, + ready: item.ready === undefined ? null : item.ready, + restartCount: Number.isInteger(item.restartCount) ? item.restartCount : null, + terminated: item.terminated || null, + waiting: item.waiting || null, + running: item.running || null, + })); +} + +function selectLogContainers(containers) { + const failed = containers.filter((item) => item.terminated?.exitCode !== null && item.terminated?.exitCode !== 0); + const waiting = containers.filter((item) => item.waiting !== null); + const step = containers.filter((item) => String(item.containerName || "").startsWith("step-")); + const selected = []; + for (const item of [...failed, ...waiting, ...step, ...containers]) { + if (selected.some((existing) => existing.containerName === item.containerName)) continue; + selected.push(item); + if (selected.length >= maxContainers) break; + } + return selected; +} + +async function readPodLog(podName, container, tailLines, limitBytes) { + const path = `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods/${encodeURIComponent(podName)}/log?container=${encodeURIComponent(container)}&tailLines=${tailLines}&limitBytes=${limitBytes}`; + try { + const text = tailBytes(await getText(path, false), limitBytes); + return { ok: true, degradedReason: null, message: null, tail: text }; + } catch (error) { + return { + ok: false, + degradedReason: isNotFoundError(error) ? "log-not-found" : error instanceof KubeReadError ? error.reason : "log-read-failed", + message: shortText(error?.message || String(error)), + tail: "", + }; + } +} + +async function getJson(path, required) { + let text = ""; + try { + text = await getText(path, required); + } catch (error) { + if (!required && isNotFoundError(error)) return null; + throw error; + } + try { + return JSON.parse(text); + } catch (error) { + throw new KubeReadError("invalid-json", `kube api returned non-JSON for ${path}: ${error?.message || String(error)}`, { path }); + } +} + +async function getText(path, required) { + if (useServiceAccount) return kubeApiGet(path, required); + const result = spawnSync("kubectl", ["get", "--raw", path], { encoding: "utf8", maxBuffer: Math.max(maxLogBytes * 2, 1024 * 1024) }); + if (result.error) { + throw new KubeReadError("transport-error", `kubectl get --raw transport error for ${path}: ${result.error.message}`, { path }); + } + if (result.status === 0) return result.stdout; + const body = result.stderr || result.stdout || `kubectl get --raw failed with exit ${result.status}`; + throw new KubeReadError(isNotFoundText(body) ? "not-found" : "kube-api-error", `kubectl get --raw failed for ${path}: ${body}`, { + path, + notFound: isNotFoundText(body), + }); +} + +function kubeApiGet(path, required) { + const host = process.env.KUBERNETES_SERVICE_HOST; + const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443"); + const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim(); + const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); + return new Promise((resolve, reject) => { + const req = https.request({ host, port, path, method: "GET", ca, headers: { authorization: `Bearer ${token}` } }, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { body += chunk; }); + res.on("end", () => { + const code = res.statusCode || 0; + if (code >= 200 && code < 300) resolve(body); + else reject(new KubeReadError(code === 404 || isNotFoundText(body) ? "not-found" : "kube-api-error", `kube api GET ${path} status ${code}: ${body || "-"}`, { + path, + statusCode: code, + notFound: code === 404 || isNotFoundText(body), + })); + }); + }); + req.on("error", (error) => reject(new KubeReadError("transport-error", `kube api GET ${path} transport error: ${error?.message || String(error)}`, { path }))); + req.end(); + }); +} + +function isNotFoundError(error) { + return error instanceof KubeReadError && error.notFound === true; +} + +function isNotFoundText(value) { + return /\b404\b|not found|NotFound/u.test(String(value || "")); +} + +function compactTerminated(value) { + if (!value) return null; + return { + reason: value.reason || null, + exitCode: Number.isInteger(value.exitCode) ? value.exitCode : null, + message: shortText(value.message || null), + startedAt: value.startedAt || null, + finishedAt: value.finishedAt || null, + }; +} + +function compactWaiting(value) { + if (!value) return null; + return { + reason: value.reason || null, + message: shortText(value.message || null), + }; +} + +function lastNodeCicdTiming(text) { + const lines = text.split(/\r?\n/u).filter((line) => line.includes("node-cicd-timing")); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const parsed = parseTimingLine(lines[index]); + if (parsed !== null) return parsed; + } + return null; +} + +function parseTimingLine(line) { + const candidates = [line.trim()]; + const brace = line.indexOf("{"); + if (brace >= 0) candidates.push(line.slice(brace).trim()); + for (const candidate of candidates) { + try { + const parsed = JSON.parse(candidate); + if (parsed && typeof parsed === "object" && JSON.stringify(parsed).includes("node-cicd-timing")) return parsed; + } catch { + // Keep trying the next bounded candidate. + } + } + return { rawTail: shortText(line) }; +} + +function lastTiming(logs) { + for (let index = logs.length - 1; index >= 0; index -= 1) { + if (logs[index].nodeCicdTiming !== null) return logs[index].nodeCicdTiming; + } + return null; +} + +function conditionByType(status, type) { + const conditions = Array.isArray(status?.conditions) ? status.conditions : []; + return conditions.find((item) => item?.type === type) || null; +} + +function timestampMs(value) { + const parsed = Date.parse(String(value || "")); + return Number.isFinite(parsed) ? parsed : null; +} + +function durationSeconds(start, end) { + const s = timestampMs(start); + const e = timestampMs(end); + return s === null || e === null || e < s ? null : Math.round((e - s) / 1000); +} + +function shortText(value) { + if (value === null || value === undefined) return null; + const text = String(value).replace(/\s+/gu, " ").trim(); + return text.length <= maxMessageBytes ? text : `${text.slice(0, maxMessageBytes - 3)}...`; +} + +function tailBytes(value, maxBytes) { + const buffer = Buffer.from(value, "utf8"); + if (buffer.length <= maxBytes) return value; + return buffer.subarray(buffer.length - maxBytes).toString("utf8"); +} + +function positiveInt(value, name) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${name} must be a positive integer`); + return parsed; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} diff --git a/scripts/src/cicd-branch-follower.ts b/scripts/src/cicd-branch-follower.ts index 5a7ee252..31fcd52d 100644 --- a/scripts/src/cicd-branch-follower.ts +++ b/scripts/src/cicd-branch-follower.ts @@ -29,6 +29,7 @@ import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGitMirrorRequired, nativeGitMirrorSummary, nativePipelineRunSummary, nativeRuntimeSummary, pipelineRunSucceeded, runtimeTargetShaFromWorkloads, runtimeWorkloadsReady } from "./cicd-native-summary"; import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuseConfig, RUNTIME_REUSE_CONFIG_PATH, runtimeReuseService, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config"; import { prioritizedTaskRunItems } from "./cicd-taskruns"; +import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown"; import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; import { arrayField, @@ -49,7 +50,7 @@ const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; export function cicdHelp(): unknown { return { - command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs", + command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun", output: "text by default; use --json, --raw, or -o json|yaml for machine output", usage: [ "bun scripts/cli.ts cicd branch-follower plan", @@ -64,6 +65,8 @@ export function cicdHelp(): unknown { "bun scripts/cli.ts cicd branch-follower cleanup-state --follower web-probe-sentinel-master --confirm", "bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02", "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master", + "bun scripts/cli.ts cicd branch-follower status --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json", + "bun scripts/cli.ts cicd branch-follower taskrun --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json", ], config: DEFAULT_CONFIG_PATH, spec: `${SPEC_REF} ${SPEC_VERSION}`, @@ -75,7 +78,7 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string const top = args[0]; if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json"); if (top !== "branch-follower") { - throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs"); + throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun"); } const options = parseOptions(args.slice(1)); const command = commandLabel(options); @@ -87,6 +90,7 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string case "apply": return renderResult(command, await applyController(registry, options), options); case "status": + if (options.taskRunName !== null) return renderResult(command, await runTaskRunDrillDown(registry, options), options); return renderResult(command, await buildStatus(registry, options), options); case "run-once": return renderResult(command, await runOnce(registry, options), options); @@ -97,6 +101,8 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string case "events": case "logs": return renderResult(command, await runFollowerDrillDown(registry, options), options); + case "taskrun": + return renderResult(command, await runTaskRunDrillDown(registry, options), options); case "help": return renderMachine(command, cicdHelp(), "json"); } @@ -107,7 +113,7 @@ function parseOptions(args: string[]): ParsedOptions { if (actionToken === undefined || isHelpToken(actionToken)) { return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1)); } - if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs"].includes(actionToken)) { + if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs", "taskrun"].includes(actionToken)) { throw new Error(`cicd branch-follower unknown action: ${actionToken}`); } const action = actionToken as BranchFollowerAction; @@ -147,6 +153,14 @@ function parseOptions(args: string[]): ParsedOptions { options.recordState = true; } else if (arg === "--step") { options.debugStep = debugStepOption(valueOption(rest, ++index, arg)); + } else if (arg === "--taskrun" || arg === "--task-run") { + options.taskRunName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg); + } else if (arg === "--pipelinerun" || arg === "--pipeline-run") { + options.pipelineRunName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg); + } else if (arg === "--logs-tail") { + options.logsTailLines = positiveInt(valueOption(rest, ++index, arg), arg); + } else if (arg === "--max-log-bytes") { + options.maxLogBytes = positiveInt(valueOption(rest, ++index, arg), arg); } else if (arg === "-o" || arg === "--output") { const value = valueOption(rest, ++index, arg); if (value !== "json" && value !== "yaml" && value !== "wide" && value !== "text") throw new Error(`${arg} must be json, yaml, wide, or text`); @@ -178,6 +192,12 @@ function parseOptions(args: string[]): ParsedOptions { if (options.action === "debug-step" && options.followerId === null) { throw new Error("debug-step requires --follower "); } + if (options.action === "taskrun" && options.taskRunName === null) { + throw new Error("taskrun requires --taskrun "); + } + if (options.taskRunName !== null && options.followerId === null) { + throw new Error("--taskrun requires --follower "); + } return options; } @@ -206,6 +226,10 @@ function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOp raw: false, recordState: false, debugStep: null, + taskRunName: null, + pipelineRunName: null, + logsTailLines: null, + maxLogBytes: null, output: "human", limit: 20, tailBytes: 12000, @@ -224,6 +248,11 @@ function simpleId(value: string, option: string): string { return value; } +function simpleK8sObjectName(value: string, option: string): string { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${option} must be a Kubernetes object or task id`); + return value; +} + function positiveInt(value: string, option: string): number { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`); @@ -337,6 +366,7 @@ function parseFollower(root: Record, index: number): FollowerSp const budgets = recordField(root, "budgets", label); const commands = recordField(root, "commands", label); const nativeStatus = recordField(root, "nativeStatus", label); + const drillDown = recordField(root, "drillDown", label); const closeout = recordField(root, "closeout", label); const configRefs = stringMap(recordField(target, "configRefs", `${label}.target`), `${label}.target.configRefs`); return { @@ -376,6 +406,7 @@ function parseFollower(root: Record, index: number): FollowerSp logs: parseCommand(recordField(commands, "logs", `${label}.commands`), `${label}.commands.logs`), }, nativeStatus: parseNativeStatus(nativeStatus, `${label}.nativeStatus`), + drillDown: parseDrillDown(drillDown, `${label}.drillDown`), closeoutChecks: stringArrayField(closeout, "checks", `${label}.closeout`), }; } @@ -438,6 +469,16 @@ function parseCommand(root: Record, label: string): CommandSpec }; } +function parseDrillDown(root: Record, label: string): FollowerSpec["drillDown"] { + return { + taskRunTimeoutSeconds: integerField(root, "taskRunTimeoutSeconds", label), + logsTailLines: integerField(root, "logsTailLines", label), + maxLogBytes: integerField(root, "maxLogBytes", label), + maxMessageBytes: integerField(root, "maxMessageBytes", label), + maxContainers: integerField(root, "maxContainers", label), + }; +} + function optionalStringArrayField(root: Record, key: string, label: string): string[] { return root[key] === undefined ? [] : stringArrayField(root, key, label); } @@ -477,6 +518,7 @@ function buildPlan(registry: BranchFollowerRegistry, options: ParsedOptions): Re }, target: follower.target, budgets: follower.budgets, + drillDown: follower.drillDown, commands: redactCommands(follower), nativeStatus: nativeStatusPlan(follower.nativeStatus), closeoutChecks: follower.closeoutChecks, @@ -765,6 +807,13 @@ async function runFollowerDrillDown(registry: BranchFollowerRegistry, options: P }; } +async function runTaskRunDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + if (options.followerId === null) throw new Error("--taskrun requires --follower "); + const follower = registry.followers.find((item) => item.id === options.followerId); + if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`); + return runBranchFollowerTaskRunDrillDown(registry, follower, options, runKubeScript); +} + async function decideAndMaybeTrigger( registry: BranchFollowerRegistry, follower: FollowerSpec, @@ -2776,6 +2825,7 @@ function tailText(text: string, maxChars: number): string { } function commandLabel(options: ParsedOptions): string { + if (options.taskRunName !== null) return "cicd branch-follower taskrun"; return `cicd branch-follower ${options.action}`; } diff --git a/scripts/src/cicd-drilldown-render.ts b/scripts/src/cicd-drilldown-render.ts index 9b1c16ee..72f8d4ba 100644 --- a/scripts/src/cicd-drilldown-render.ts +++ b/scripts/src/cicd-drilldown-render.ts @@ -2,6 +2,7 @@ // Responsibility: bounded human summaries for branch-follower events/logs gates. export function renderDrillDownHuman(payload: Record): string { + if (payload.action === "taskrun") return renderTaskRunHuman(payload); if (payload.follower === undefined) { const followers = arrayRecords(payload.followers); return [ @@ -26,6 +27,75 @@ export function renderDrillDownHuman(payload: Record): string { ].filter((line) => line !== "").join("\n"); } +function renderTaskRunHuman(payload: Record): string { + const result = asOptionalRecord(payload.result); + const taskRun = asOptionalRecord(result?.taskRun); + const pod = asOptionalRecord(result?.pod); + const policy = asOptionalRecord(payload.policy); + const containers = arrayRecords(result?.containers); + const logs = arrayRecords(result?.logs); + const errors = arrayRecords(result?.errors); + const timing = asOptionalRecord(result?.nodeCicdTiming); + const query = asOptionalRecord(payload.query); + const command = asOptionalRecord(payload.command); + return [ + `CI/CD BRANCH-FOLLOWER TASKRUN (${payload.ok === false ? "failed" : "ok"})`, + "", + table( + ["FOLLOWER", "ADAPTER", "TASKRUN", "PIPELINERUN", "POD", "STATUS", "REASON", "DURATION", "CONTAINERS"], + [[ + payload.follower, + payload.adapter ?? "-", + taskRun?.name ?? query?.taskRun ?? "-", + taskRun?.pipelineRun ?? query?.pipelineRun ?? "-", + taskRun?.podName ?? "-", + asOptionalRecord(taskRun?.condition)?.status ?? "-", + asOptionalRecord(taskRun?.condition)?.reason ?? "-", + taskRun?.durationSeconds ?? "-", + pod?.containerCount ?? "-", + ]], + ), + containers.length === 0 ? "" : `\nCONTAINERS\n${table(["NAME", "CONTAINER", "STATE", "REASON", "EXIT", "STARTED", "FINISHED", "MESSAGE"], containers.map(containerRow))}`, + logs.length === 0 ? "" : `\nLOG TAILS\n${table(["POD", "CONTAINER", "STATUS", "REASON", "LINES", "BYTES", "TIMING", "MESSAGE"], logs.map(logRow))}`, + errors.length === 0 ? "" : `\nERRORS\n${table(["POD", "CONTAINER", "REASON", "MESSAGE"], errors.map((item) => [item.pod, item.container, item.degradedReason, item.message]))}`, + timing === null ? "" : `\nNODE_CICD_TIMING\n${JSON.stringify(timing, null, 2)}`, + "", + `policy: tailLines=${policy?.logsTailLines ?? "-"} maxLogBytes=${policy?.maxLogBytes ?? "-"} timeoutSeconds=${policy?.taskRunTimeoutSeconds ?? "-"} maxContainers=${policy?.maxContainers ?? "-"}`, + command?.stderrTail ? `stderr: ${command.stderrTail}` : "", + "", + ].filter((line) => line !== "").join("\n"); +} + +function logRow(item: Record): unknown[] { + return [ + item.pod, + item.container, + item.ok === false ? "failed" : "ok", + item.degradedReason ?? "-", + item.lineCount, + item.bytes, + asOptionalRecord(item.nodeCicdTiming) === null ? "-" : "node-cicd-timing", + item.message ?? "-", + ]; +} + +function containerRow(item: Record): unknown[] { + const terminated = asOptionalRecord(item.terminated); + const waiting = asOptionalRecord(item.waiting); + const running = asOptionalRecord(item.running); + const state = terminated !== null ? "terminated" : waiting !== null ? "waiting" : running !== null ? "running" : "-"; + return [ + item.name, + item.containerName, + state, + terminated?.reason ?? waiting?.reason ?? "-", + terminated?.exitCode ?? "-", + terminated?.startedAt ?? running?.startedAt ?? "-", + terminated?.finishedAt ?? "-", + terminated?.message ?? waiting?.message ?? "-", + ]; +} + function nativeGateRows(native: Record | null): unknown[][] { if (native === null) return []; const rows: unknown[][] = []; diff --git a/scripts/src/cicd-render.ts b/scripts/src/cicd-render.ts index 88626908..43fae256 100644 --- a/scripts/src/cicd-render.ts +++ b/scripts/src/cicd-render.ts @@ -27,7 +27,7 @@ function renderHuman(command: string, payload: Record, options: if (command.endsWith(" run-once")) return renderRunOnceHuman(payload); if (command.endsWith(" debug-step")) return renderDebugStepHuman(payload); if (command.endsWith(" cleanup-state")) return renderCleanupStateHuman(payload); - if (command.endsWith(" events") || command.endsWith(" logs")) return renderDrillDownHuman(payload); + if (command.endsWith(" events") || command.endsWith(" logs") || command.endsWith(" taskrun")) return renderDrillDownHuman(payload); return `${JSON.stringify(payload, null, 2)}\n`; } diff --git a/scripts/src/cicd-taskrun-drilldown.ts b/scripts/src/cicd-taskrun-drilldown.ts new file mode 100644 index 00000000..7734bbc0 --- /dev/null +++ b/scripts/src/cicd-taskrun-drilldown.ts @@ -0,0 +1,87 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower TaskRun drill-down. +// Responsibility: follower-scoped read-only TaskRun/Pod/container/log-tail visibility. +import type { CommandResult } from "./command"; +import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types"; +import { nativeCicdScriptLoadShell } from "./cicd-native-bundle"; +import { redactText, shQuote } from "./platform-infra-ops-library"; + +type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult; + +export async function runBranchFollowerTaskRunDrillDown( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + options: ParsedOptions, + runKubeScript: KubeScriptRunner, +): Promise> { + if (follower.nativeStatus.tekton === null) throw new Error(`follower ${follower.id} has no Tekton native status config`); + const taskRun = options.taskRunName; + if (taskRun === null) throw new Error("taskrun drill-down requires --taskrun "); + const policy = { + taskRunTimeoutSeconds: options.timeoutSeconds ?? follower.drillDown.taskRunTimeoutSeconds, + logsTailLines: options.logsTailLines ?? follower.drillDown.logsTailLines, + maxLogBytes: options.maxLogBytes ?? follower.drillDown.maxLogBytes, + maxMessageBytes: follower.drillDown.maxMessageBytes, + maxContainers: follower.drillDown.maxContainers, + }; + const script = [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["taskrun-drilldown.mjs"]), + `TEKTON_NAMESPACE=${shQuote(follower.nativeStatus.tekton.namespace)}`, + `TASKRUN_QUERY=${shQuote(taskRun)}`, + `PIPELINE_RUN_NAME=${shQuote(options.pipelineRunName ?? "")}`, + `PIPELINE_RUN_PREFIX=${shQuote(follower.nativeStatus.tekton.pipelineRunPrefix)}`, + `LOGS_TAIL_LINES=${policy.logsTailLines}`, + `MAX_LOG_BYTES=${policy.maxLogBytes}`, + `MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`, + `MAX_CONTAINERS=${policy.maxContainers}`, + "export TEKTON_NAMESPACE TASKRUN_QUERY PIPELINE_RUN_NAME PIPELINE_RUN_PREFIX LOGS_TAIL_LINES MAX_LOG_BYTES MAX_MESSAGE_BYTES MAX_CONTAINERS", + "node \"$tmpdir/taskrun-drilldown.mjs\"", + ].join("\n"); + const startedAt = Date.now(); + const result = runKubeScript(registry, options, script, "", policy.taskRunTimeoutSeconds * 1000); + const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + return { + ok: result.exitCode === 0 && parsed !== null && parsed.ok !== false, + action: "taskrun", + follower: follower.id, + adapter: follower.adapter, + statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", + parsedDownstreamCliOutput: false, + query: { + taskRun, + pipelineRun: options.pipelineRunName, + tektonNamespace: follower.nativeStatus.tekton.namespace, + pipelineRunPrefix: follower.nativeStatus.tekton.pipelineRunPrefix, + }, + policy, + result: parsed, + command: { + exitCode: result.exitCode, + timedOut: result.timedOut, + elapsedMs: Date.now() - startedAt, + stderrTail: result.exitCode === 0 ? "" : redactText(tailText(result.stderr || result.stdout, 1200)), + }, + next: { + status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, + taskRunJson: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --taskrun ${taskRun} --logs-tail ${policy.logsTailLines} --json`, + }, + }; +} + +function parseJsonObject(text: string): Record | null { + const trimmed = text.trim(); + if (trimmed.length === 0) return null; + try { + const parsed = JSON.parse(trimmed) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; + } catch { + return null; + } +} + +function tailText(text: string, maxChars: number): string { + return text.length <= maxChars ? text : text.slice(text.length - maxChars); +} diff --git a/scripts/src/cicd-types.ts b/scripts/src/cicd-types.ts index 314a7d05..bb50b0ad 100644 --- a/scripts/src/cicd-types.ts +++ b/scripts/src/cicd-types.ts @@ -2,7 +2,7 @@ // Responsibility: type contracts shared by branch follower entry, controller render, and native K8s helpers. export type OutputMode = "human" | "json" | "yaml"; -export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs"; +export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs" | "taskrun"; export type BranchFollowerDebugStep = "state-read" | "controller-source" | "status-read" | "decide" | "state-write"; export type BranchFollowerPhase = | "Observed" @@ -31,6 +31,10 @@ export interface ParsedOptions { raw: boolean; recordState: boolean; debugStep: BranchFollowerDebugStep | null; + taskRunName: string | null; + pipelineRunName: string | null; + logsTailLines: number | null; + maxLogBytes: number | null; output: OutputMode; limit: number; tailBytes: number; @@ -79,6 +83,13 @@ export interface FollowerSpec { logs: CommandSpec; }; nativeStatus: NativeStatusSpec; + drillDown: { + taskRunTimeoutSeconds: number; + logsTailLines: number; + maxLogBytes: number; + maxMessageBytes: number; + maxContainers: number; + }; closeoutChecks: string[]; }