// SPEC: PJ2026-01060703 CI/CD branch follower job/runtime drill-down. // Responsibility: follower-scoped read-only K8s Job and runtime workload visibility. import type { CommandResult } from "./command"; import { resolveAgentRunLaneTarget } from "./agentrun-lanes"; import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types"; import { nativeCicdScriptLoadShell } from "./cicd-native-bundle"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe"; import { redactText, shQuote } from "./platform-infra-ops-library"; type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult; export async function runBranchFollowerJobDrillDown( registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner, ): Promise> { const query = options.jobName; if (query === null) throw new Error("job drill-down requires --job "); if (follower.drillDown === null) return missingPolicyPayload("job", follower, registry, options, query); const sourceCommit = options.sourceCommit ?? null; const target = resolveJobTarget(registry, follower, query, sourceCommit); const policy = drillDownPolicy(follower, options); if (target === null) { return { ok: false, action: "job", follower: follower.id, adapter: follower.adapter, degradedReason: "job-target-unresolved", message: `job stage ${query} cannot be resolved for ${follower.id}; pass a concrete Kubernetes Job name`, query: { job: query, sourceCommit }, policy, result: null, statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", parsedDownstreamCliOutput: false, next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` }, }; } const script = [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["k8s-job-drilldown.mjs"]), `JOB_NAMESPACE=${shQuote(target.namespace)}`, `JOB_NAME=${shQuote(target.jobName)}`, `STAGE_NAME=${shQuote(target.stage)}`, `SOURCE_COMMIT=${shQuote(sourceCommit ?? "")}`, `LOGS_TAIL_LINES=${policy.logsTailLines}`, `MAX_LOG_BYTES=${policy.maxLogBytes}`, `MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`, `MAX_CONTAINERS=${policy.maxContainers}`, "export JOB_NAMESPACE JOB_NAME STAGE_NAME SOURCE_COMMIT LOGS_TAIL_LINES MAX_LOG_BYTES MAX_MESSAGE_BYTES MAX_CONTAINERS", "node \"$tmpdir/k8s-job-drilldown.mjs\"", ].join("\n"); const startedAt = Date.now(); const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000); return drillDownPayload("job", follower, registry, options, policy, target, result, Date.now() - startedAt); } export async function runBranchFollowerRuntimeDrillDown( registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner, ): Promise> { if (follower.nativeStatus.runtime === null) throw new Error(`follower ${follower.id} has no runtime native status config`); if (follower.drillDown === null) return missingPolicyPayload("runtime", follower, registry, options, options.workloadName); const policy = drillDownPolicy(follower, options); const workloads = follower.nativeStatus.runtime.workloads .filter((item) => options.workloadName === null || item.name === options.workloadName) .map((item) => ({ kind: item.kind, name: item.name, sourceCommit: item.sourceCommit, })); if (workloads.length === 0) throw new Error(`unknown runtime workload ${options.workloadName ?? "-"}`); const workloadsB64 = Buffer.from(JSON.stringify(workloads), "utf8").toString("base64"); const script = [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["runtime-drilldown.mjs"]), `RUNTIME_NAMESPACE=${shQuote(follower.nativeStatus.runtime.namespace)}`, `EXPECTED_SHA=${shQuote(options.sourceCommit ?? "")}`, `WORKLOADS_B64=${shQuote(workloadsB64)}`, `MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`, `MAX_CONTAINERS=${policy.maxContainers}`, "export RUNTIME_NAMESPACE EXPECTED_SHA WORKLOADS_B64 MAX_MESSAGE_BYTES MAX_CONTAINERS", "node \"$tmpdir/runtime-drilldown.mjs\"", ].join("\n"); const startedAt = Date.now(); const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000); return drillDownPayload("runtime", follower, registry, options, policy, { namespace: follower.nativeStatus.runtime.namespace, workload: options.workloadName, sourceCommit: options.sourceCommit ?? null, }, result, Date.now() - startedAt); } function drillDownPolicy(follower: FollowerSpec, options: ParsedOptions): Record { const drillDown = follower.drillDown; if (drillDown === null) throw new Error(`follower ${follower.id} registry is missing drillDown policy`); return { timeoutSeconds: options.timeoutSeconds ?? drillDown.taskRunTimeoutSeconds, logsTailLines: options.logsTailLines ?? drillDown.logsTailLines, maxLogBytes: options.maxLogBytes ?? drillDown.maxLogBytes, maxMessageBytes: drillDown.maxMessageBytes, maxContainers: drillDown.maxContainers, }; } function drillDownPayload( action: "job" | "runtime", follower: FollowerSpec, registry: BranchFollowerRegistry, options: ParsedOptions, policy: Record, target: Record, command: CommandResult, elapsedMs: number, ): Record { const parsedResult = command.exitCode === 0 ? parseJsonObject(command.stdout) : { value: null, error: "target-command-failed" }; const parsed = parsedResult.value; const includeTails = command.exitCode !== 0 || parsed === null; return { ok: command.exitCode === 0 && parsed !== null && parsed.ok !== false, action, follower: follower.id, adapter: follower.adapter, statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", parsedDownstreamCliOutput: false, query: target, policy, result: parsed, command: { identity: { route: options.inCluster ? "in-cluster" : registry.controller.kubeRoute, script: action === "job" ? "scripts/native/cicd/k8s-job-drilldown.mjs" : "scripts/native/cicd/runtime-drilldown.mjs", }, exitCode: command.exitCode, timedOut: command.timedOut, elapsedMs, parseError: parsedResult.error, stdoutTail: includeTails ? redactText(tailText(command.stdout, 1600)) : "", stderrTail: includeTails ? redactText(tailText(command.stderr, 1200)) : "", }, next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` }, }; } function missingPolicyPayload(action: string, follower: FollowerSpec, registry: BranchFollowerRegistry, options: ParsedOptions, query: string | null): Record { return { ok: false, action, follower: follower.id, adapter: follower.adapter, degradedReason: "drilldown-policy-missing", message: `follower ${follower.id} registry is missing drillDown policy; apply the current config before drill-down`, query, policy: null, result: null, statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw", parsedDownstreamCliOutput: false, next: { apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, }, }; } function resolveJobTarget(registry: BranchFollowerRegistry, follower: FollowerSpec, query: string, sourceCommit: string | null): { namespace: string; jobName: string; stage: string } | null { if (!isStageAlias(query)) return { namespace: follower.target.namespace, jobName: query, stage: "explicit-job" }; if (sourceCommit === null) return null; if (query === "git-mirror-sync" || query === "git-mirror-flush") { const action = query === "git-mirror-sync" ? "sync" : "flush"; const target = gitMirrorJobTarget(follower, sourceCommit, action); return target === null ? null : { ...target, stage: query }; } if (query === "control-plane-refresh" && follower.adapter === "hwlab-node-runtime") { return { namespace: registry.controller.namespace, jobName: nativeCapabilityJobName(follower.id, "control-plane-refresh", sourceCommit), stage: query }; } if (follower.adapter === "agentrun-yaml-lane" && (query === "image-build" || query === "gitops-publish")) { const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); const prefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; if (query === "image-build") return { namespace: spec.ci.namespace, jobName: `${prefix}-build-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query }; return { namespace: spec.gitMirror.namespace, jobName: `${prefix}-gitops-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query }; } return null; } function isStageAlias(value: string): boolean { return ["git-mirror-sync", "git-mirror-flush", "control-plane-refresh", "image-build", "gitops-publish"].includes(value); } function gitMirrorJobTarget(follower: FollowerSpec, sourceCommit: string, action: "sync" | "flush"): { namespace: string; jobName: string } | null { const jobName = nativeCapabilityJobName(follower.id, action, sourceCommit); if (follower.adapter === "hwlab-node-runtime") { const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); return { namespace: nodeRuntimeGitMirrorTarget(spec).namespace, jobName }; } if (follower.adapter === "agentrun-yaml-lane") { const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); return { namespace: spec.gitMirror.namespace, jobName }; } return null; } function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`; return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); } function safeK8sNameSegment(value: string): string { const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, ""); return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, ""); } function parseJsonObject(text: string): { value: Record | null; error: string | null } { const trimmed = text.trim(); if (trimmed.length === 0) return { value: null, error: "empty-stdout" }; try { const parsed = JSON.parse(trimmed) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? { value: parsed as Record, error: null } : { value: null, error: "stdout-json-not-object" }; } catch (error) { return { value: null, error: `stdout-json-parse-failed: ${error instanceof Error ? error.message : String(error)}` }; } } function tailText(text: string, maxChars: number): string { return text.length <= maxChars ? text : text.slice(text.length - maxChars); }