feat(cicd): add branch follower closeout drilldown
This commit is contained in:
@@ -30,6 +30,7 @@ import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGi
|
||||
import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuseConfig, RUNTIME_REUSE_CONFIG_PATH, runtimeReuseService, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config";
|
||||
import { prioritizedTaskRunItems } from "./cicd-taskruns";
|
||||
import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown";
|
||||
import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown";
|
||||
import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
|
||||
import {
|
||||
arrayField,
|
||||
@@ -50,7 +51,7 @@ const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower";
|
||||
|
||||
export function cicdHelp(): unknown {
|
||||
return {
|
||||
command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun",
|
||||
command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun|job|runtime",
|
||||
output: "text by default; use --json, --raw, or -o json|yaml for machine output",
|
||||
usage: [
|
||||
"bun scripts/cli.ts cicd branch-follower plan",
|
||||
@@ -67,6 +68,8 @@ export function cicdHelp(): unknown {
|
||||
"bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master",
|
||||
"bun scripts/cli.ts cicd branch-follower status --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json",
|
||||
"bun scripts/cli.ts cicd branch-follower taskrun --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json",
|
||||
"bun scripts/cli.ts cicd branch-follower job --follower agentrun-jd01-v02 --source-commit <sha> --job image-build --json",
|
||||
"bun scripts/cli.ts cicd branch-follower runtime --follower agentrun-jd01-v02 --workload agentrun-mgr --source-commit <sha> --json",
|
||||
],
|
||||
config: DEFAULT_CONFIG_PATH,
|
||||
spec: `${SPEC_REF} ${SPEC_VERSION}`,
|
||||
@@ -78,7 +81,7 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string
|
||||
const top = args[0];
|
||||
if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json");
|
||||
if (top !== "branch-follower") {
|
||||
throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun");
|
||||
throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun|job|runtime");
|
||||
}
|
||||
const options = parseOptions(args.slice(1));
|
||||
const command = commandLabel(options);
|
||||
@@ -103,6 +106,10 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string
|
||||
return renderResult(command, await runFollowerDrillDown(registry, options), options);
|
||||
case "taskrun":
|
||||
return renderResult(command, await runTaskRunDrillDown(registry, options), options);
|
||||
case "job":
|
||||
return renderResult(command, await runJobDrillDown(registry, options), options);
|
||||
case "runtime":
|
||||
return renderResult(command, await runRuntimeDrillDown(registry, options), options);
|
||||
case "help":
|
||||
return renderMachine(command, cicdHelp(), "json");
|
||||
}
|
||||
@@ -113,7 +120,7 @@ function parseOptions(args: string[]): ParsedOptions {
|
||||
if (actionToken === undefined || isHelpToken(actionToken)) {
|
||||
return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1));
|
||||
}
|
||||
if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs", "taskrun"].includes(actionToken)) {
|
||||
if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs", "taskrun", "job", "runtime"].includes(actionToken)) {
|
||||
throw new Error(`cicd branch-follower unknown action: ${actionToken}`);
|
||||
}
|
||||
const action = actionToken as BranchFollowerAction;
|
||||
@@ -157,6 +164,12 @@ function parseOptions(args: string[]): ParsedOptions {
|
||||
options.taskRunName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg);
|
||||
} else if (arg === "--pipelinerun" || arg === "--pipeline-run") {
|
||||
options.pipelineRunName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg);
|
||||
} else if (arg === "--job") {
|
||||
options.jobName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg);
|
||||
} else if (arg === "--source-commit" || arg === "--source-sha") {
|
||||
options.sourceCommit = simpleGitSha(valueOption(rest, ++index, arg), arg);
|
||||
} else if (arg === "--workload") {
|
||||
options.workloadName = simpleK8sObjectName(valueOption(rest, ++index, arg), arg);
|
||||
} else if (arg === "--logs-tail") {
|
||||
options.logsTailLines = positiveInt(valueOption(rest, ++index, arg), arg);
|
||||
} else if (arg === "--max-log-bytes") {
|
||||
@@ -195,9 +208,15 @@ function parseOptions(args: string[]): ParsedOptions {
|
||||
if (options.action === "taskrun" && options.taskRunName === null) {
|
||||
throw new Error("taskrun requires --taskrun <taskrun-name|pipeline-task>");
|
||||
}
|
||||
if (options.action === "job" && options.jobName === null) {
|
||||
throw new Error("job requires --job <stage|job-name>");
|
||||
}
|
||||
if (options.taskRunName !== null && options.followerId === null) {
|
||||
throw new Error("--taskrun requires --follower <id>");
|
||||
}
|
||||
if ((options.action === "job" || options.action === "runtime" || options.jobName !== null || options.workloadName !== null) && options.followerId === null) {
|
||||
throw new Error(`${options.action} requires --follower <id>`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
@@ -228,6 +247,9 @@ function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOp
|
||||
debugStep: null,
|
||||
taskRunName: null,
|
||||
pipelineRunName: null,
|
||||
jobName: null,
|
||||
sourceCommit: null,
|
||||
workloadName: null,
|
||||
logsTailLines: null,
|
||||
maxLogBytes: null,
|
||||
output: "human",
|
||||
@@ -253,6 +275,11 @@ function simpleK8sObjectName(value: string, option: string): string {
|
||||
return value;
|
||||
}
|
||||
|
||||
function simpleGitSha(value: string, option: string): string {
|
||||
if (!/^[A-Fa-f0-9]{7,64}$/u.test(value)) throw new Error(`${option} must be a git sha`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveInt(value: string, option: string): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`);
|
||||
@@ -814,6 +841,20 @@ async function runTaskRunDrillDown(registry: BranchFollowerRegistry, options: Pa
|
||||
return runBranchFollowerTaskRunDrillDown(registry, follower, options, runKubeScript);
|
||||
}
|
||||
|
||||
async function runJobDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise<Record<string, unknown>> {
|
||||
if (options.followerId === null) throw new Error("job requires --follower <id>");
|
||||
const follower = registry.followers.find((item) => item.id === options.followerId);
|
||||
if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`);
|
||||
return runBranchFollowerJobDrillDown(registry, follower, options, runKubeScript);
|
||||
}
|
||||
|
||||
async function runRuntimeDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise<Record<string, unknown>> {
|
||||
if (options.followerId === null) throw new Error("runtime requires --follower <id>");
|
||||
const follower = registry.followers.find((item) => item.id === options.followerId);
|
||||
if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`);
|
||||
return runBranchFollowerRuntimeDrillDown(registry, follower, options, runKubeScript);
|
||||
}
|
||||
|
||||
async function decideAndMaybeTrigger(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
@@ -2299,9 +2340,11 @@ function buildFollowerTimings(
|
||||
const nativePayload = asOptionalRecord(live.payload);
|
||||
const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live);
|
||||
const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha);
|
||||
const storedStages = live.observedSha !== null && stringOrNull(storedTimings?.sourceCommit) === live.observedSha ? storedStageTimings(storedTimings ?? null) : [];
|
||||
const stages = dedupeTimingStages([
|
||||
...stageTimingsFromCommand(triggerCommand),
|
||||
...stageTimingsFromNativePayload(nativePayload),
|
||||
...storedStages,
|
||||
]).slice(0, 24);
|
||||
const stageSourceCommit = stages.length > 0 ? live.observedSha : null;
|
||||
return {
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
export function renderDrillDownHuman(payload: Record<string, unknown>): string {
|
||||
if (payload.action === "taskrun") return renderTaskRunHuman(payload);
|
||||
if (payload.action === "job") return renderJobHuman(payload);
|
||||
if (payload.action === "runtime") return renderRuntimeHuman(payload);
|
||||
if (payload.follower === undefined) {
|
||||
const followers = arrayRecords(payload.followers);
|
||||
return [
|
||||
@@ -27,6 +29,80 @@ export function renderDrillDownHuman(payload: Record<string, unknown>): string {
|
||||
].filter((line) => line !== "").join("\n");
|
||||
}
|
||||
|
||||
function renderJobHuman(payload: Record<string, unknown>): string {
|
||||
const result = asOptionalRecord(payload.result);
|
||||
const job = asOptionalRecord(result?.job);
|
||||
const query = asOptionalRecord(payload.query);
|
||||
const policy = asOptionalRecord(payload.policy);
|
||||
const pods = arrayRecords(result?.pods);
|
||||
const logs = arrayRecords(result?.logs);
|
||||
const errors = arrayRecords(result?.errors);
|
||||
const command = asOptionalRecord(payload.command);
|
||||
const identity = asOptionalRecord(command?.identity);
|
||||
return [
|
||||
`CI/CD BRANCH-FOLLOWER JOB (${payload.ok === false ? "failed" : "ok"})`,
|
||||
"",
|
||||
table(
|
||||
["FOLLOWER", "ADAPTER", "STAGE", "NAMESPACE", "JOB", "STATUS", "REASON", "DURATION", "PODS"],
|
||||
[[
|
||||
payload.follower,
|
||||
payload.adapter ?? "-",
|
||||
job?.stage ?? query?.stage ?? "-",
|
||||
job?.namespace ?? query?.namespace ?? "-",
|
||||
job?.name ?? query?.jobName ?? "-",
|
||||
jobStatus(job),
|
||||
asOptionalRecord(job?.condition)?.reason ?? result?.degradedReason ?? "-",
|
||||
job?.durationSeconds ?? "-",
|
||||
pods.length,
|
||||
]],
|
||||
),
|
||||
pods.length === 0 ? "" : `\nPODS\n${table(["POD", "PHASE", "READY", "START", "CONTAINERS", "REASON"], pods.map(jobPodRow))}`,
|
||||
logs.length === 0 ? "" : `\nLOG TAILS\n${table(["POD", "CONTAINER", "STATUS", "REASON", "LINES", "BYTES", "TIMING", "MESSAGE"], logs.map(logRow))}`,
|
||||
errors.length === 0 ? "" : `\nERRORS\n${table(["POD", "CONTAINER", "REASON", "MESSAGE"], errors.map((item) => [item.pod, item.container, item.degradedReason, item.message]))}`,
|
||||
command === null ? "" : `\nTARGET COMMAND\n${table(["ROUTE", "SCRIPT", "EXIT", "PARSE_ERROR"], [[identity?.route ?? "-", identity?.script ?? "-", command.exitCode ?? "-", command.parseError ?? "-"]])}`,
|
||||
command?.stdoutTail ? `\nSTDOUT_TAIL\n${command.stdoutTail}` : "",
|
||||
command?.stderrTail ? `\nSTDERR_TAIL\n${command.stderrTail}` : "",
|
||||
"",
|
||||
`policy: tailLines=${policy?.logsTailLines ?? "-"} maxLogBytes=${policy?.maxLogBytes ?? "-"} timeoutSeconds=${policy?.timeoutSeconds ?? "-"} maxContainers=${policy?.maxContainers ?? "-"}`,
|
||||
"",
|
||||
].filter((line) => line !== "").join("\n");
|
||||
}
|
||||
|
||||
function renderRuntimeHuman(payload: Record<string, unknown>): string {
|
||||
const result = asOptionalRecord(payload.result);
|
||||
const query = asOptionalRecord(payload.query);
|
||||
const policy = asOptionalRecord(payload.policy);
|
||||
const workloads = arrayRecords(result?.workloads);
|
||||
const pods = workloads.flatMap((workload) => arrayRecords(workload.pods).map((pod) => ({ ...pod, workload: `${workload.kind ?? "-"}/${workload.name ?? "-"}` }))).slice(0, 12);
|
||||
const command = asOptionalRecord(payload.command);
|
||||
const identity = asOptionalRecord(command?.identity);
|
||||
return [
|
||||
`CI/CD BRANCH-FOLLOWER RUNTIME (${payload.ok === false ? "failed" : "ok"})`,
|
||||
"",
|
||||
table(
|
||||
["FOLLOWER", "ADAPTER", "NAMESPACE", "EXPECTED", "TARGET", "READY", "ALIGNED", "BLOCKING"],
|
||||
[[
|
||||
payload.follower,
|
||||
payload.adapter ?? "-",
|
||||
result?.namespace ?? query?.namespace ?? "-",
|
||||
shortSha(stringOrNull(result?.expectedSha)),
|
||||
shortSha(stringOrNull(result?.targetSha)),
|
||||
result?.ready ?? "-",
|
||||
result?.aligned ?? "-",
|
||||
result?.blockingReason ?? "-",
|
||||
]],
|
||||
),
|
||||
workloads.length === 0 ? "" : `\nWORKLOADS\n${table(["KIND", "NAME", "READY", "ALIGNED", "REPLICAS", "UPDATED", "SOURCE", "BLOCKING"], workloads.map(runtimeWorkloadRow))}`,
|
||||
pods.length === 0 ? "" : `\nPODS\n${table(["WORKLOAD", "POD", "PHASE", "READY", "START", "SOURCE", "CONTAINERS"], pods.map(runtimePodRow))}`,
|
||||
command === null ? "" : `\nTARGET COMMAND\n${table(["ROUTE", "SCRIPT", "EXIT", "PARSE_ERROR"], [[identity?.route ?? "-", identity?.script ?? "-", command.exitCode ?? "-", command.parseError ?? "-"]])}`,
|
||||
command?.stdoutTail ? `\nSTDOUT_TAIL\n${command.stdoutTail}` : "",
|
||||
command?.stderrTail ? `\nSTDERR_TAIL\n${command.stderrTail}` : "",
|
||||
"",
|
||||
`policy: timeoutSeconds=${policy?.timeoutSeconds ?? "-"} maxContainers=${policy?.maxContainers ?? "-"}`,
|
||||
"",
|
||||
].filter((line) => line !== "").join("\n");
|
||||
}
|
||||
|
||||
function renderTaskRunHuman(payload: Record<string, unknown>): string {
|
||||
const result = asOptionalRecord(payload.result);
|
||||
const taskRun = asOptionalRecord(result?.taskRun);
|
||||
@@ -69,6 +145,52 @@ function renderTaskRunHuman(payload: Record<string, unknown>): string {
|
||||
].filter((line) => line !== "").join("\n");
|
||||
}
|
||||
|
||||
function jobStatus(job: Record<string, unknown> | null): string {
|
||||
if (job === null) return "-";
|
||||
if (job.completed === true) return "completed";
|
||||
if (job.failedState === true) return "failed";
|
||||
if (numberOrNull(job.active) !== null && numberOrNull(job.active)! > 0) return "active";
|
||||
return stringOrNull(asOptionalRecord(job.condition)?.type) ?? "-";
|
||||
}
|
||||
|
||||
function jobPodRow(item: Record<string, unknown>): unknown[] {
|
||||
return [
|
||||
item.name,
|
||||
item.phase,
|
||||
item.ready,
|
||||
item.startTime ?? item.createdAt ?? "-",
|
||||
item.containerCount,
|
||||
item.reason ?? "-",
|
||||
];
|
||||
}
|
||||
|
||||
function runtimeWorkloadRow(item: Record<string, unknown>): unknown[] {
|
||||
const sourceCommit = asOptionalRecord(item.sourceCommit);
|
||||
return [
|
||||
item.kind,
|
||||
item.name,
|
||||
item.ready,
|
||||
item.aligned,
|
||||
`${item.readyReplicas ?? "-"}/${item.replicas ?? "-"}`,
|
||||
item.updatedReplicas ?? "-",
|
||||
shortSha(stringOrNull(sourceCommit?.value)),
|
||||
item.blockingReason ?? "-",
|
||||
];
|
||||
}
|
||||
|
||||
function runtimePodRow(item: Record<string, unknown>): unknown[] {
|
||||
const sourceCommit = asOptionalRecord(item.sourceCommit);
|
||||
return [
|
||||
item.workload,
|
||||
item.name,
|
||||
item.phase,
|
||||
item.ready,
|
||||
item.startTime ?? item.createdAt ?? "-",
|
||||
shortSha(stringOrNull(sourceCommit?.value)),
|
||||
arrayRecords(item.containers).length,
|
||||
];
|
||||
}
|
||||
|
||||
function logRow(item: Record<string, unknown>): unknown[] {
|
||||
return [
|
||||
item.pod,
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower job/runtime drill-down.
|
||||
// Responsibility: follower-scoped read-only K8s Job and runtime workload visibility.
|
||||
import type { CommandResult } from "./command";
|
||||
import { resolveAgentRunLaneTarget } from "./agentrun-lanes";
|
||||
import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types";
|
||||
import { nativeCicdScriptLoadShell } from "./cicd-native-bundle";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe";
|
||||
import { redactText, shQuote } from "./platform-infra-ops-library";
|
||||
|
||||
type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
|
||||
|
||||
export async function runBranchFollowerJobDrillDown(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
options: ParsedOptions,
|
||||
runKubeScript: KubeScriptRunner,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const query = options.jobName;
|
||||
if (query === null) throw new Error("job drill-down requires --job <stage|job-name>");
|
||||
if (follower.drillDown === null) return missingPolicyPayload("job", follower, registry, options, query);
|
||||
const sourceCommit = options.sourceCommit ?? null;
|
||||
const target = resolveJobTarget(registry, follower, query, sourceCommit);
|
||||
const policy = drillDownPolicy(follower, options);
|
||||
if (target === null) {
|
||||
return {
|
||||
ok: false,
|
||||
action: "job",
|
||||
follower: follower.id,
|
||||
adapter: follower.adapter,
|
||||
degradedReason: "job-target-unresolved",
|
||||
message: `job stage ${query} cannot be resolved for ${follower.id}; pass a concrete Kubernetes Job name`,
|
||||
query: { job: query, sourceCommit },
|
||||
policy,
|
||||
result: null,
|
||||
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
||||
parsedDownstreamCliOutput: false,
|
||||
next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` },
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
"tmpdir=$(mktemp -d)",
|
||||
"cleanup() { rm -rf \"$tmpdir\"; }",
|
||||
"trap cleanup EXIT INT TERM",
|
||||
nativeCicdScriptLoadShell(["k8s-job-drilldown.mjs"]),
|
||||
`JOB_NAMESPACE=${shQuote(target.namespace)}`,
|
||||
`JOB_NAME=${shQuote(target.jobName)}`,
|
||||
`STAGE_NAME=${shQuote(target.stage)}`,
|
||||
`SOURCE_COMMIT=${shQuote(sourceCommit ?? "")}`,
|
||||
`LOGS_TAIL_LINES=${policy.logsTailLines}`,
|
||||
`MAX_LOG_BYTES=${policy.maxLogBytes}`,
|
||||
`MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`,
|
||||
`MAX_CONTAINERS=${policy.maxContainers}`,
|
||||
"export JOB_NAMESPACE JOB_NAME STAGE_NAME SOURCE_COMMIT LOGS_TAIL_LINES MAX_LOG_BYTES MAX_MESSAGE_BYTES MAX_CONTAINERS",
|
||||
"node \"$tmpdir/k8s-job-drilldown.mjs\"",
|
||||
].join("\n");
|
||||
const startedAt = Date.now();
|
||||
const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000);
|
||||
return drillDownPayload("job", follower, registry, options, policy, target, result, Date.now() - startedAt);
|
||||
}
|
||||
|
||||
export async function runBranchFollowerRuntimeDrillDown(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
options: ParsedOptions,
|
||||
runKubeScript: KubeScriptRunner,
|
||||
): Promise<Record<string, unknown>> {
|
||||
if (follower.nativeStatus.runtime === null) throw new Error(`follower ${follower.id} has no runtime native status config`);
|
||||
if (follower.drillDown === null) return missingPolicyPayload("runtime", follower, registry, options, options.workloadName);
|
||||
const policy = drillDownPolicy(follower, options);
|
||||
const workloads = follower.nativeStatus.runtime.workloads
|
||||
.filter((item) => options.workloadName === null || item.name === options.workloadName)
|
||||
.map((item) => ({
|
||||
kind: item.kind,
|
||||
name: item.name,
|
||||
sourceCommit: item.sourceCommit,
|
||||
}));
|
||||
if (workloads.length === 0) throw new Error(`unknown runtime workload ${options.workloadName ?? "-"}`);
|
||||
const workloadsB64 = Buffer.from(JSON.stringify(workloads), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
"tmpdir=$(mktemp -d)",
|
||||
"cleanup() { rm -rf \"$tmpdir\"; }",
|
||||
"trap cleanup EXIT INT TERM",
|
||||
nativeCicdScriptLoadShell(["runtime-drilldown.mjs"]),
|
||||
`RUNTIME_NAMESPACE=${shQuote(follower.nativeStatus.runtime.namespace)}`,
|
||||
`EXPECTED_SHA=${shQuote(options.sourceCommit ?? "")}`,
|
||||
`WORKLOADS_B64=${shQuote(workloadsB64)}`,
|
||||
`MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`,
|
||||
`MAX_CONTAINERS=${policy.maxContainers}`,
|
||||
"export RUNTIME_NAMESPACE EXPECTED_SHA WORKLOADS_B64 MAX_MESSAGE_BYTES MAX_CONTAINERS",
|
||||
"node \"$tmpdir/runtime-drilldown.mjs\"",
|
||||
].join("\n");
|
||||
const startedAt = Date.now();
|
||||
const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000);
|
||||
return drillDownPayload("runtime", follower, registry, options, policy, {
|
||||
namespace: follower.nativeStatus.runtime.namespace,
|
||||
workload: options.workloadName,
|
||||
sourceCommit: options.sourceCommit ?? null,
|
||||
}, result, Date.now() - startedAt);
|
||||
}
|
||||
|
||||
function drillDownPolicy(follower: FollowerSpec, options: ParsedOptions): Record<string, number> {
|
||||
const drillDown = follower.drillDown;
|
||||
if (drillDown === null) throw new Error(`follower ${follower.id} registry is missing drillDown policy`);
|
||||
return {
|
||||
timeoutSeconds: options.timeoutSeconds ?? drillDown.taskRunTimeoutSeconds,
|
||||
logsTailLines: options.logsTailLines ?? drillDown.logsTailLines,
|
||||
maxLogBytes: options.maxLogBytes ?? drillDown.maxLogBytes,
|
||||
maxMessageBytes: drillDown.maxMessageBytes,
|
||||
maxContainers: drillDown.maxContainers,
|
||||
};
|
||||
}
|
||||
|
||||
function drillDownPayload(
|
||||
action: "job" | "runtime",
|
||||
follower: FollowerSpec,
|
||||
registry: BranchFollowerRegistry,
|
||||
options: ParsedOptions,
|
||||
policy: Record<string, number>,
|
||||
target: Record<string, unknown>,
|
||||
command: CommandResult,
|
||||
elapsedMs: number,
|
||||
): Record<string, unknown> {
|
||||
const parsedResult = command.exitCode === 0 ? parseJsonObject(command.stdout) : { value: null, error: "target-command-failed" };
|
||||
const parsed = parsedResult.value;
|
||||
const includeTails = command.exitCode !== 0 || parsed === null;
|
||||
return {
|
||||
ok: command.exitCode === 0 && parsed !== null && parsed.ok !== false,
|
||||
action,
|
||||
follower: follower.id,
|
||||
adapter: follower.adapter,
|
||||
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
||||
parsedDownstreamCliOutput: false,
|
||||
query: target,
|
||||
policy,
|
||||
result: parsed,
|
||||
command: {
|
||||
identity: {
|
||||
route: options.inCluster ? "in-cluster" : registry.controller.kubeRoute,
|
||||
script: action === "job" ? "scripts/native/cicd/k8s-job-drilldown.mjs" : "scripts/native/cicd/runtime-drilldown.mjs",
|
||||
},
|
||||
exitCode: command.exitCode,
|
||||
timedOut: command.timedOut,
|
||||
elapsedMs,
|
||||
parseError: parsedResult.error,
|
||||
stdoutTail: includeTails ? redactText(tailText(command.stdout, 1600)) : "",
|
||||
stderrTail: includeTails ? redactText(tailText(command.stderr, 1200)) : "",
|
||||
},
|
||||
next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` },
|
||||
};
|
||||
}
|
||||
|
||||
function missingPolicyPayload(action: string, follower: FollowerSpec, registry: BranchFollowerRegistry, options: ParsedOptions, query: string | null): Record<string, unknown> {
|
||||
return {
|
||||
ok: false,
|
||||
action,
|
||||
follower: follower.id,
|
||||
adapter: follower.adapter,
|
||||
degradedReason: "drilldown-policy-missing",
|
||||
message: `follower ${follower.id} registry is missing drillDown policy; apply the current config before drill-down`,
|
||||
query,
|
||||
policy: null,
|
||||
result: null,
|
||||
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
||||
parsedDownstreamCliOutput: false,
|
||||
next: {
|
||||
apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
||||
status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveJobTarget(registry: BranchFollowerRegistry, follower: FollowerSpec, query: string, sourceCommit: string | null): { namespace: string; jobName: string; stage: string } | null {
|
||||
if (!isStageAlias(query)) return { namespace: follower.target.namespace, jobName: query, stage: "explicit-job" };
|
||||
if (sourceCommit === null) return null;
|
||||
if (query === "git-mirror-sync" || query === "git-mirror-flush") {
|
||||
const action = query === "git-mirror-sync" ? "sync" : "flush";
|
||||
const target = gitMirrorJobTarget(follower, sourceCommit, action);
|
||||
return target === null ? null : { ...target, stage: query };
|
||||
}
|
||||
if (query === "control-plane-refresh" && follower.adapter === "hwlab-node-runtime") {
|
||||
return { namespace: registry.controller.namespace, jobName: nativeCapabilityJobName(follower.id, "control-plane-refresh", sourceCommit), stage: query };
|
||||
}
|
||||
if (follower.adapter === "agentrun-yaml-lane" && (query === "image-build" || query === "gitops-publish")) {
|
||||
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
||||
const prefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
||||
if (query === "image-build") return { namespace: spec.ci.namespace, jobName: `${prefix}-build-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query };
|
||||
return { namespace: spec.gitMirror.namespace, jobName: `${prefix}-gitops-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isStageAlias(value: string): boolean {
|
||||
return ["git-mirror-sync", "git-mirror-flush", "control-plane-refresh", "image-build", "gitops-publish"].includes(value);
|
||||
}
|
||||
|
||||
function gitMirrorJobTarget(follower: FollowerSpec, sourceCommit: string, action: "sync" | "flush"): { namespace: string; jobName: string } | null {
|
||||
const jobName = nativeCapabilityJobName(follower.id, action, sourceCommit);
|
||||
if (follower.adapter === "hwlab-node-runtime") {
|
||||
const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node);
|
||||
return { namespace: nodeRuntimeGitMirrorTarget(spec).namespace, jobName };
|
||||
}
|
||||
if (follower.adapter === "agentrun-yaml-lane") {
|
||||
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
||||
return { namespace: spec.gitMirror.namespace, jobName };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function nativeCapabilityJobName(followerId: string, action: string, sha: string): string {
|
||||
const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`;
|
||||
return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63);
|
||||
}
|
||||
|
||||
function safeK8sNameSegment(value: string): string {
|
||||
const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, "");
|
||||
return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, "");
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): { value: Record<string, unknown> | null; error: string | null } {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length === 0) return { value: null, error: "empty-stdout" };
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
|
||||
? { value: parsed as Record<string, unknown>, error: null }
|
||||
: { value: null, error: "stdout-json-not-object" };
|
||||
} catch (error) {
|
||||
return { value: null, error: `stdout-json-parse-failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
}
|
||||
}
|
||||
|
||||
function tailText(text: string, maxChars: number): string {
|
||||
return text.length <= maxChars ? text : text.slice(text.length - maxChars);
|
||||
}
|
||||
@@ -27,7 +27,7 @@ function renderHuman(command: string, payload: Record<string, unknown>, options:
|
||||
if (command.endsWith(" run-once")) return renderRunOnceHuman(payload);
|
||||
if (command.endsWith(" debug-step")) return renderDebugStepHuman(payload);
|
||||
if (command.endsWith(" cleanup-state")) return renderCleanupStateHuman(payload);
|
||||
if (command.endsWith(" events") || command.endsWith(" logs") || command.endsWith(" taskrun")) return renderDrillDownHuman(payload);
|
||||
if (command.endsWith(" events") || command.endsWith(" logs") || command.endsWith(" taskrun") || command.endsWith(" job") || command.endsWith(" runtime")) return renderDrillDownHuman(payload);
|
||||
return `${JSON.stringify(payload, null, 2)}\n`;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Responsibility: type contracts shared by branch follower entry, controller render, and native K8s helpers.
|
||||
|
||||
export type OutputMode = "human" | "json" | "yaml";
|
||||
export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs" | "taskrun";
|
||||
export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs" | "taskrun" | "job" | "runtime";
|
||||
export type BranchFollowerDebugStep = "state-read" | "controller-source" | "status-read" | "decide" | "state-write";
|
||||
export type BranchFollowerPhase =
|
||||
| "Observed"
|
||||
@@ -33,6 +33,9 @@ export interface ParsedOptions {
|
||||
debugStep: BranchFollowerDebugStep | null;
|
||||
taskRunName: string | null;
|
||||
pipelineRunName: string | null;
|
||||
jobName: string | null;
|
||||
sourceCommit: string | null;
|
||||
workloadName: string | null;
|
||||
logsTailLines: number | null;
|
||||
maxLogBytes: number | null;
|
||||
output: OutputMode;
|
||||
|
||||
Reference in New Issue
Block a user