feat(cicd): add branch follower closeout drilldown
This commit is contained in:
Codex
@@ -0,0 +1,237 @@
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower job/runtime drill-down.
|
||||
// Responsibility: follower-scoped read-only K8s Job and runtime workload visibility.
|
||||
import type { CommandResult } from "./command";
|
||||
import { resolveAgentRunLaneTarget } from "./agentrun-lanes";
|
||||
import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types";
|
||||
import { nativeCicdScriptLoadShell } from "./cicd-native-bundle";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe";
|
||||
import { redactText, shQuote } from "./platform-infra-ops-library";
|
||||
|
||||
type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
|
||||
|
||||
export async function runBranchFollowerJobDrillDown(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
options: ParsedOptions,
|
||||
runKubeScript: KubeScriptRunner,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const query = options.jobName;
|
||||
if (query === null) throw new Error("job drill-down requires --job <stage|job-name>");
|
||||
if (follower.drillDown === null) return missingPolicyPayload("job", follower, registry, options, query);
|
||||
const sourceCommit = options.sourceCommit ?? null;
|
||||
const target = resolveJobTarget(registry, follower, query, sourceCommit);
|
||||
const policy = drillDownPolicy(follower, options);
|
||||
if (target === null) {
|
||||
return {
|
||||
ok: false,
|
||||
action: "job",
|
||||
follower: follower.id,
|
||||
adapter: follower.adapter,
|
||||
degradedReason: "job-target-unresolved",
|
||||
message: `job stage ${query} cannot be resolved for ${follower.id}; pass a concrete Kubernetes Job name`,
|
||||
query: { job: query, sourceCommit },
|
||||
policy,
|
||||
result: null,
|
||||
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
||||
parsedDownstreamCliOutput: false,
|
||||
next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` },
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
"tmpdir=$(mktemp -d)",
|
||||
"cleanup() { rm -rf \"$tmpdir\"; }",
|
||||
"trap cleanup EXIT INT TERM",
|
||||
nativeCicdScriptLoadShell(["k8s-job-drilldown.mjs"]),
|
||||
`JOB_NAMESPACE=${shQuote(target.namespace)}`,
|
||||
`JOB_NAME=${shQuote(target.jobName)}`,
|
||||
`STAGE_NAME=${shQuote(target.stage)}`,
|
||||
`SOURCE_COMMIT=${shQuote(sourceCommit ?? "")}`,
|
||||
`LOGS_TAIL_LINES=${policy.logsTailLines}`,
|
||||
`MAX_LOG_BYTES=${policy.maxLogBytes}`,
|
||||
`MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`,
|
||||
`MAX_CONTAINERS=${policy.maxContainers}`,
|
||||
"export JOB_NAMESPACE JOB_NAME STAGE_NAME SOURCE_COMMIT LOGS_TAIL_LINES MAX_LOG_BYTES MAX_MESSAGE_BYTES MAX_CONTAINERS",
|
||||
"node \"$tmpdir/k8s-job-drilldown.mjs\"",
|
||||
].join("\n");
|
||||
const startedAt = Date.now();
|
||||
const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000);
|
||||
return drillDownPayload("job", follower, registry, options, policy, target, result, Date.now() - startedAt);
|
||||
}
|
||||
|
||||
export async function runBranchFollowerRuntimeDrillDown(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
options: ParsedOptions,
|
||||
runKubeScript: KubeScriptRunner,
|
||||
): Promise<Record<string, unknown>> {
|
||||
if (follower.nativeStatus.runtime === null) throw new Error(`follower ${follower.id} has no runtime native status config`);
|
||||
if (follower.drillDown === null) return missingPolicyPayload("runtime", follower, registry, options, options.workloadName);
|
||||
const policy = drillDownPolicy(follower, options);
|
||||
const workloads = follower.nativeStatus.runtime.workloads
|
||||
.filter((item) => options.workloadName === null || item.name === options.workloadName)
|
||||
.map((item) => ({
|
||||
kind: item.kind,
|
||||
name: item.name,
|
||||
sourceCommit: item.sourceCommit,
|
||||
}));
|
||||
if (workloads.length === 0) throw new Error(`unknown runtime workload ${options.workloadName ?? "-"}`);
|
||||
const workloadsB64 = Buffer.from(JSON.stringify(workloads), "utf8").toString("base64");
|
||||
const script = [
|
||||
"set -eu",
|
||||
"tmpdir=$(mktemp -d)",
|
||||
"cleanup() { rm -rf \"$tmpdir\"; }",
|
||||
"trap cleanup EXIT INT TERM",
|
||||
nativeCicdScriptLoadShell(["runtime-drilldown.mjs"]),
|
||||
`RUNTIME_NAMESPACE=${shQuote(follower.nativeStatus.runtime.namespace)}`,
|
||||
`EXPECTED_SHA=${shQuote(options.sourceCommit ?? "")}`,
|
||||
`WORKLOADS_B64=${shQuote(workloadsB64)}`,
|
||||
`MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`,
|
||||
`MAX_CONTAINERS=${policy.maxContainers}`,
|
||||
"export RUNTIME_NAMESPACE EXPECTED_SHA WORKLOADS_B64 MAX_MESSAGE_BYTES MAX_CONTAINERS",
|
||||
"node \"$tmpdir/runtime-drilldown.mjs\"",
|
||||
].join("\n");
|
||||
const startedAt = Date.now();
|
||||
const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000);
|
||||
return drillDownPayload("runtime", follower, registry, options, policy, {
|
||||
namespace: follower.nativeStatus.runtime.namespace,
|
||||
workload: options.workloadName,
|
||||
sourceCommit: options.sourceCommit ?? null,
|
||||
}, result, Date.now() - startedAt);
|
||||
}
|
||||
|
||||
function drillDownPolicy(follower: FollowerSpec, options: ParsedOptions): Record<string, number> {
|
||||
const drillDown = follower.drillDown;
|
||||
if (drillDown === null) throw new Error(`follower ${follower.id} registry is missing drillDown policy`);
|
||||
return {
|
||||
timeoutSeconds: options.timeoutSeconds ?? drillDown.taskRunTimeoutSeconds,
|
||||
logsTailLines: options.logsTailLines ?? drillDown.logsTailLines,
|
||||
maxLogBytes: options.maxLogBytes ?? drillDown.maxLogBytes,
|
||||
maxMessageBytes: drillDown.maxMessageBytes,
|
||||
maxContainers: drillDown.maxContainers,
|
||||
};
|
||||
}
|
||||
|
||||
function drillDownPayload(
|
||||
action: "job" | "runtime",
|
||||
follower: FollowerSpec,
|
||||
registry: BranchFollowerRegistry,
|
||||
options: ParsedOptions,
|
||||
policy: Record<string, number>,
|
||||
target: Record<string, unknown>,
|
||||
command: CommandResult,
|
||||
elapsedMs: number,
|
||||
): Record<string, unknown> {
|
||||
const parsedResult = command.exitCode === 0 ? parseJsonObject(command.stdout) : { value: null, error: "target-command-failed" };
|
||||
const parsed = parsedResult.value;
|
||||
const includeTails = command.exitCode !== 0 || parsed === null;
|
||||
return {
|
||||
ok: command.exitCode === 0 && parsed !== null && parsed.ok !== false,
|
||||
action,
|
||||
follower: follower.id,
|
||||
adapter: follower.adapter,
|
||||
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
||||
parsedDownstreamCliOutput: false,
|
||||
query: target,
|
||||
policy,
|
||||
result: parsed,
|
||||
command: {
|
||||
identity: {
|
||||
route: options.inCluster ? "in-cluster" : registry.controller.kubeRoute,
|
||||
script: action === "job" ? "scripts/native/cicd/k8s-job-drilldown.mjs" : "scripts/native/cicd/runtime-drilldown.mjs",
|
||||
},
|
||||
exitCode: command.exitCode,
|
||||
timedOut: command.timedOut,
|
||||
elapsedMs,
|
||||
parseError: parsedResult.error,
|
||||
stdoutTail: includeTails ? redactText(tailText(command.stdout, 1600)) : "",
|
||||
stderrTail: includeTails ? redactText(tailText(command.stderr, 1200)) : "",
|
||||
},
|
||||
next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` },
|
||||
};
|
||||
}
|
||||
|
||||
function missingPolicyPayload(action: string, follower: FollowerSpec, registry: BranchFollowerRegistry, options: ParsedOptions, query: string | null): Record<string, unknown> {
|
||||
return {
|
||||
ok: false,
|
||||
action,
|
||||
follower: follower.id,
|
||||
adapter: follower.adapter,
|
||||
degradedReason: "drilldown-policy-missing",
|
||||
message: `follower ${follower.id} registry is missing drillDown policy; apply the current config before drill-down`,
|
||||
query,
|
||||
policy: null,
|
||||
result: null,
|
||||
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
||||
parsedDownstreamCliOutput: false,
|
||||
next: {
|
||||
apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
||||
status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveJobTarget(registry: BranchFollowerRegistry, follower: FollowerSpec, query: string, sourceCommit: string | null): { namespace: string; jobName: string; stage: string } | null {
|
||||
if (!isStageAlias(query)) return { namespace: follower.target.namespace, jobName: query, stage: "explicit-job" };
|
||||
if (sourceCommit === null) return null;
|
||||
if (query === "git-mirror-sync" || query === "git-mirror-flush") {
|
||||
const action = query === "git-mirror-sync" ? "sync" : "flush";
|
||||
const target = gitMirrorJobTarget(follower, sourceCommit, action);
|
||||
return target === null ? null : { ...target, stage: query };
|
||||
}
|
||||
if (query === "control-plane-refresh" && follower.adapter === "hwlab-node-runtime") {
|
||||
return { namespace: registry.controller.namespace, jobName: nativeCapabilityJobName(follower.id, "control-plane-refresh", sourceCommit), stage: query };
|
||||
}
|
||||
if (follower.adapter === "agentrun-yaml-lane" && (query === "image-build" || query === "gitops-publish")) {
|
||||
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
||||
const prefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
||||
if (query === "image-build") return { namespace: spec.ci.namespace, jobName: `${prefix}-build-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query };
|
||||
return { namespace: spec.gitMirror.namespace, jobName: `${prefix}-gitops-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isStageAlias(value: string): boolean {
|
||||
return ["git-mirror-sync", "git-mirror-flush", "control-plane-refresh", "image-build", "gitops-publish"].includes(value);
|
||||
}
|
||||
|
||||
function gitMirrorJobTarget(follower: FollowerSpec, sourceCommit: string, action: "sync" | "flush"): { namespace: string; jobName: string } | null {
|
||||
const jobName = nativeCapabilityJobName(follower.id, action, sourceCommit);
|
||||
if (follower.adapter === "hwlab-node-runtime") {
|
||||
const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node);
|
||||
return { namespace: nodeRuntimeGitMirrorTarget(spec).namespace, jobName };
|
||||
}
|
||||
if (follower.adapter === "agentrun-yaml-lane") {
|
||||
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
||||
return { namespace: spec.gitMirror.namespace, jobName };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function nativeCapabilityJobName(followerId: string, action: string, sha: string): string {
|
||||
const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`;
|
||||
return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63);
|
||||
}
|
||||
|
||||
function safeK8sNameSegment(value: string): string {
|
||||
const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, "");
|
||||
return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, "");
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): { value: Record<string, unknown> | null; error: string | null } {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length === 0) return { value: null, error: "empty-stdout" };
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
|
||||
? { value: parsed as Record<string, unknown>, error: null }
|
||||
: { value: null, error: "stdout-json-not-object" };
|
||||
} catch (error) {
|
||||
return { value: null, error: `stdout-json-parse-failed: ${error instanceof Error ? error.message : String(error)}` };
|
||||
}
|
||||
}
|
||||
|
||||
function tailText(text: string, maxChars: number): string {
|
||||
return text.length <= maxChars ? text : text.slice(text.length - maxChars);
|
||||
}
|
||||
Reference in New Issue
Block a user