Codex
238 lines
11 KiB
TypeScript
238 lines
11 KiB
TypeScript
// SPEC: PJ2026-01060703 CI/CD branch follower job/runtime drill-down.
|
|
// Responsibility: follower-scoped read-only K8s Job and runtime workload visibility.
|
|
import type { CommandResult } from "./command";
|
|
import { resolveAgentRunLaneTarget } from "./agentrun-lanes";
|
|
import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types";
|
|
import { nativeCicdScriptLoadShell } from "./cicd-native-bundle";
|
|
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
|
import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe";
|
|
import { redactText, shQuote } from "./platform-infra-ops-library";
|
|
|
|
type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
|
|
|
|
export async function runBranchFollowerJobDrillDown(
|
|
registry: BranchFollowerRegistry,
|
|
follower: FollowerSpec,
|
|
options: ParsedOptions,
|
|
runKubeScript: KubeScriptRunner,
|
|
): Promise<Record<string, unknown>> {
|
|
const query = options.jobName;
|
|
if (query === null) throw new Error("job drill-down requires --job <stage|job-name>");
|
|
if (follower.drillDown === null) return missingPolicyPayload("job", follower, registry, options, query);
|
|
const sourceCommit = options.sourceCommit ?? null;
|
|
const target = resolveJobTarget(registry, follower, query, sourceCommit);
|
|
const policy = drillDownPolicy(follower, options);
|
|
if (target === null) {
|
|
return {
|
|
ok: false,
|
|
action: "job",
|
|
follower: follower.id,
|
|
adapter: follower.adapter,
|
|
degradedReason: "job-target-unresolved",
|
|
message: `job stage ${query} cannot be resolved for ${follower.id}; pass a concrete Kubernetes Job name`,
|
|
query: { job: query, sourceCommit },
|
|
policy,
|
|
result: null,
|
|
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
|
parsedDownstreamCliOutput: false,
|
|
next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` },
|
|
};
|
|
}
|
|
const script = [
|
|
"set -eu",
|
|
"tmpdir=$(mktemp -d)",
|
|
"cleanup() { rm -rf \"$tmpdir\"; }",
|
|
"trap cleanup EXIT INT TERM",
|
|
nativeCicdScriptLoadShell(["k8s-job-drilldown.mjs"]),
|
|
`JOB_NAMESPACE=${shQuote(target.namespace)}`,
|
|
`JOB_NAME=${shQuote(target.jobName)}`,
|
|
`STAGE_NAME=${shQuote(target.stage)}`,
|
|
`SOURCE_COMMIT=${shQuote(sourceCommit ?? "")}`,
|
|
`LOGS_TAIL_LINES=${policy.logsTailLines}`,
|
|
`MAX_LOG_BYTES=${policy.maxLogBytes}`,
|
|
`MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`,
|
|
`MAX_CONTAINERS=${policy.maxContainers}`,
|
|
"export JOB_NAMESPACE JOB_NAME STAGE_NAME SOURCE_COMMIT LOGS_TAIL_LINES MAX_LOG_BYTES MAX_MESSAGE_BYTES MAX_CONTAINERS",
|
|
"node \"$tmpdir/k8s-job-drilldown.mjs\"",
|
|
].join("\n");
|
|
const startedAt = Date.now();
|
|
const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000);
|
|
return drillDownPayload("job", follower, registry, options, policy, target, result, Date.now() - startedAt);
|
|
}
|
|
|
|
export async function runBranchFollowerRuntimeDrillDown(
|
|
registry: BranchFollowerRegistry,
|
|
follower: FollowerSpec,
|
|
options: ParsedOptions,
|
|
runKubeScript: KubeScriptRunner,
|
|
): Promise<Record<string, unknown>> {
|
|
if (follower.nativeStatus.runtime === null) throw new Error(`follower ${follower.id} has no runtime native status config`);
|
|
if (follower.drillDown === null) return missingPolicyPayload("runtime", follower, registry, options, options.workloadName);
|
|
const policy = drillDownPolicy(follower, options);
|
|
const workloads = follower.nativeStatus.runtime.workloads
|
|
.filter((item) => options.workloadName === null || item.name === options.workloadName)
|
|
.map((item) => ({
|
|
kind: item.kind,
|
|
name: item.name,
|
|
sourceCommit: item.sourceCommit,
|
|
}));
|
|
if (workloads.length === 0) throw new Error(`unknown runtime workload ${options.workloadName ?? "-"}`);
|
|
const workloadsB64 = Buffer.from(JSON.stringify(workloads), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
"tmpdir=$(mktemp -d)",
|
|
"cleanup() { rm -rf \"$tmpdir\"; }",
|
|
"trap cleanup EXIT INT TERM",
|
|
nativeCicdScriptLoadShell(["runtime-drilldown.mjs"]),
|
|
`RUNTIME_NAMESPACE=${shQuote(follower.nativeStatus.runtime.namespace)}`,
|
|
`EXPECTED_SHA=${shQuote(options.sourceCommit ?? "")}`,
|
|
`WORKLOADS_B64=${shQuote(workloadsB64)}`,
|
|
`MAX_MESSAGE_BYTES=${policy.maxMessageBytes}`,
|
|
`MAX_CONTAINERS=${policy.maxContainers}`,
|
|
"export RUNTIME_NAMESPACE EXPECTED_SHA WORKLOADS_B64 MAX_MESSAGE_BYTES MAX_CONTAINERS",
|
|
"node \"$tmpdir/runtime-drilldown.mjs\"",
|
|
].join("\n");
|
|
const startedAt = Date.now();
|
|
const result = runKubeScript(registry, options, script, "", policy.timeoutSeconds * 1000);
|
|
return drillDownPayload("runtime", follower, registry, options, policy, {
|
|
namespace: follower.nativeStatus.runtime.namespace,
|
|
workload: options.workloadName,
|
|
sourceCommit: options.sourceCommit ?? null,
|
|
}, result, Date.now() - startedAt);
|
|
}
|
|
|
|
function drillDownPolicy(follower: FollowerSpec, options: ParsedOptions): Record<string, number> {
|
|
const drillDown = follower.drillDown;
|
|
if (drillDown === null) throw new Error(`follower ${follower.id} registry is missing drillDown policy`);
|
|
return {
|
|
timeoutSeconds: options.timeoutSeconds ?? drillDown.taskRunTimeoutSeconds,
|
|
logsTailLines: options.logsTailLines ?? drillDown.logsTailLines,
|
|
maxLogBytes: options.maxLogBytes ?? drillDown.maxLogBytes,
|
|
maxMessageBytes: drillDown.maxMessageBytes,
|
|
maxContainers: drillDown.maxContainers,
|
|
};
|
|
}
|
|
|
|
function drillDownPayload(
|
|
action: "job" | "runtime",
|
|
follower: FollowerSpec,
|
|
registry: BranchFollowerRegistry,
|
|
options: ParsedOptions,
|
|
policy: Record<string, number>,
|
|
target: Record<string, unknown>,
|
|
command: CommandResult,
|
|
elapsedMs: number,
|
|
): Record<string, unknown> {
|
|
const parsedResult = command.exitCode === 0 ? parseJsonObject(command.stdout) : { value: null, error: "target-command-failed" };
|
|
const parsed = parsedResult.value;
|
|
const includeTails = command.exitCode !== 0 || parsed === null;
|
|
return {
|
|
ok: command.exitCode === 0 && parsed !== null && parsed.ok !== false,
|
|
action,
|
|
follower: follower.id,
|
|
adapter: follower.adapter,
|
|
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
|
parsedDownstreamCliOutput: false,
|
|
query: target,
|
|
policy,
|
|
result: parsed,
|
|
command: {
|
|
identity: {
|
|
route: options.inCluster ? "in-cluster" : registry.controller.kubeRoute,
|
|
script: action === "job" ? "scripts/native/cicd/k8s-job-drilldown.mjs" : "scripts/native/cicd/runtime-drilldown.mjs",
|
|
},
|
|
exitCode: command.exitCode,
|
|
timedOut: command.timedOut,
|
|
elapsedMs,
|
|
parseError: parsedResult.error,
|
|
stdoutTail: includeTails ? redactText(tailText(command.stdout, 1600)) : "",
|
|
stderrTail: includeTails ? redactText(tailText(command.stderr, 1200)) : "",
|
|
},
|
|
next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}` },
|
|
};
|
|
}
|
|
|
|
function missingPolicyPayload(action: string, follower: FollowerSpec, registry: BranchFollowerRegistry, options: ParsedOptions, query: string | null): Record<string, unknown> {
|
|
return {
|
|
ok: false,
|
|
action,
|
|
follower: follower.id,
|
|
adapter: follower.adapter,
|
|
degradedReason: "drilldown-policy-missing",
|
|
message: `follower ${follower.id} registry is missing drillDown policy; apply the current config before drill-down`,
|
|
query,
|
|
policy: null,
|
|
result: null,
|
|
statusAuthority: options.inCluster ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
|
|
parsedDownstreamCliOutput: false,
|
|
next: {
|
|
apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
|
status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
function resolveJobTarget(registry: BranchFollowerRegistry, follower: FollowerSpec, query: string, sourceCommit: string | null): { namespace: string; jobName: string; stage: string } | null {
|
|
if (!isStageAlias(query)) return { namespace: follower.target.namespace, jobName: query, stage: "explicit-job" };
|
|
if (sourceCommit === null) return null;
|
|
if (query === "git-mirror-sync" || query === "git-mirror-flush") {
|
|
const action = query === "git-mirror-sync" ? "sync" : "flush";
|
|
const target = gitMirrorJobTarget(follower, sourceCommit, action);
|
|
return target === null ? null : { ...target, stage: query };
|
|
}
|
|
if (query === "control-plane-refresh" && follower.adapter === "hwlab-node-runtime") {
|
|
return { namespace: registry.controller.namespace, jobName: nativeCapabilityJobName(follower.id, "control-plane-refresh", sourceCommit), stage: query };
|
|
}
|
|
if (follower.adapter === "agentrun-yaml-lane" && (query === "image-build" || query === "gitops-publish")) {
|
|
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
|
const prefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
|
if (query === "image-build") return { namespace: spec.ci.namespace, jobName: `${prefix}-build-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query };
|
|
return { namespace: spec.gitMirror.namespace, jobName: `${prefix}-gitops-${sourceCommit.slice(0, 12)}`.slice(0, 63), stage: query };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function isStageAlias(value: string): boolean {
|
|
return ["git-mirror-sync", "git-mirror-flush", "control-plane-refresh", "image-build", "gitops-publish"].includes(value);
|
|
}
|
|
|
|
function gitMirrorJobTarget(follower: FollowerSpec, sourceCommit: string, action: "sync" | "flush"): { namespace: string; jobName: string } | null {
|
|
const jobName = nativeCapabilityJobName(follower.id, action, sourceCommit);
|
|
if (follower.adapter === "hwlab-node-runtime") {
|
|
const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node);
|
|
return { namespace: nodeRuntimeGitMirrorTarget(spec).namespace, jobName };
|
|
}
|
|
if (follower.adapter === "agentrun-yaml-lane") {
|
|
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
|
return { namespace: spec.gitMirror.namespace, jobName };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function nativeCapabilityJobName(followerId: string, action: string, sha: string): string {
|
|
const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`;
|
|
return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63);
|
|
}
|
|
|
|
function safeK8sNameSegment(value: string): string {
|
|
const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, "");
|
|
return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, "");
|
|
}
|
|
|
|
function parseJsonObject(text: string): { value: Record<string, unknown> | null; error: string | null } {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return { value: null, error: "empty-stdout" };
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
|
|
? { value: parsed as Record<string, unknown>, error: null }
|
|
: { value: null, error: "stdout-json-not-object" };
|
|
} catch (error) {
|
|
return { value: null, error: `stdout-json-parse-failed: ${error instanceof Error ? error.message : String(error)}` };
|
|
}
|
|
}
|
|
|
|
function tailText(text: string, maxChars: number): string {
|
|
return text.length <= maxChars ? text : text.slice(text.length - maxChars);
|
|
}
|