Codex
312 lines
13 KiB
TypeScript
312 lines
13 KiB
TypeScript
import type { AgentRunLaneSpec } from "../agentrun-lanes";
|
|
|
|
import { cleanupRunnersFactsNodeScript } from "./secrets";
|
|
import { shQuote } from "./utils";
|
|
|
|
export interface RunnerJobObservationIdentity {
|
|
readonly runnerJobId: string;
|
|
readonly runId: string;
|
|
readonly commandId: string;
|
|
readonly runnerId: string;
|
|
readonly namespace: string;
|
|
readonly jobName: string;
|
|
}
|
|
|
|
export function runnerJobObservationScript(spec: AgentRunLaneSpec, identity: RunnerJobObservationIdentity): string {
|
|
const identityJsonB64 = Buffer.from(JSON.stringify(identity), "utf8").toString("base64");
|
|
return [
|
|
"set -eu",
|
|
`namespace=${shQuote(spec.runtime.namespace)}`,
|
|
`manager_deployment=${shQuote(spec.runtime.managerDeployment)}`,
|
|
`runner_job_id=${shQuote(identity.runnerJobId)}`,
|
|
`job_name=${shQuote(identity.jobName)}`,
|
|
`identity_json_b64=${shQuote(identityJsonB64)}`,
|
|
"tmp_dir=$(mktemp -d)",
|
|
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
|
"facts_exit=0",
|
|
"set +e",
|
|
"kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" RETENTION_RUNNER_JOB_ID=\"$runner_job_id\" sh -lc 'cat >/tmp/agentrun-runner-observation.mjs && bun /tmp/agentrun-runner-observation.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'",
|
|
cleanupRunnersFactsNodeScript(),
|
|
"NODE",
|
|
"facts_exit=$?",
|
|
"set -e",
|
|
"if [ \"$facts_exit\" -ne 0 ]; then printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"; fi",
|
|
"job_exit=0",
|
|
"set +e",
|
|
"kubectl -n \"$namespace\" get job \"$job_name\" -o json > \"$tmp_dir/job.json\" 2> \"$tmp_dir/job.err\"",
|
|
"job_exit=$?",
|
|
"set -e",
|
|
"if [ \"$job_exit\" -ne 0 ]; then printf '%s\\n' '{}' > \"$tmp_dir/job.json\"; fi",
|
|
"pods_exit=0",
|
|
"set +e",
|
|
"kubectl -n \"$namespace\" get pods -l \"job-name=$job_name\" -o json > \"$tmp_dir/pods.json\" 2> \"$tmp_dir/pods.err\"",
|
|
"pods_exit=$?",
|
|
"set -e",
|
|
"if [ \"$pods_exit\" -ne 0 ]; then printf '%s\\n' '{\"items\":[]}' > \"$tmp_dir/pods.json\"; fi",
|
|
"node -e 'const p=JSON.parse(require(\"node:fs\").readFileSync(process.argv[1],\"utf8\")); for(const x of Array.isArray(p.items)?p.items.slice(0,8):[]){const n=x?.metadata?.name;if(typeof n===\"string\"&&n)console.log(n)}' \"$tmp_dir/pods.json\" > \"$tmp_dir/pod-names.txt\"",
|
|
"events_failed=0",
|
|
"event_index=0",
|
|
"while IFS= read -r pod_name; do",
|
|
" [ -n \"$pod_name\" ] || continue",
|
|
" event_file=\"$tmp_dir/event-$event_index.json\"",
|
|
" set +e",
|
|
" kubectl -n \"$namespace\" get events --field-selector \"involvedObject.kind=Pod,involvedObject.name=$pod_name,type=Warning\" -o json > \"$event_file\" 2> \"$tmp_dir/event-$event_index.err\"",
|
|
" event_exit=$?",
|
|
" set -e",
|
|
" if [ \"$event_exit\" -ne 0 ]; then events_failed=$((events_failed + 1)); printf '%s\\n' '{\"items\":[]}' > \"$event_file\"; fi",
|
|
" event_index=$((event_index + 1))",
|
|
"done < \"$tmp_dir/pod-names.txt\"",
|
|
"IDENTITY_JSON_B64=\"$identity_json_b64\" FACTS_EXIT=\"$facts_exit\" JOB_EXIT=\"$job_exit\" PODS_EXIT=\"$pods_exit\" EVENTS_FAILED=\"$events_failed\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE'",
|
|
runnerJobObservationNodeScript(),
|
|
"NODE",
|
|
].join("\n");
|
|
}
|
|
|
|
function runnerJobObservationNodeScript(): string {
|
|
return String.raw`
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
const tmp = process.env.TMP_DIR;
|
|
const expected = JSON.parse(Buffer.from(process.env.IDENTITY_JSON_B64, "base64").toString("utf8"));
|
|
const factsExit = Number(process.env.FACTS_EXIT);
|
|
const jobExit = Number(process.env.JOB_EXIT);
|
|
const podsExit = Number(process.env.PODS_EXIT);
|
|
const eventsFailed = Number(process.env.EVENTS_FAILED);
|
|
|
|
function readJson(name) {
|
|
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
|
}
|
|
|
|
function iso(value) {
|
|
if (typeof value !== "string" || value.length === 0) return null;
|
|
const ms = Date.parse(value);
|
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
|
}
|
|
|
|
function ageMs(value) {
|
|
const ms = Date.parse(value || "");
|
|
return Number.isFinite(ms) ? Math.max(0, Date.now() - ms) : null;
|
|
}
|
|
|
|
function bounded(value, max = 320) {
|
|
if (typeof value !== "string" || value.length === 0) return null;
|
|
const oneLine = value.replace(/\s+/g, " ").trim();
|
|
return oneLine.length <= max ? oneLine : oneLine.slice(0, max);
|
|
}
|
|
|
|
function annotationsOf(value) {
|
|
return value?.metadata?.annotations && typeof value.metadata.annotations === "object" ? value.metadata.annotations : {};
|
|
}
|
|
|
|
function stateKind(state) {
|
|
if (state?.running) return "running";
|
|
if (state?.terminated) return "terminated";
|
|
if (state?.waiting) return "waiting";
|
|
return "unknown";
|
|
}
|
|
|
|
function waitingFromStatus(status, source) {
|
|
const waiting = status?.state?.waiting;
|
|
if (!waiting || typeof waiting.reason !== "string" || waiting.reason.length === 0) return [];
|
|
return [{
|
|
source,
|
|
name: typeof status.name === "string" ? status.name : null,
|
|
reason: waiting.reason,
|
|
message: bounded(waiting.message),
|
|
valuesPrinted: false,
|
|
}];
|
|
}
|
|
|
|
function warningFromEvent(event) {
|
|
if (event?.type !== "Warning" || typeof event?.reason !== "string" || event.reason.length === 0) return null;
|
|
return {
|
|
source: "event",
|
|
name: typeof event?.metadata?.name === "string" ? event.metadata.name : null,
|
|
reason: event.reason,
|
|
message: bounded(event.message),
|
|
count: Number.isFinite(event.count) ? event.count : null,
|
|
lastObservedAt: iso(event?.series?.lastObservedTime || event?.eventTime || event?.lastTimestamp || event?.firstTimestamp),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function identityMatches(value) {
|
|
const annotations = annotationsOf(value);
|
|
const optionalRunnerJobId = annotations["agentrun.pikastech.local/runner-job-id"];
|
|
return annotations["agentrun.pikastech.local/run-id"] === expected.runId
|
|
&& annotations["agentrun.pikastech.local/command-id"] === expected.commandId
|
|
&& annotations["agentrun.pikastech.local/runner-id"] === expected.runnerId
|
|
&& (optionalRunnerJobId === undefined || optionalRunnerJobId === expected.runnerJobId);
|
|
}
|
|
|
|
const facts = readJson("runner-facts.json");
|
|
const factItems = Array.isArray(facts.items) ? facts.items : [];
|
|
const fact = factItems.find((item) => item?.id === expected.runnerJobId) || null;
|
|
const job = readJson("job.json");
|
|
const podsPayload = readJson("pods.json");
|
|
const pods = Array.isArray(podsPayload.items) ? podsPayload.items.slice(0, 8) : [];
|
|
const eventDocuments = fs.readdirSync(tmp)
|
|
.filter((name) => /^event-\d+\.json$/u.test(name))
|
|
.sort()
|
|
.map(readJson);
|
|
const warningEvents = eventDocuments.flatMap((value) => Array.isArray(value.items) ? value.items : []);
|
|
const warningsByPod = new Map();
|
|
for (const event of warningEvents) {
|
|
const podName = event?.involvedObject?.kind === "Pod" && typeof event?.involvedObject?.name === "string"
|
|
? event.involvedObject.name
|
|
: null;
|
|
if (podName === null) continue;
|
|
const items = warningsByPod.get(podName) || [];
|
|
items.push(event);
|
|
warningsByPod.set(podName, items);
|
|
}
|
|
|
|
const podObservations = pods.map((pod) => {
|
|
const statuses = Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : [];
|
|
const initStatuses = Array.isArray(pod?.status?.initContainerStatuses) ? pod.status.initContainerStatuses : [];
|
|
const runner = statuses.find((status) => status?.name === "runner") || null;
|
|
const conditions = Array.isArray(pod?.status?.conditions) ? pod.status.conditions : [];
|
|
const eventWarnings = (warningsByPod.get(pod?.metadata?.name || "") || []).map(warningFromEvent).filter(Boolean);
|
|
const waiting = [
|
|
...statuses.flatMap((status) => waitingFromStatus(status, "container")),
|
|
...initStatuses.flatMap((status) => waitingFromStatus(status, "init-container")),
|
|
...conditions.flatMap((condition) => condition?.status === "False" && typeof condition?.reason === "string" && condition.reason.length > 0
|
|
? [{ source: "condition", name: condition.type || null, reason: condition.reason, message: bounded(condition.message), valuesPrinted: false }]
|
|
: []),
|
|
...eventWarnings,
|
|
];
|
|
const ready = conditions.some((condition) => condition?.type === "Ready" && condition?.status === "True");
|
|
return {
|
|
name: pod?.metadata?.name || null,
|
|
uid: pod?.metadata?.uid || null,
|
|
resourceVersion: pod?.metadata?.resourceVersion || null,
|
|
phase: pod?.status?.phase || null,
|
|
createdAt: iso(pod?.metadata?.creationTimestamp),
|
|
ageMs: ageMs(pod?.metadata?.creationTimestamp),
|
|
ready,
|
|
identityMatches: identityMatches(pod),
|
|
runnerContainerState: stateKind(runner?.state || {}),
|
|
runnerContainerReady: runner?.ready === true,
|
|
runnerContainerRestartCount: Number(runner?.restartCount || 0),
|
|
waiting: waiting.slice(0, 8),
|
|
waitingOmittedCount: Math.max(0, waiting.length - 8),
|
|
eventFactsAvailable: eventsFailed === 0,
|
|
valuesPrinted: false,
|
|
};
|
|
});
|
|
|
|
const managerIdentityMatches = fact !== null
|
|
&& fact.runId === expected.runId
|
|
&& fact.commandId === expected.commandId
|
|
&& fact.runnerId === expected.runnerId
|
|
&& fact.namespace === expected.namespace
|
|
&& fact.jobName === expected.jobName;
|
|
const jobPresent = jobExit === 0 && job?.metadata?.name === expected.jobName;
|
|
const jobIdentityMatches = jobPresent && identityMatches(job);
|
|
const podIdentityMatches = podObservations.length > 0 && podObservations.every((pod) => pod.identityMatches === true);
|
|
const podPhases = podObservations.map((pod) => pod.phase).filter((value) => typeof value === "string");
|
|
const waitingReasons = [...new Set(podObservations.flatMap((pod) => pod.waiting.map((item) => item.reason)).filter(Boolean))].slice(0, 8);
|
|
const state = podPhases.includes("Pending")
|
|
? "Pending"
|
|
: podPhases.includes("Running")
|
|
? "Running"
|
|
: podPhases.includes("Failed")
|
|
? "Failed"
|
|
: podPhases.includes("Succeeded")
|
|
? "Succeeded"
|
|
: jobPresent
|
|
? "Unknown"
|
|
: "NotFound";
|
|
const classification = state === "Pending" && waitingReasons.includes("FailedMount")
|
|
? "pending-failed-mount"
|
|
: state.toLowerCase();
|
|
const warningEventsAvailable = podObservations.length > 0 && eventsFailed === 0;
|
|
const ok = factsExit === 0
|
|
&& facts.ok === true
|
|
&& factItems.length === 1
|
|
&& managerIdentityMatches
|
|
&& jobPresent
|
|
&& jobIdentityMatches
|
|
&& podsExit === 0
|
|
&& podIdentityMatches
|
|
&& warningEventsAvailable;
|
|
const reason = ok
|
|
? null
|
|
: factsExit !== 0 || facts.ok !== true
|
|
? "manager-runner-facts-query-failed"
|
|
: factItems.length !== 1 || !managerIdentityMatches
|
|
? "manager-runner-identity-mismatch"
|
|
: !jobPresent
|
|
? "runner-job-not-found"
|
|
: !jobIdentityMatches
|
|
? "runner-job-identity-mismatch"
|
|
: podsExit !== 0 || !podIdentityMatches
|
|
? "runner-pod-observation-incomplete"
|
|
: "runner-warning-event-observation-incomplete";
|
|
|
|
console.log(JSON.stringify({
|
|
ok,
|
|
reason,
|
|
observedAt: new Date().toISOString(),
|
|
target: { namespace: process.env.NAMESPACE },
|
|
managerFacts: {
|
|
ok: factsExit === 0 && facts.ok === true && factItems.length === 1,
|
|
queryMode: facts.queryMode || null,
|
|
runnerJobId: expected.runnerJobId,
|
|
itemCount: factItems.length,
|
|
identityMatches: managerIdentityMatches,
|
|
valuesPrinted: false,
|
|
},
|
|
kubernetesFacts: {
|
|
jobQuery: "exact-name",
|
|
podQuery: "exact-job-name-label",
|
|
warningEventQuery: "exact-pod-name",
|
|
jobPresent,
|
|
podCount: podObservations.length,
|
|
warningEventCount: warningEvents.length,
|
|
warningEventsAvailable,
|
|
valuesPrinted: false,
|
|
},
|
|
runner: {
|
|
name: expected.jobName,
|
|
namespace: expected.namespace,
|
|
runId: expected.runId,
|
|
commandId: expected.commandId,
|
|
runnerJobId: expected.runnerJobId,
|
|
runnerId: expected.runnerId,
|
|
resourceUid: job?.metadata?.uid || null,
|
|
resourceVersion: job?.metadata?.resourceVersion || null,
|
|
createdAt: iso(job?.metadata?.creationTimestamp),
|
|
lastActiveAt: fact?.runnerHeartbeatAt || fact?.runUpdatedAt || fact?.runnerJobUpdatedAt || null,
|
|
lastActiveAgeMs: ageMs(fact?.runnerHeartbeatAt || fact?.runUpdatedAt || fact?.runnerJobUpdatedAt),
|
|
runStatus: fact?.runStatus || null,
|
|
runClaimedBy: fact?.runClaimedBy || null,
|
|
commandState: fact?.commandState || null,
|
|
heartbeatAt: fact?.runnerHeartbeatAt || null,
|
|
heartbeatAgeMs: ageMs(fact?.runnerHeartbeatAt),
|
|
jobStatus: {
|
|
active: Number(job?.status?.active || 0),
|
|
succeeded: Number(job?.status?.succeeded || 0),
|
|
failed: Number(job?.status?.failed || 0),
|
|
ready: Number(job?.status?.ready || 0),
|
|
},
|
|
podPhases,
|
|
waitingReasons,
|
|
podObservations: podObservations.slice(0, 4),
|
|
podObservationOmittedCount: Math.max(0, podObservations.length - 4),
|
|
identityComplete: managerIdentityMatches && jobIdentityMatches && podIdentityMatches,
|
|
managerFactsComplete: factsExit === 0 && facts.ok === true && factItems.length === 1,
|
|
podFactsComplete: podsExit === 0 && podIdentityMatches && warningEventsAvailable,
|
|
state,
|
|
classification,
|
|
cleanupAuthority: "agentrun-manager-pre-create-retention",
|
|
cleanupDecisionComputed: false,
|
|
mutation: false,
|
|
valuesPrinted: false,
|
|
},
|
|
mutation: false,
|
|
valuesPrinted: false,
|
|
}));
|
|
`;
|
|
}
|