mirrors/pikasTech-agentrun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: protect runner cleanup artifacts

Browse Source
This commit is contained in:
lyon
2026-06-20 16:27:25 +08:00
parent f95822e8ba
commit 8d297a9823
4 changed files with 130 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mgr/kubernetes-runner-job.ts
+1
View File
@@ -234,6 +234,7 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
workReady: staticWorkReadyCapabilitySummary(),
retention: {
ttlSecondsAfterFinished: render.ttlSecondsAfterFinished,
ttlPolicy: render.ttlPolicy,
runnerIdleTimeoutMs: render.runnerIdleTimeoutMs,
preCreateCleanup: preCreateRetention,
},
src/mgr/runner-retention.ts
+36 -4
View File
@@ -32,6 +32,7 @@ export interface RunnerRetentionSummary extends JsonRecord {
orphanNonTerminalRunnerPodCountBefore: number;
inactiveCandidateCount: number;
protectedActiveRunnerCount: number;
protectedActiveRunners: JsonRecord[];
selectedRunnerCount: number;
deletedRunnerJobCount: number;
deletedRunnerPodCount: number;
@@ -90,7 +91,8 @@ export async function enforceRunnerRetentionBeforeCreate(input: { store: AgentRu
.filter((item) => !item.protectedActive)
.sort(compareCandidates);
const selected = candidates.slice(0, requiredDeleteCount);
const protectedActiveRunnerCount = before.jobs.filter((item) => item.protectedActive).length + before.pods.filter((item) => item.protectedActive).length;
const protectedActiveRunners = [...before.jobs, ...before.pods].filter((item) => item.protectedActive);
const protectedActiveRunnerCount = protectedActiveRunners.length;
if (requiredDeleteCount > 0 && selected.length < requiredDeleteCount) {
throw new AgentRunError("infra-failed", "runner retention cannot safely make room for a new runner", {
httpStatus: 409,
@@ -103,7 +105,9 @@ export async function enforceRunnerRetentionBeforeCreate(input: { store: AgentRu
requiredDeleteCount,
inactiveCandidateCount: candidates.length,
protectedActiveRunnerCount,
protectedActiveRunners: protectedActiveRunners.map(protectedRunnerSummary),
activeRunRisk: protectedActiveRunnerCount > 0,
cleanupPolicy: cleanupPolicySummary(),
valuesPrinted: false,
},
});
@@ -135,12 +139,14 @@ export async function enforceRunnerRetentionBeforeCreate(input: { store: AgentRu
orphanNonTerminalRunnerPodCountBefore: before.orphanNonTerminalRunnerPodCount,
inactiveCandidateCount: candidates.length,
protectedActiveRunnerCount,
protectedActiveRunners: protectedActiveRunners.map(protectedRunnerSummary),
selectedRunnerCount: selected.length,
deletedRunnerJobCount,
deletedRunnerPodCount,
deletedAssociatedResourceCount,
activeRunRisk: protectedActiveRunnerCount > 0,
reason: requiredDeleteCount === 0 ? "within-limit" : "pre-create-retention",
cleanupPolicy: cleanupPolicySummary(),
selected: selected.map((item) => ({ resourceKind: item.resourceKind, name: item.name, jobName: item.jobName, candidateKind: item.candidateKind, protectedActive: false, lastActiveAt: new Date(item.lastActiveAtMs).toISOString(), valuesPrinted: false })),
valuesPrinted: false,
} satisfies RunnerRetentionSummary;
@@ -236,9 +242,9 @@ async function activityFor(store: AgentRunStore, input: { runId: string | null;
try { command = await store.getCommand(input.commandId); } catch { command = null; }
}
const lastActiveAtMs = Math.max(input.createdAtMs, isoMs(run?.updatedAt), isoMs(command?.updatedAt), isoMs(run?.leaseExpiresAt));
const active = run && !isTerminalRunStatus(run.status) && command && !isTerminalCommandState(command.state) && input.runnerId && run.claimedBy === input.runnerId && heartbeatFresh(run.leaseExpiresAt, input.activeHeartbeatMaxAgeMs);
if (active) {
return { runId: input.runId, commandId: input.commandId, runnerId: input.runnerId, protectedActive: true, protectedReason: "fresh-active-run", candidateKind: "inactive", lastActiveAtMs };
if (run && command && !isTerminalRunStatus(run.status) && !isTerminalCommandState(command.state) && !input.terminal) {
const freshClaim = input.runnerId && run.claimedBy === input.runnerId && heartbeatFresh(run.leaseExpiresAt, input.activeHeartbeatMaxAgeMs);
return { runId: input.runId, commandId: input.commandId, runnerId: input.runnerId, protectedActive: true, protectedReason: freshClaim ? "fresh-active-run" : "nonterminal-runner-resource", candidateKind: "inactive", lastActiveAtMs };
}
const candidateKind: CandidateKind = command && isTerminalCommandState(command.state) && !input.terminal ? "idle" : input.terminal || (run !== null && isTerminalRunStatus(run.status)) ? "terminal" : "inactive";
return { runId: input.runId, commandId: input.commandId, runnerId: input.runnerId, protectedActive: false, protectedReason: null, candidateKind, lastActiveAtMs };
@@ -292,6 +298,32 @@ function compareCandidates(left: RunnerResourceEntry, right: RunnerResourceEntry
return left.name.localeCompare(right.name);
}
function protectedRunnerSummary(item: RunnerResourceEntry): JsonRecord {
return {
resourceKind: item.resourceKind,
name: item.name,
jobName: item.jobName,
runId: item.runId,
commandId: item.commandId,
runnerId: item.runnerId,
protectedReason: item.protectedReason,
terminal: item.terminal,
podPhase: item.podPhase,
lastActiveAt: new Date(item.lastActiveAtMs).toISOString(),
valuesPrinted: false,
};
}
function cleanupPolicySummary(): JsonRecord {
return {
mode: "db-ledger-and-k8s-observation",
forceActive: false,
activeProtection: "protect non-terminal DB run/command while Kubernetes runner resource is non-terminal; fresh heartbeat is not required during manager rolling recovery",
deletionOrder: "idle, terminal, inactive by lastActiveAt",
valuesPrinted: false,
};
}
function labelSelector(labels: Record<string, string>): string {
return Object.entries(labels).map(([key, value]) => `${key}=${value}`).join(",");
}