fix: enforce runner retention before job create

This commit is contained in:
lyon
2026-06-19 22:35:55 +08:00
parent 56ecb493ef
commit 1be0e8e02f
5 changed files with 587 additions and 15 deletions
+72 -3
View File
@@ -8,6 +8,7 @@ import { asRecord, validateBackendProfile, validateCreateCommand, validateCreate
import { isBackendProfile } from "../common/backend-profiles.js";
import type { ApiErrorBody, ApiOkBody, CommandRecord, JsonRecord, JsonValue, QueueTaskRecord, RunEvent, RunRecord, SessionRecord } from "../common/types.js";
import { createKubernetesRunnerJob, type RunnerJobDefaults } from "./kubernetes-runner-job.js";
import type { RunnerRetentionOptions } from "./runner-retention.js";
import { dispatchQueueTask, refreshQueueTaskFromCore } from "./queue-dispatch.js";
import { buildRunResult } from "./result.js";
import { runnerJobStatusSummary } from "./runner-job-status.js";
@@ -37,6 +38,10 @@ function sessionPvcOptionsForRequest(serverDefaults: { kubectlHandler?: import("
function runnerJobDefaultsForRequest(defaults: ManagerServerOptions["runnerJobDefaults"], sourceCommit: string): RunnerJobDefaults {
const namespace = defaults?.namespace ?? process.env.AGENTRUN_RUNTIME_NAMESPACE ?? "agentrun-v01";
const serviceAccountName = defaults?.serviceAccountName ?? process.env.AGENTRUN_RUNNER_SERVICE_ACCOUNT ?? "agentrun-v01-runner";
const jobNamePrefix = defaults?.jobNamePrefix ?? process.env.AGENTRUN_RUNNER_JOB_NAME_PREFIX ?? serviceAccountName;
const lane = defaults?.lane ?? process.env.AGENTRUN_LANE ?? "v0.1";
const retention = runnerRetentionOptionsForRequest(defaults?.retention, namespace, jobNamePrefix, defaults?.kubectlCommand);
return {
namespace,
managerUrl: defaults?.managerUrl ?? process.env.AGENTRUN_INTERNAL_MGR_URL ?? `http://agentrun-mgr.${namespace}.svc.cluster.local:8080`,
@@ -45,10 +50,36 @@ function runnerJobDefaultsForRequest(defaults: ManagerServerOptions["runnerJobDe
sourceCommit,
...optionalStringRecord("envIdentity", defaults?.envIdentity ?? process.env.AGENTRUN_ENV_IDENTITY),
...optionalStringRecord("artifactCatalogFile", defaults?.artifactCatalogFile ?? process.env.AGENTRUN_ARTIFACT_CATALOG_FILE),
serviceAccountName: defaults?.serviceAccountName ?? process.env.AGENTRUN_RUNNER_SERVICE_ACCOUNT ?? "agentrun-v01-runner",
serviceAccountName,
jobNamePrefix,
lane,
...(defaults?.runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs: defaults.runnerIdleTimeoutMs } : optionalPositiveIntegerRecord("runnerIdleTimeoutMs", process.env.AGENTRUN_RUNNER_IDLE_TIMEOUT_MS)),
...(defaults?.kubectlCommand ? { kubectlCommand: defaults.kubectlCommand } : {}),
...(defaults?.unideskSshEndpointEnv ? { unideskSshEndpointEnv: defaults.unideskSshEndpointEnv } : {}),
...(retention ? { retention } : {}),
};
}
function runnerRetentionOptionsForRequest(defaults: RunnerRetentionOptions | undefined, namespace: string, jobNamePrefix: string, kubectlCommand?: string): RunnerRetentionOptions | undefined {
if (defaults) return { ...defaults, namespace, ...(kubectlCommand ? { kubectlCommand } : {}) };
const maxRunners = optionalPositiveInteger("AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", process.env.AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS);
if (maxRunners === undefined) return undefined;
const activeHeartbeatMaxAgeMs = requiredPositiveInteger("AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", process.env.AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS);
const cleanupOrder = process.env.AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER ?? "oldest-inactive-last-active-first";
if (cleanupOrder !== "oldest-inactive-last-active-first") throw new AgentRunError("schema-invalid", "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER is unsupported", { httpStatus: 500 });
const jobNamePrefixes = stringListEnv("AGENTRUN_RUNNER_RETENTION_JOB_NAME_PREFIXES", process.env.AGENTRUN_RUNNER_RETENTION_JOB_NAME_PREFIXES);
return {
namespace,
maxRunners,
cleanupOrder,
activeHeartbeatMaxAgeMs,
matchLabels: jsonRecordEnv("AGENTRUN_RUNNER_RETENTION_MATCH_LABELS_JSON", process.env.AGENTRUN_RUNNER_RETENTION_MATCH_LABELS_JSON),
jobNamePrefixes: jobNamePrefixes.length > 0 ? jobNamePrefixes : [jobNamePrefix],
ageBasedCleanup: {
enabled: booleanEnv("AGENTRUN_RUNNER_RETENTION_AGE_BASED_CLEANUP_ENABLED", process.env.AGENTRUN_RUNNER_RETENTION_AGE_BASED_CLEANUP_ENABLED ?? "false"),
maxAgeHours: optionalPositiveInteger("AGENTRUN_RUNNER_RETENTION_AGE_BASED_MAX_AGE_HOURS", process.env.AGENTRUN_RUNNER_RETENTION_AGE_BASED_MAX_AGE_HOURS) ?? null,
},
...(kubectlCommand ? { kubectlCommand } : {}),
};
}
@@ -65,9 +96,12 @@ export interface ManagerServerOptions {
envIdentity?: string;
artifactCatalogFile?: string;
serviceAccountName?: string;
jobNamePrefix?: string;
lane?: string;
runnerIdleTimeoutMs?: number;
kubectlCommand?: string;
unideskSshEndpointEnv?: JsonRecord;
retention?: RunnerRetentionOptions;
};
sessionPvcOptions?: { kubectlHandler?: import("./session-pvc.js").KubectlHandler; kubectlCommand?: string; storageClassName?: string; size?: string };
providerProfileOptions?: { namespace?: string; kubectlCommand?: string };
@@ -926,10 +960,45 @@ function optionalStringRecord(key: string, value: unknown): JsonRecord {
}
function optionalPositiveIntegerRecord(key: string, value: unknown): JsonRecord {
if (value === undefined || value === null || value === "") return {};
const parsed = optionalPositiveInteger(key, value);
if (parsed === undefined) return {};
return { [key]: parsed };
}
function optionalPositiveInteger(key: string, value: unknown): number | undefined {
if (value === undefined || value === null || value === "") return undefined;
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed <= 0) throw new AgentRunError("schema-invalid", `${key} must be a positive integer`, { httpStatus: 400 });
return { [key]: parsed };
return parsed;
}
function requiredPositiveInteger(key: string, value: unknown): number {
const parsed = optionalPositiveInteger(key, value);
if (parsed === undefined) throw new AgentRunError("schema-invalid", `${key} is required when runner retention is enabled`, { httpStatus: 500 });
return parsed;
}
function jsonRecordEnv(key: string, value: unknown): Record<string, string> {
if (typeof value !== "string" || value.trim().length === 0) throw new AgentRunError("schema-invalid", `${key} is required when runner retention is enabled`, { httpStatus: 500 });
const parsed = JSON.parse(value) as unknown;
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new AgentRunError("schema-invalid", `${key} must be a JSON object`, { httpStatus: 500 });
const out: Record<string, string> = {};
for (const [entryKey, entryValue] of Object.entries(parsed)) {
if (typeof entryValue !== "string" || entryValue.length === 0) throw new AgentRunError("schema-invalid", `${key}.${entryKey} must be a non-empty string`, { httpStatus: 500 });
out[entryKey] = entryValue;
}
return out;
}
function stringListEnv(key: string, value: unknown): string[] {
if (typeof value !== "string") throw new AgentRunError("schema-invalid", `${key} is required when runner retention is enabled`, { httpStatus: 500 });
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
}
function booleanEnv(key: string, value: string): boolean {
if (value === "true") return true;
if (value === "false") return false;
throw new AgentRunError("schema-invalid", `${key} must be true or false`, { httpStatus: 500 });
}
function normalizeError(error: unknown): AgentRunError {