fix: enforce runner retention before job create
This commit is contained in:
+72
-3
@@ -8,6 +8,7 @@ import { asRecord, validateBackendProfile, validateCreateCommand, validateCreate
|
||||
import { isBackendProfile } from "../common/backend-profiles.js";
|
||||
import type { ApiErrorBody, ApiOkBody, CommandRecord, JsonRecord, JsonValue, QueueTaskRecord, RunEvent, RunRecord, SessionRecord } from "../common/types.js";
|
||||
import { createKubernetesRunnerJob, type RunnerJobDefaults } from "./kubernetes-runner-job.js";
|
||||
import type { RunnerRetentionOptions } from "./runner-retention.js";
|
||||
import { dispatchQueueTask, refreshQueueTaskFromCore } from "./queue-dispatch.js";
|
||||
import { buildRunResult } from "./result.js";
|
||||
import { runnerJobStatusSummary } from "./runner-job-status.js";
|
||||
@@ -37,6 +38,10 @@ function sessionPvcOptionsForRequest(serverDefaults: { kubectlHandler?: import("
|
||||
|
||||
function runnerJobDefaultsForRequest(defaults: ManagerServerOptions["runnerJobDefaults"], sourceCommit: string): RunnerJobDefaults {
|
||||
const namespace = defaults?.namespace ?? process.env.AGENTRUN_RUNTIME_NAMESPACE ?? "agentrun-v01";
|
||||
const serviceAccountName = defaults?.serviceAccountName ?? process.env.AGENTRUN_RUNNER_SERVICE_ACCOUNT ?? "agentrun-v01-runner";
|
||||
const jobNamePrefix = defaults?.jobNamePrefix ?? process.env.AGENTRUN_RUNNER_JOB_NAME_PREFIX ?? serviceAccountName;
|
||||
const lane = defaults?.lane ?? process.env.AGENTRUN_LANE ?? "v0.1";
|
||||
const retention = runnerRetentionOptionsForRequest(defaults?.retention, namespace, jobNamePrefix, defaults?.kubectlCommand);
|
||||
return {
|
||||
namespace,
|
||||
managerUrl: defaults?.managerUrl ?? process.env.AGENTRUN_INTERNAL_MGR_URL ?? `http://agentrun-mgr.${namespace}.svc.cluster.local:8080`,
|
||||
@@ -45,10 +50,36 @@ function runnerJobDefaultsForRequest(defaults: ManagerServerOptions["runnerJobDe
|
||||
sourceCommit,
|
||||
...optionalStringRecord("envIdentity", defaults?.envIdentity ?? process.env.AGENTRUN_ENV_IDENTITY),
|
||||
...optionalStringRecord("artifactCatalogFile", defaults?.artifactCatalogFile ?? process.env.AGENTRUN_ARTIFACT_CATALOG_FILE),
|
||||
serviceAccountName: defaults?.serviceAccountName ?? process.env.AGENTRUN_RUNNER_SERVICE_ACCOUNT ?? "agentrun-v01-runner",
|
||||
serviceAccountName,
|
||||
jobNamePrefix,
|
||||
lane,
|
||||
...(defaults?.runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs: defaults.runnerIdleTimeoutMs } : optionalPositiveIntegerRecord("runnerIdleTimeoutMs", process.env.AGENTRUN_RUNNER_IDLE_TIMEOUT_MS)),
|
||||
...(defaults?.kubectlCommand ? { kubectlCommand: defaults.kubectlCommand } : {}),
|
||||
...(defaults?.unideskSshEndpointEnv ? { unideskSshEndpointEnv: defaults.unideskSshEndpointEnv } : {}),
|
||||
...(retention ? { retention } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function runnerRetentionOptionsForRequest(defaults: RunnerRetentionOptions | undefined, namespace: string, jobNamePrefix: string, kubectlCommand?: string): RunnerRetentionOptions | undefined {
|
||||
if (defaults) return { ...defaults, namespace, ...(kubectlCommand ? { kubectlCommand } : {}) };
|
||||
const maxRunners = optionalPositiveInteger("AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", process.env.AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS);
|
||||
if (maxRunners === undefined) return undefined;
|
||||
const activeHeartbeatMaxAgeMs = requiredPositiveInteger("AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", process.env.AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS);
|
||||
const cleanupOrder = process.env.AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER ?? "oldest-inactive-last-active-first";
|
||||
if (cleanupOrder !== "oldest-inactive-last-active-first") throw new AgentRunError("schema-invalid", "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER is unsupported", { httpStatus: 500 });
|
||||
const jobNamePrefixes = stringListEnv("AGENTRUN_RUNNER_RETENTION_JOB_NAME_PREFIXES", process.env.AGENTRUN_RUNNER_RETENTION_JOB_NAME_PREFIXES);
|
||||
return {
|
||||
namespace,
|
||||
maxRunners,
|
||||
cleanupOrder,
|
||||
activeHeartbeatMaxAgeMs,
|
||||
matchLabels: jsonRecordEnv("AGENTRUN_RUNNER_RETENTION_MATCH_LABELS_JSON", process.env.AGENTRUN_RUNNER_RETENTION_MATCH_LABELS_JSON),
|
||||
jobNamePrefixes: jobNamePrefixes.length > 0 ? jobNamePrefixes : [jobNamePrefix],
|
||||
ageBasedCleanup: {
|
||||
enabled: booleanEnv("AGENTRUN_RUNNER_RETENTION_AGE_BASED_CLEANUP_ENABLED", process.env.AGENTRUN_RUNNER_RETENTION_AGE_BASED_CLEANUP_ENABLED ?? "false"),
|
||||
maxAgeHours: optionalPositiveInteger("AGENTRUN_RUNNER_RETENTION_AGE_BASED_MAX_AGE_HOURS", process.env.AGENTRUN_RUNNER_RETENTION_AGE_BASED_MAX_AGE_HOURS) ?? null,
|
||||
},
|
||||
...(kubectlCommand ? { kubectlCommand } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -65,9 +96,12 @@ export interface ManagerServerOptions {
|
||||
envIdentity?: string;
|
||||
artifactCatalogFile?: string;
|
||||
serviceAccountName?: string;
|
||||
jobNamePrefix?: string;
|
||||
lane?: string;
|
||||
runnerIdleTimeoutMs?: number;
|
||||
kubectlCommand?: string;
|
||||
unideskSshEndpointEnv?: JsonRecord;
|
||||
retention?: RunnerRetentionOptions;
|
||||
};
|
||||
sessionPvcOptions?: { kubectlHandler?: import("./session-pvc.js").KubectlHandler; kubectlCommand?: string; storageClassName?: string; size?: string };
|
||||
providerProfileOptions?: { namespace?: string; kubectlCommand?: string };
|
||||
@@ -926,10 +960,45 @@ function optionalStringRecord(key: string, value: unknown): JsonRecord {
|
||||
}
|
||||
|
||||
function optionalPositiveIntegerRecord(key: string, value: unknown): JsonRecord {
|
||||
if (value === undefined || value === null || value === "") return {};
|
||||
const parsed = optionalPositiveInteger(key, value);
|
||||
if (parsed === undefined) return {};
|
||||
return { [key]: parsed };
|
||||
}
|
||||
|
||||
function optionalPositiveInteger(key: string, value: unknown): number | undefined {
|
||||
if (value === undefined || value === null || value === "") return undefined;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) throw new AgentRunError("schema-invalid", `${key} must be a positive integer`, { httpStatus: 400 });
|
||||
return { [key]: parsed };
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function requiredPositiveInteger(key: string, value: unknown): number {
|
||||
const parsed = optionalPositiveInteger(key, value);
|
||||
if (parsed === undefined) throw new AgentRunError("schema-invalid", `${key} is required when runner retention is enabled`, { httpStatus: 500 });
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function jsonRecordEnv(key: string, value: unknown): Record<string, string> {
|
||||
if (typeof value !== "string" || value.trim().length === 0) throw new AgentRunError("schema-invalid", `${key} is required when runner retention is enabled`, { httpStatus: 500 });
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new AgentRunError("schema-invalid", `${key} must be a JSON object`, { httpStatus: 500 });
|
||||
const out: Record<string, string> = {};
|
||||
for (const [entryKey, entryValue] of Object.entries(parsed)) {
|
||||
if (typeof entryValue !== "string" || entryValue.length === 0) throw new AgentRunError("schema-invalid", `${key}.${entryKey} must be a non-empty string`, { httpStatus: 500 });
|
||||
out[entryKey] = entryValue;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function stringListEnv(key: string, value: unknown): string[] {
|
||||
if (typeof value !== "string") throw new AgentRunError("schema-invalid", `${key} is required when runner retention is enabled`, { httpStatus: 500 });
|
||||
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function booleanEnv(key: string, value: string): boolean {
|
||||
if (value === "true") return true;
|
||||
if (value === "false") return false;
|
||||
throw new AgentRunError("schema-invalid", `${key} must be true or false`, { httpStatus: 500 });
|
||||
}
|
||||
|
||||
function normalizeError(error: unknown): AgentRunError {
|
||||
|
||||
Reference in New Issue
Block a user