Codex
370 lines
16 KiB
TypeScript
370 lines
16 KiB
TypeScript
import { Buffer } from "node:buffer";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
|
|
import { type UniDeskConfig, rootPath } from "./config";
|
|
import { remoteGcDegradedFailure } from "./gc-remote-degraded";
|
|
import { runSshCommandCapture } from "./ssh";
|
|
|
|
type RemoteGcAction = "plan" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
|
|
|
|
interface RemoteGcOptions {
|
|
confirm: boolean;
|
|
journal: boolean;
|
|
journalTargetBytes: number;
|
|
dockerLogs: boolean;
|
|
dockerLogMaxBytes: number;
|
|
buildCache: boolean;
|
|
buildCacheUntil: string;
|
|
tmp: boolean;
|
|
tmpMinAgeHours: number;
|
|
toolCaches: boolean;
|
|
webObserveArtifacts: boolean;
|
|
k3sImageCache: boolean;
|
|
hostContainerdCache: boolean;
|
|
localPathOrphans: boolean;
|
|
aptCache: boolean;
|
|
coreDumps: boolean;
|
|
coreDumpMinAgeHours: number;
|
|
hwlabRegistry: boolean;
|
|
registryGcOnly: boolean;
|
|
registryKeepPerRepo: number;
|
|
registryMinAgeHours: number;
|
|
targetUsePercent?: number;
|
|
jobId?: string;
|
|
limit: number;
|
|
resultLimit: number;
|
|
full: boolean;
|
|
historyLimit: number;
|
|
saveSnapshot: boolean;
|
|
}
|
|
|
|
const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
|
confirm: false,
|
|
journal: true,
|
|
journalTargetBytes: 512 * 1024 * 1024,
|
|
dockerLogs: true,
|
|
dockerLogMaxBytes: 50 * 1024 * 1024,
|
|
buildCache: true,
|
|
buildCacheUntil: "24h",
|
|
tmp: true,
|
|
tmpMinAgeHours: 24,
|
|
toolCaches: false,
|
|
webObserveArtifacts: false,
|
|
k3sImageCache: false,
|
|
hostContainerdCache: false,
|
|
localPathOrphans: false,
|
|
aptCache: true,
|
|
coreDumps: true,
|
|
coreDumpMinAgeHours: 1,
|
|
hwlabRegistry: false,
|
|
registryGcOnly: false,
|
|
registryKeepPerRepo: 20,
|
|
registryMinAgeHours: 48,
|
|
limit: 50,
|
|
resultLimit: 50,
|
|
full: false,
|
|
historyLimit: 12,
|
|
saveSnapshot: true,
|
|
};
|
|
|
|
const GC_CONFIG_RELATIVE_PATH = "config/unidesk-cli.yaml";
|
|
const GC_REMOTE_CONFIG_REF = `${GC_CONFIG_RELATIVE_PATH}#gc.remote.targets`;
|
|
const GC_REMOTE_RUNNER_RELATIVE_PATH = "scripts/src/gc-remote-runner.py";
|
|
const GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER = "__UNIDESK_GC_REMOTE_CONFIG_BASE64__";
|
|
const GC_REMOTE_WEB_OBSERVE_RELATIVE_PATH = "scripts/src/gc-remote-web-observe.py";
|
|
const GC_REMOTE_WEB_OBSERVE_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_WEB_OBSERVE_HELPERS__";
|
|
const GC_REMOTE_CONTAINERD_RELATIVE_PATH = "scripts/src/gc-remote-containerd.py";
|
|
const GC_REMOTE_CONTAINERD_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_CONTAINERD_HELPERS__";
|
|
const GC_REMOTE_PVC_RELATIVE_PATH = "scripts/src/gc-remote-pvc.py";
|
|
const GC_REMOTE_PVC_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_PVC_HELPERS__";
|
|
const GC_REMOTE_GROWTH_RELATIVE_PATH = "scripts/src/gc-remote-growth.py";
|
|
const GC_REMOTE_GROWTH_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_GROWTH_HELPERS__";
|
|
const GC_REMOTE_REGISTRY_RELATIVE_PATH = "scripts/src/gc-remote-registry.py";
|
|
const GC_REMOTE_REGISTRY_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_REGISTRY_HELPERS__";
|
|
const GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH = "scripts/src/gc-remote-policy-runner.py";
|
|
const GC_REMOTE_POLICY_RUNNER_PLACEHOLDER = "__UNIDESK_GC_REMOTE_POLICY_RUNNER_BASE64__";
|
|
|
|
export async function runRemoteGcCommand(config: UniDeskConfig, providerId: string | undefined, action: string | undefined, args: string[]): Promise<unknown> {
|
|
if (providerId === undefined || providerId.length === 0) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-provider-required",
|
|
usage: "bun scripts/cli.ts gc remote <providerId> plan|snapshot|trend|run|status|policy [--confirm]",
|
|
};
|
|
}
|
|
const subaction = action ?? "plan";
|
|
if (subaction === "policy") {
|
|
const [policyAction = "plan", ...policyArgs] = args;
|
|
const options = parseRemoteGcOptions(policyArgs);
|
|
if (policyAction === "plan" || policyAction === "render" || policyAction === "dry-run") {
|
|
return await runRemoteGc(config, providerId, "policy-plan", options);
|
|
}
|
|
if (policyAction === "install") {
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-policy-install-requires-confirm",
|
|
dryRun: true,
|
|
mutation: false,
|
|
requiredFlag: "--confirm",
|
|
planCommand: `bun scripts/cli.ts gc remote ${providerId} policy plan`,
|
|
installCommand: `bun scripts/cli.ts gc remote ${providerId} policy install --confirm`,
|
|
};
|
|
}
|
|
return await runRemoteGc(config, providerId, "policy-install", options);
|
|
}
|
|
if (policyAction === "status") {
|
|
return await runRemoteGc(config, providerId, "policy-status", options);
|
|
}
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-gc-remote-policy-action",
|
|
action: policyAction,
|
|
supportedActions: ["plan", "render", "dry-run", "install", "status"],
|
|
};
|
|
}
|
|
const options = parseRemoteGcOptions(args);
|
|
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
|
|
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
|
|
if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options);
|
|
if (subaction === "status") return await runRemoteGc(config, providerId, "status", options);
|
|
if (subaction === "run") {
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-run-requires-confirm",
|
|
dryRun: true,
|
|
mutation: false,
|
|
requiredFlag: "--confirm",
|
|
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan`,
|
|
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm`,
|
|
};
|
|
}
|
|
return await runRemoteGc(config, providerId, "run", options);
|
|
}
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-gc-remote-action",
|
|
action: subaction,
|
|
supportedActions: ["plan", "snapshot", "trend", "run", "status", "policy"],
|
|
};
|
|
}
|
|
|
|
function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
|
|
const options = { ...DEFAULT_REMOTE_OPTIONS };
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (arg === "--confirm") {
|
|
options.confirm = true;
|
|
} else if (arg === "--dry-run") {
|
|
// accepted for plan compatibility
|
|
} else if (arg === "--journal-target-size") {
|
|
options.journalTargetBytes = parseSizeOption(arg, args[++index]);
|
|
} else if (arg === "--docker-log-max-bytes") {
|
|
options.dockerLogMaxBytes = parseSizeOption(arg, args[++index]);
|
|
} else if (arg === "--build-cache-until") {
|
|
const value = args[++index];
|
|
if (!value || !/^\d+(s|m|h|d)$/u.test(value)) throw new Error(`${arg} must look like 24h, 7d, 30m or 60s`);
|
|
options.buildCacheUntil = value;
|
|
} else if (arg === "--tmp-min-age-hours") {
|
|
options.tmpMinAgeHours = parseNonNegativeNumber(arg, args[++index]);
|
|
} else if (arg === "--core-dump-min-age-hours") {
|
|
options.coreDumpMinAgeHours = parseNonNegativeNumber(arg, args[++index]);
|
|
} else if (arg === "--registry-keep-per-repo") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value < 1) throw new Error("--registry-keep-per-repo must be an integer >= 1");
|
|
options.registryKeepPerRepo = Math.min(value, 50);
|
|
} else if (arg === "--registry-min-age-hours") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
options.registryMinAgeHours = value;
|
|
} else if (arg === "--target-use-percent") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value < 1 || value > 99) throw new Error("--target-use-percent must be an integer from 1 to 99");
|
|
options.targetUsePercent = value;
|
|
} else if (arg === "--job-id") {
|
|
const value = args[++index];
|
|
if (!value || !/^[A-Za-z0-9._-]{1,128}$/u.test(value)) throw new Error("--job-id must be a safe job id");
|
|
options.jobId = value;
|
|
} else if (arg === "--limit") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error("--limit must be a positive integer");
|
|
options.limit = Math.min(value, 5000);
|
|
} else if (arg === "--result-limit") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error("--result-limit must be a positive integer");
|
|
options.resultLimit = Math.min(value, 5000);
|
|
} else if (arg === "--history-limit") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value <= 1) throw new Error("--history-limit must be an integer greater than 1");
|
|
options.historyLimit = Math.min(value, 200);
|
|
} else if (arg === "--no-journal") {
|
|
options.journal = false;
|
|
} else if (arg === "--no-docker-logs") {
|
|
options.dockerLogs = false;
|
|
} else if (arg === "--no-build-cache") {
|
|
options.buildCache = false;
|
|
} else if (arg === "--no-tmp") {
|
|
options.tmp = false;
|
|
} else if (arg === "--include-tool-caches") {
|
|
options.toolCaches = true;
|
|
} else if (arg === "--no-tool-caches") {
|
|
options.toolCaches = false;
|
|
} else if (arg === "--include-web-observe-artifacts") {
|
|
options.webObserveArtifacts = true;
|
|
} else if (arg === "--no-web-observe-artifacts") {
|
|
options.webObserveArtifacts = false;
|
|
} else if (arg === "--include-k3s-image-cache") {
|
|
options.k3sImageCache = true;
|
|
} else if (arg === "--no-k3s-image-cache") {
|
|
options.k3sImageCache = false;
|
|
} else if (arg === "--include-host-containerd-cache") {
|
|
options.hostContainerdCache = true;
|
|
} else if (arg === "--no-host-containerd-cache") {
|
|
options.hostContainerdCache = false;
|
|
} else if (arg === "--include-local-path-orphans") {
|
|
options.localPathOrphans = true;
|
|
} else if (arg === "--no-local-path-orphans") {
|
|
options.localPathOrphans = false;
|
|
} else if (arg === "--no-apt-cache") {
|
|
options.aptCache = false;
|
|
} else if (arg === "--no-core-dumps") {
|
|
options.coreDumps = false;
|
|
} else if (arg === "--include-hwlab-registry") {
|
|
options.hwlabRegistry = true;
|
|
} else if (arg === "--registry-gc-only") {
|
|
options.hwlabRegistry = true;
|
|
options.registryGcOnly = true;
|
|
} else if (arg === "--full" || arg === "--raw") {
|
|
options.full = true;
|
|
} else if (arg === "--no-save" || arg === "--no-snapshot-save") {
|
|
options.saveSnapshot = false;
|
|
} else {
|
|
throw new Error(`unknown gc remote option: ${arg}`);
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function parseNonNegativeNumber(name: string, raw: string | undefined): number {
|
|
const value = Number(raw);
|
|
if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`);
|
|
return value;
|
|
}
|
|
|
|
function parseSizeOption(name: string, raw: string | undefined): number {
|
|
const value = parseSize(raw ?? "");
|
|
if (value === null || value <= 0) throw new Error(`${name} must be a positive size such as 512M, 1GiB or 50000000`);
|
|
return value;
|
|
}
|
|
|
|
function parseSize(raw: string): number | null {
|
|
const match = raw.trim().match(/^(\d+(?:\.\d+)?)\s*(b|k|kb|kib|m|mb|mib|g|gb|gib)?$/iu);
|
|
if (!match) return null;
|
|
const value = Number(match[1]);
|
|
const unit = (match[2] ?? "b").toLowerCase();
|
|
const multiplier =
|
|
unit === "g" || unit === "gb" || unit === "gib" ? 1024 ** 3
|
|
: unit === "m" || unit === "mb" || unit === "mib" ? 1024 ** 2
|
|
: unit === "k" || unit === "kb" || unit === "kib" ? 1024
|
|
: 1;
|
|
const bytes = Math.floor(value * multiplier);
|
|
return Number.isFinite(bytes) ? bytes : null;
|
|
}
|
|
|
|
function yamlRecordOrEmpty(value: unknown, label: string): Record<string, unknown> {
|
|
if (value === undefined || value === null) return {};
|
|
if (typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be a YAML object`);
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function loadRemoteGcTargetConfig(providerId: string): Record<string, unknown> {
|
|
const configPath = rootPath(GC_CONFIG_RELATIVE_PATH);
|
|
if (!existsSync(configPath)) return {};
|
|
const parsed = yamlRecordOrEmpty(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown, GC_CONFIG_RELATIVE_PATH);
|
|
const gc = yamlRecordOrEmpty(parsed.gc, `${GC_CONFIG_RELATIVE_PATH}#gc`);
|
|
const remote = yamlRecordOrEmpty(gc.remote, `${GC_CONFIG_RELATIVE_PATH}#gc.remote`);
|
|
const targets = yamlRecordOrEmpty(remote.targets, GC_REMOTE_CONFIG_REF);
|
|
const candidates = [providerId, providerId.toUpperCase(), providerId.toLowerCase()];
|
|
for (const key of candidates) {
|
|
if (Object.prototype.hasOwnProperty.call(targets, key)) return yamlRecordOrEmpty(targets[key], `${GC_REMOTE_CONFIG_REF}.${key}`);
|
|
}
|
|
return {};
|
|
}
|
|
|
|
async function runRemoteGc(config: UniDeskConfig, providerId: string, action: RemoteGcAction, options: RemoteGcOptions): Promise<unknown> {
|
|
const remoteTarget = loadRemoteGcTargetConfig(providerId);
|
|
const scriptConfig = Buffer.from(JSON.stringify({ providerId, action, options, remoteTarget }), "utf8").toString("base64");
|
|
const result = await runSshCommandCapture(config, providerId, ["py"], remoteGcPython(scriptConfig));
|
|
if (result.exitCode !== 0) {
|
|
const degraded = remoteGcDegradedFailure(providerId, action, result);
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-command-failed",
|
|
providerId,
|
|
action: `gc remote ${action}`,
|
|
exitCode: result.exitCode,
|
|
degradedReason: degraded.degradedReason,
|
|
transport: degraded.transport,
|
|
safeCandidateCount: null,
|
|
runAllowed: false,
|
|
mutation: false,
|
|
degraded,
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
stderrTail: result.stderr.slice(-4000),
|
|
next: degraded.next,
|
|
};
|
|
}
|
|
try {
|
|
return JSON.parse(result.stdout) as unknown;
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-invalid-json",
|
|
providerId,
|
|
action: `gc remote ${action}`,
|
|
message: error instanceof Error ? error.message : String(error),
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
stderrTail: result.stderr.slice(-4000),
|
|
};
|
|
}
|
|
}
|
|
|
|
function remoteGcPython(configBase64: string): string {
|
|
const template = readFileSync(rootPath(GC_REMOTE_RUNNER_RELATIVE_PATH), "utf8");
|
|
if (!template.includes(GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_WEB_OBSERVE_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_WEB_OBSERVE_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_CONTAINERD_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_CONTAINERD_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_PVC_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_PVC_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_GROWTH_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_GROWTH_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_REGISTRY_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_REGISTRY_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_POLICY_RUNNER_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_POLICY_RUNNER_PLACEHOLDER}`);
|
|
}
|
|
const webObserveHelpers = readFileSync(rootPath(GC_REMOTE_WEB_OBSERVE_RELATIVE_PATH), "utf8");
|
|
const containerdHelpers = readFileSync(rootPath(GC_REMOTE_CONTAINERD_RELATIVE_PATH), "utf8");
|
|
const pvcHelpers = readFileSync(rootPath(GC_REMOTE_PVC_RELATIVE_PATH), "utf8");
|
|
const growthHelpers = readFileSync(rootPath(GC_REMOTE_GROWTH_RELATIVE_PATH), "utf8");
|
|
const registryHelpers = readFileSync(rootPath(GC_REMOTE_REGISTRY_RELATIVE_PATH), "utf8");
|
|
const policyRunner = readFileSync(rootPath(GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH), "utf8");
|
|
return template
|
|
.replace(GC_REMOTE_WEB_OBSERVE_PLACEHOLDER, webObserveHelpers.trimEnd())
|
|
.replace(GC_REMOTE_CONTAINERD_PLACEHOLDER, containerdHelpers.trimEnd())
|
|
.replace(GC_REMOTE_PVC_PLACEHOLDER, pvcHelpers.trimEnd())
|
|
.replace(GC_REMOTE_GROWTH_PLACEHOLDER, growthHelpers.trimEnd())
|
|
.replace(GC_REMOTE_REGISTRY_PLACEHOLDER, registryHelpers.trimEnd())
|
|
.replace(GC_REMOTE_POLICY_RUNNER_PLACEHOLDER, Buffer.from(policyRunner, "utf8").toString("base64"))
|
|
.replace(GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER, configBase64);
|
|
}
|