feat: add resource monitoring and provider upgrade
This commit is contained in:
Codex
@@ -1,5 +1,5 @@
|
||||
FROM oven/bun:1-alpine
|
||||
RUN apk add --no-cache bash docker-cli openssh-client
|
||||
RUN apk add --no-cache bash docker-cli docker-cli-compose openssh-client
|
||||
WORKDIR /app/src/components/provider-gateway
|
||||
COPY src/components/provider-gateway/package.json ./package.json
|
||||
RUN bun install --production
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
import { appendFileSync, existsSync, mkdirSync } from "node:fs";
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import {
|
||||
type CoreDispatchMessage,
|
||||
type DockerContainerSummary,
|
||||
type DockerImageSummary,
|
||||
type DockerNetworkSummary,
|
||||
type DockerStatusSnapshot,
|
||||
type DockerVolumeSummary,
|
||||
type JsonValue,
|
||||
type ProviderLabels,
|
||||
type ProviderTaskStatusMessage,
|
||||
type SystemStatusSnapshot,
|
||||
parseJsonObject,
|
||||
} from "../../shared/src/index";
|
||||
|
||||
@@ -18,6 +24,15 @@ interface RuntimeConfig {
|
||||
reconnectBaseMs: number;
|
||||
reconnectMaxMs: number;
|
||||
dockerSocketPath: string;
|
||||
monitorDiskPath: string;
|
||||
upgradeEnabled: boolean;
|
||||
upgradeHostProjectRoot: string;
|
||||
upgradeWorkspacePath: string;
|
||||
upgradeComposeFile: string;
|
||||
upgradeEnvFile: string;
|
||||
upgradeComposeProject: string;
|
||||
upgradeService: string;
|
||||
upgradeRunnerImage: string;
|
||||
logFile: string;
|
||||
}
|
||||
|
||||
@@ -26,6 +41,11 @@ const config = readConfig();
|
||||
const logger = createLogger("provider-gateway", config.logFile);
|
||||
let socket: WebSocket | null = null;
|
||||
let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let dockerStatusTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let systemStatusTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let dockerStatusRunning = false;
|
||||
let systemStatusRunning = false;
|
||||
let previousCpuSample: { idle: number; total: number } | null = null;
|
||||
let reconnectAttempt = 0;
|
||||
let stopping = false;
|
||||
|
||||
@@ -46,6 +66,13 @@ function readNumberEnv(name: string): number {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readBooleanEnv(name: string): boolean {
|
||||
const raw = requiredEnv(name);
|
||||
if (raw === "true") return true;
|
||||
if (raw === "false") return false;
|
||||
throw new Error(`Environment variable ${name} must be true or false, got ${raw}`);
|
||||
}
|
||||
|
||||
function readConfig(): RuntimeConfig {
|
||||
return {
|
||||
serverUrl: requiredEnv("PROVIDER_SERVER_URL"),
|
||||
@@ -57,6 +84,15 @@ function readConfig(): RuntimeConfig {
|
||||
reconnectBaseMs: readNumberEnv("RECONNECT_BASE_MS"),
|
||||
reconnectMaxMs: readNumberEnv("RECONNECT_MAX_MS"),
|
||||
dockerSocketPath: requiredEnv("DOCKER_SOCKET_PATH"),
|
||||
monitorDiskPath: requiredEnv("MONITOR_DISK_PATH"),
|
||||
upgradeEnabled: readBooleanEnv("PROVIDER_UPGRADE_ENABLED"),
|
||||
upgradeHostProjectRoot: requiredEnv("PROVIDER_UPGRADE_HOST_PROJECT_ROOT"),
|
||||
upgradeWorkspacePath: requiredEnv("PROVIDER_UPGRADE_WORKSPACE_PATH"),
|
||||
upgradeComposeFile: requiredEnv("PROVIDER_UPGRADE_COMPOSE_FILE"),
|
||||
upgradeEnvFile: requiredEnv("PROVIDER_UPGRADE_ENV_FILE"),
|
||||
upgradeComposeProject: requiredEnv("PROVIDER_UPGRADE_COMPOSE_PROJECT"),
|
||||
upgradeService: requiredEnv("PROVIDER_UPGRADE_SERVICE"),
|
||||
upgradeRunnerImage: requiredEnv("PROVIDER_UPGRADE_RUNNER_IMAGE"),
|
||||
logFile: requiredEnv("LOG_FILE"),
|
||||
};
|
||||
}
|
||||
@@ -105,7 +141,7 @@ function sendRegister(): void {
|
||||
name: config.providerName,
|
||||
labels: currentLabels(),
|
||||
startedAt: startedAt.toISOString(),
|
||||
capabilities: ["heartbeat", "docker.ps", "echo"],
|
||||
capabilities: ["heartbeat", "system.status", "docker.status", "docker.ps", "provider.upgrade", "echo"],
|
||||
});
|
||||
}
|
||||
|
||||
@@ -118,6 +154,49 @@ function sendHeartbeat(): void {
|
||||
});
|
||||
}
|
||||
|
||||
async function sendDockerStatus(): Promise<void> {
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN || dockerStatusRunning) return;
|
||||
dockerStatusRunning = true;
|
||||
try {
|
||||
const status = await collectDockerStatus();
|
||||
sendJson({
|
||||
type: "docker_status",
|
||||
providerId: config.providerId,
|
||||
at: new Date().toISOString(),
|
||||
status,
|
||||
});
|
||||
logger("debug", "docker_status_sent", { providerId: config.providerId, counts: status.counts });
|
||||
} catch (error) {
|
||||
logger("error", "docker_status_failed", { error: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
dockerStatusRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendSystemStatus(): Promise<void> {
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN || systemStatusRunning) return;
|
||||
systemStatusRunning = true;
|
||||
try {
|
||||
const status = await collectSystemStatus();
|
||||
sendJson({
|
||||
type: "system_status",
|
||||
providerId: config.providerId,
|
||||
at: new Date().toISOString(),
|
||||
status,
|
||||
});
|
||||
logger("debug", "system_status_sent", {
|
||||
providerId: config.providerId,
|
||||
cpuPercent: status.cpu.percent,
|
||||
memoryPercent: status.memory.percent,
|
||||
diskPercent: status.disk.percent,
|
||||
});
|
||||
} catch (error) {
|
||||
logger("error", "system_status_failed", { error: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
systemStatusRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendTaskStatus(taskId: string, status: ProviderTaskStatusMessage["status"], message: string, result?: JsonValue): Promise<void> {
|
||||
sendJson({
|
||||
type: "task_status",
|
||||
@@ -130,22 +209,274 @@ async function sendTaskStatus(taskId: string, status: ProviderTaskStatusMessage[
|
||||
});
|
||||
}
|
||||
|
||||
async function runDockerPs(): Promise<JsonValue> {
|
||||
const proc = Bun.spawn(["docker", "ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"], {
|
||||
async function runProcessCommand(command: string, args: string[], timeoutMs = 6000): Promise<{ ok: boolean; stdout: string; stderr: string; exitCode: number }> {
|
||||
const proc = Bun.spawn([command, ...args], {
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
const timeout = setTimeout(() => proc.kill("SIGKILL"), 5000);
|
||||
const timeout = setTimeout(() => proc.kill("SIGKILL"), timeoutMs);
|
||||
const [stdout, stderr, exitCode] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
clearTimeout(timeout);
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`docker ps failed with exit ${exitCode}: ${stderr}`);
|
||||
return { ok: exitCode === 0, stdout, stderr, exitCode };
|
||||
}
|
||||
|
||||
async function runDockerCommand(args: string[], timeoutMs = 6000): Promise<{ ok: boolean; stdout: string; stderr: string; exitCode: number }> {
|
||||
return runProcessCommand("docker", args, timeoutMs);
|
||||
}
|
||||
|
||||
function stringField(row: Record<string, unknown>, key: string): string {
|
||||
const value = row[key];
|
||||
return typeof value === "string" ? value : value === undefined || value === null ? "" : String(value);
|
||||
}
|
||||
|
||||
function parseJsonLines(stdout: string, limit: number): Array<Record<string, unknown>> {
|
||||
const rows: Array<Record<string, unknown>> = [];
|
||||
for (const line of stdout.split("\n")) {
|
||||
const text = line.trim();
|
||||
if (!text) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) rows.push(parsed as Record<string, unknown>);
|
||||
} catch (error) {
|
||||
logger("warn", "docker_json_line_parse_failed", { line: text.slice(0, 200), error: String(error) });
|
||||
}
|
||||
if (rows.length >= limit) break;
|
||||
}
|
||||
const containers = stdout
|
||||
return rows;
|
||||
}
|
||||
|
||||
function toContainer(row: Record<string, unknown>): DockerContainerSummary {
|
||||
return {
|
||||
id: stringField(row, "ID"),
|
||||
name: stringField(row, "Names"),
|
||||
image: stringField(row, "Image"),
|
||||
state: stringField(row, "State"),
|
||||
status: stringField(row, "Status"),
|
||||
ports: stringField(row, "Ports"),
|
||||
createdAt: stringField(row, "CreatedAt"),
|
||||
runningFor: stringField(row, "RunningFor"),
|
||||
size: stringField(row, "Size"),
|
||||
networks: stringField(row, "Networks"),
|
||||
};
|
||||
}
|
||||
|
||||
function toImage(row: Record<string, unknown>): DockerImageSummary {
|
||||
return {
|
||||
id: stringField(row, "ID"),
|
||||
repository: stringField(row, "Repository"),
|
||||
tag: stringField(row, "Tag"),
|
||||
size: stringField(row, "Size"),
|
||||
createdSince: stringField(row, "CreatedSince"),
|
||||
containers: stringField(row, "Containers"),
|
||||
};
|
||||
}
|
||||
|
||||
function toVolume(row: Record<string, unknown>): DockerVolumeSummary {
|
||||
return {
|
||||
name: stringField(row, "Name"),
|
||||
driver: stringField(row, "Driver"),
|
||||
scope: stringField(row, "Scope"),
|
||||
mountpoint: stringField(row, "Mountpoint"),
|
||||
};
|
||||
}
|
||||
|
||||
function toNetwork(row: Record<string, unknown>): DockerNetworkSummary {
|
||||
return {
|
||||
id: stringField(row, "ID"),
|
||||
name: stringField(row, "Name"),
|
||||
driver: stringField(row, "Driver"),
|
||||
scope: stringField(row, "Scope"),
|
||||
internal: stringField(row, "Internal"),
|
||||
ipv4: stringField(row, "IPv4"),
|
||||
ipv6: stringField(row, "IPv6"),
|
||||
};
|
||||
}
|
||||
|
||||
function jsonRecord(value: unknown): Record<string, JsonValue> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
|
||||
return value as Record<string, JsonValue>;
|
||||
}
|
||||
|
||||
function clampPercent(value: number): number {
|
||||
return Math.max(0, Math.min(100, Number(value.toFixed(1))));
|
||||
}
|
||||
|
||||
function readCpuSample(): { idle: number; total: number; cores: number } {
|
||||
const stat = readFileSync("/proc/stat", "utf8");
|
||||
const lines = stat.split("\n");
|
||||
const aggregate = lines[0]?.trim().split(/\s+/).slice(1).map(Number) ?? [];
|
||||
if (aggregate.length < 5 || aggregate.some((item) => !Number.isFinite(item))) {
|
||||
throw new Error("Unable to parse /proc/stat aggregate CPU row");
|
||||
}
|
||||
const idle = aggregate[3] + aggregate[4];
|
||||
const total = aggregate.reduce((sum, item) => sum + item, 0);
|
||||
const cores = Math.max(1, lines.filter((line) => /^cpu\d+\s/.test(line)).length);
|
||||
return { idle, total, cores };
|
||||
}
|
||||
|
||||
function readLoadAverage(): { load1: number; load5: number; load15: number } {
|
||||
const [load1, load5, load15] = readFileSync("/proc/loadavg", "utf8").trim().split(/\s+/).map(Number);
|
||||
return {
|
||||
load1: Number.isFinite(load1) ? load1 : 0,
|
||||
load5: Number.isFinite(load5) ? load5 : 0,
|
||||
load15: Number.isFinite(load15) ? load15 : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function readMemInfo(): Record<string, number> {
|
||||
const result: Record<string, number> = {};
|
||||
for (const line of readFileSync("/proc/meminfo", "utf8").split("\n")) {
|
||||
const match = line.match(/^([A-Za-z_()]+):\s+(\d+)\s+kB/);
|
||||
if (match) result[match[1]] = Number(match[2]) * 1024;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function readDiskUsage(path: string): Promise<{ mount: string; totalBytes: number; usedBytes: number; availableBytes: number; percent: number }> {
|
||||
const result = await runProcessCommand("df", ["-PB1", path], 3000);
|
||||
if (!result.ok) throw new Error(`df failed with exit ${result.exitCode}: ${result.stderr}`);
|
||||
const lines = result.stdout.trim().split("\n").filter(Boolean);
|
||||
const values = lines[1]?.trim().split(/\s+/);
|
||||
if (!values || values.length < 6) throw new Error(`Unable to parse df output: ${result.stdout.slice(0, 300)}`);
|
||||
const totalBytes = Number(values[1]);
|
||||
const usedBytes = Number(values[2]);
|
||||
const availableBytes = Number(values[3]);
|
||||
if (![totalBytes, usedBytes, availableBytes].every(Number.isFinite) || totalBytes <= 0) {
|
||||
throw new Error(`Invalid df numeric output: ${lines[1]}`);
|
||||
}
|
||||
return {
|
||||
mount: values.slice(5).join(" "),
|
||||
totalBytes,
|
||||
usedBytes,
|
||||
availableBytes,
|
||||
percent: clampPercent((usedBytes / totalBytes) * 100),
|
||||
};
|
||||
}
|
||||
|
||||
async function collectSystemStatus(): Promise<SystemStatusSnapshot> {
|
||||
const collectedAt = new Date().toISOString();
|
||||
const errors: JsonValue[] = [];
|
||||
|
||||
let cpu: Record<string, JsonValue> = { percent: 0, cores: 0, load1: 0, load5: 0, load15: 0 };
|
||||
try {
|
||||
const sample = readCpuSample();
|
||||
const load = readLoadAverage();
|
||||
let percent = clampPercent((load.load1 / sample.cores) * 100);
|
||||
if (previousCpuSample !== null) {
|
||||
const totalDelta = sample.total - previousCpuSample.total;
|
||||
const idleDelta = sample.idle - previousCpuSample.idle;
|
||||
if (totalDelta > 0) percent = clampPercent((1 - idleDelta / totalDelta) * 100);
|
||||
}
|
||||
previousCpuSample = { idle: sample.idle, total: sample.total };
|
||||
cpu = { percent, cores: sample.cores, ...load };
|
||||
} catch (error) {
|
||||
errors.push({ source: "proc.stat", error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
|
||||
let memory: Record<string, JsonValue> = { totalBytes: 0, usedBytes: 0, availableBytes: 0, percent: 0 };
|
||||
try {
|
||||
const mem = readMemInfo();
|
||||
const totalBytes = mem.MemTotal ?? 0;
|
||||
const availableBytes = mem.MemAvailable ?? mem.MemFree ?? 0;
|
||||
const usedBytes = Math.max(0, totalBytes - availableBytes);
|
||||
memory = {
|
||||
totalBytes,
|
||||
usedBytes,
|
||||
availableBytes,
|
||||
percent: totalBytes > 0 ? clampPercent((usedBytes / totalBytes) * 100) : 0,
|
||||
};
|
||||
} catch (error) {
|
||||
errors.push({ source: "proc.meminfo", error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
|
||||
let disk: Record<string, JsonValue> = { path: config.monitorDiskPath, mount: "", totalBytes: 0, usedBytes: 0, availableBytes: 0, percent: 0 };
|
||||
try {
|
||||
disk = { path: config.monitorDiskPath, ...(await readDiskUsage(config.monitorDiskPath)) };
|
||||
} catch (error) {
|
||||
errors.push({ source: "df", path: config.monitorDiskPath, error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
|
||||
return { ok: errors.length === 0, collectedAt, cpu, memory, disk, errors };
|
||||
}
|
||||
|
||||
async function collectDockerStatus(): Promise<DockerStatusSnapshot> {
|
||||
const collectedAt = new Date().toISOString();
|
||||
const socketPresent = existsSync(config.dockerSocketPath);
|
||||
const errors: JsonValue[] = [];
|
||||
const [infoResult, containersResult, imagesResult, volumesResult, networksResult] = await Promise.all([
|
||||
runDockerCommand(["info", "--format", "{{json .}}"]),
|
||||
runDockerCommand(["ps", "-a", "--format", "{{json .}}"]),
|
||||
runDockerCommand(["images", "--format", "{{json .}}"]),
|
||||
runDockerCommand(["volume", "ls", "--format", "{{json .}}"]),
|
||||
runDockerCommand(["network", "ls", "--format", "{{json .}}"]),
|
||||
]);
|
||||
|
||||
let info: Record<string, JsonValue> = {};
|
||||
if (infoResult.ok) {
|
||||
try {
|
||||
info = jsonRecord(JSON.parse(infoResult.stdout) as unknown);
|
||||
} catch (error) {
|
||||
errors.push({ source: "docker.info", error: String(error) });
|
||||
}
|
||||
} else {
|
||||
errors.push({ source: "docker.info", exitCode: infoResult.exitCode, stderr: infoResult.stderr.slice(0, 500) });
|
||||
}
|
||||
for (const [source, result] of Object.entries({ containers: containersResult, images: imagesResult, volumes: volumesResult, networks: networksResult })) {
|
||||
if (!result.ok) errors.push({ source, exitCode: result.exitCode, stderr: result.stderr.slice(0, 500) });
|
||||
}
|
||||
|
||||
const containers = containersResult.ok ? parseJsonLines(containersResult.stdout, 80).map(toContainer) : [];
|
||||
const images = imagesResult.ok ? parseJsonLines(imagesResult.stdout, 80).map(toImage) : [];
|
||||
const volumes = volumesResult.ok ? parseJsonLines(volumesResult.stdout, 60).map(toVolume) : [];
|
||||
const networks = networksResult.ok ? parseJsonLines(networksResult.stdout, 60).map(toNetwork) : [];
|
||||
const running = containers.filter((item) => item.state === "running").length;
|
||||
const paused = containers.filter((item) => item.state === "paused").length;
|
||||
const stopped = containers.filter((item) => item.state !== "running" && item.state !== "paused").length;
|
||||
|
||||
return {
|
||||
ok: errors.length === 0,
|
||||
socketPresent,
|
||||
collectedAt,
|
||||
daemon: {
|
||||
name: info.Name ?? "",
|
||||
serverVersion: info.ServerVersion ?? "",
|
||||
operatingSystem: info.OperatingSystem ?? "",
|
||||
osType: info.OSType ?? "",
|
||||
architecture: info.Architecture ?? "",
|
||||
cpus: info.NCPU ?? 0,
|
||||
memoryBytes: info.MemTotal ?? 0,
|
||||
dockerRootDir: info.DockerRootDir ?? "",
|
||||
driver: info.Driver ?? "",
|
||||
},
|
||||
counts: {
|
||||
containers: containers.length,
|
||||
running,
|
||||
paused,
|
||||
stopped,
|
||||
images: images.length,
|
||||
volumes: volumes.length,
|
||||
networks: networks.length,
|
||||
daemonContainers: info.Containers ?? containers.length,
|
||||
daemonImages: info.Images ?? images.length,
|
||||
},
|
||||
containers,
|
||||
images,
|
||||
volumes,
|
||||
networks,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
async function runDockerPs(): Promise<JsonValue> {
|
||||
const result = await runDockerCommand(["ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"], 5000);
|
||||
if (!result.ok) {
|
||||
throw new Error(`docker ps failed with exit ${result.exitCode}: ${result.stderr}`);
|
||||
}
|
||||
const containers = result.stdout
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0)
|
||||
@@ -154,7 +485,91 @@ async function runDockerPs(): Promise<JsonValue> {
|
||||
const [id, name, image, status, ports] = line.split("\t");
|
||||
return { id, name, image, status, ports };
|
||||
});
|
||||
return { containers, count: containers.length, stderr };
|
||||
return { containers, count: containers.length, stderr: result.stderr };
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function safeDockerName(value: string): string {
|
||||
return value.replace(/[^a-zA-Z0-9_.-]/g, "-").slice(0, 80);
|
||||
}
|
||||
|
||||
function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
const workspace = config.upgradeWorkspacePath;
|
||||
const composeCommand = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
`${workspace}/${config.upgradeEnvFile}`,
|
||||
"-f",
|
||||
`${workspace}/${config.upgradeComposeFile}`,
|
||||
"-p",
|
||||
config.upgradeComposeProject,
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--build",
|
||||
config.upgradeService,
|
||||
];
|
||||
const updaterName = `unidesk-provider-upgrader-${safeDockerName(taskId)}`;
|
||||
const script = `set -euo pipefail; sleep 2; cd ${shellQuote(workspace)}; ${composeCommand.map(shellQuote).join(" ")}`;
|
||||
const dockerRunCommand = [
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--rm",
|
||||
"--name",
|
||||
updaterName,
|
||||
"-v",
|
||||
"/var/run/docker.sock:/var/run/docker.sock",
|
||||
"-v",
|
||||
`${config.upgradeHostProjectRoot}:${workspace}:ro`,
|
||||
"-w",
|
||||
workspace,
|
||||
config.upgradeRunnerImage,
|
||||
"sh",
|
||||
"-lc",
|
||||
script,
|
||||
];
|
||||
return {
|
||||
enabled: config.upgradeEnabled,
|
||||
taskId,
|
||||
updaterName,
|
||||
runnerImage: config.upgradeRunnerImage,
|
||||
hostProjectRoot: config.upgradeHostProjectRoot,
|
||||
workspace,
|
||||
compose: {
|
||||
project: config.upgradeComposeProject,
|
||||
service: config.upgradeService,
|
||||
composeFile: config.upgradeComposeFile,
|
||||
envFile: config.upgradeEnvFile,
|
||||
},
|
||||
composeCommand,
|
||||
dockerRunCommand,
|
||||
};
|
||||
}
|
||||
|
||||
async function runProviderUpgrade(taskId: string, payload: Record<string, JsonValue>): Promise<JsonValue> {
|
||||
const mode = payload.mode === "schedule" ? "schedule" : "plan";
|
||||
const plan = upgradePlan(taskId);
|
||||
if (mode === "plan") return { mode, message: "provider gateway upgrade plan generated; no container changed", plan };
|
||||
if (!config.upgradeEnabled) {
|
||||
throw new Error("provider gateway remote upgrade is disabled by PROVIDER_UPGRADE_ENABLED=false");
|
||||
}
|
||||
const dockerRunCommand = (plan.dockerRunCommand as JsonValue[]).map((item) => String(item));
|
||||
const result = await runDockerCommand(dockerRunCommand.slice(1), 12_000);
|
||||
if (!result.ok) {
|
||||
throw new Error(`provider upgrade scheduler failed with exit ${result.exitCode}: ${result.stderr}`);
|
||||
}
|
||||
return {
|
||||
mode,
|
||||
message: "provider gateway upgrade scheduled by detached updater container",
|
||||
updaterContainerId: result.stdout.trim(),
|
||||
plan,
|
||||
stderr: result.stderr.slice(0, 500),
|
||||
};
|
||||
}
|
||||
|
||||
async function handleDispatch(message: CoreDispatchMessage): Promise<void> {
|
||||
@@ -167,6 +582,11 @@ async function handleDispatch(message: CoreDispatchMessage): Promise<void> {
|
||||
await sendTaskStatus(message.taskId, "succeeded", "docker ps completed", result);
|
||||
return;
|
||||
}
|
||||
if (message.command === "provider.upgrade") {
|
||||
const result = await runProviderUpgrade(message.taskId, message.payload);
|
||||
await sendTaskStatus(message.taskId, "succeeded", "provider upgrade command completed", result);
|
||||
return;
|
||||
}
|
||||
await sendTaskStatus(message.taskId, "succeeded", "echo completed", { echo: message.payload });
|
||||
} catch (error) {
|
||||
const text = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
||||
@@ -205,14 +625,28 @@ function connect(): void {
|
||||
logger("info", "connect_open", { providerId: config.providerId });
|
||||
sendRegister();
|
||||
sendHeartbeat();
|
||||
sendSystemStatus().catch((error) => logger("error", "system_status_initial_failed", { error: String(error) }));
|
||||
sendDockerStatus().catch((error) => logger("error", "docker_status_initial_failed", { error: String(error) }));
|
||||
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
||||
if (systemStatusTimer !== null) clearInterval(systemStatusTimer);
|
||||
if (dockerStatusTimer !== null) clearInterval(dockerStatusTimer);
|
||||
heartbeatTimer = setInterval(sendHeartbeat, config.heartbeatIntervalMs);
|
||||
systemStatusTimer = setInterval(() => {
|
||||
sendSystemStatus().catch((error) => logger("error", "system_status_interval_failed", { error: String(error) }));
|
||||
}, config.heartbeatIntervalMs);
|
||||
dockerStatusTimer = setInterval(() => {
|
||||
sendDockerStatus().catch((error) => logger("error", "docker_status_interval_failed", { error: String(error) }));
|
||||
}, config.heartbeatIntervalMs);
|
||||
});
|
||||
socket.addEventListener("message", (event) => handleMessage(event as MessageEvent<string>));
|
||||
socket.addEventListener("close", (event) => {
|
||||
logger("warn", "connect_close", { code: event.code, reason: event.reason });
|
||||
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
||||
if (systemStatusTimer !== null) clearInterval(systemStatusTimer);
|
||||
if (dockerStatusTimer !== null) clearInterval(dockerStatusTimer);
|
||||
heartbeatTimer = null;
|
||||
systemStatusTimer = null;
|
||||
dockerStatusTimer = null;
|
||||
scheduleReconnect();
|
||||
});
|
||||
socket.addEventListener("error", () => {
|
||||
@@ -224,6 +658,8 @@ process.on("SIGTERM", () => {
|
||||
stopping = true;
|
||||
logger("warn", "sigterm_received");
|
||||
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
||||
if (systemStatusTimer !== null) clearInterval(systemStatusTimer);
|
||||
if (dockerStatusTimer !== null) clearInterval(dockerStatusTimer);
|
||||
socket?.close(1000, "provider shutdown");
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user