feat: add code queue services and baidu netdisk
This commit is contained in:
Codex
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@unidesk/provider-gateway",
|
||||
"version": "0.2.12",
|
||||
"version": "0.2.17",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
||||
import {
|
||||
type CoreDispatchMessage,
|
||||
type CoreHostSshCloseMessage,
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
type SystemStatusSnapshot,
|
||||
parseJsonObject,
|
||||
} from "../../shared/src/index";
|
||||
import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../shared/src/rotating-jsonl";
|
||||
|
||||
interface RuntimeConfig {
|
||||
serverUrl: string;
|
||||
@@ -65,6 +66,8 @@ let reconnectAttempt = 0;
|
||||
let stopping = false;
|
||||
let upgradeSleepUntil = 0;
|
||||
let upgradeSleepTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let selfContainerGuard: ProviderGatewayContainerGuardState | null = null;
|
||||
let selfContainerGuardLastCheckMs = 0;
|
||||
|
||||
interface MicroserviceHttpCacheEntry {
|
||||
createdAt: number;
|
||||
@@ -83,11 +86,53 @@ interface HostSshStdin {
|
||||
end(): unknown;
|
||||
}
|
||||
|
||||
interface ProviderGatewayContainerGuardState {
|
||||
checkedAt: string;
|
||||
containerId: string;
|
||||
containerName: string;
|
||||
restartPolicy: string;
|
||||
restartPolicyMaximumRetryCount: number;
|
||||
restartPolicyOk: boolean;
|
||||
pidMode: string;
|
||||
pidModeOk: boolean;
|
||||
remediatedRestartPolicy: boolean;
|
||||
ok: boolean;
|
||||
error: string;
|
||||
}
|
||||
|
||||
interface DockerContainerInspectDetails {
|
||||
id: string;
|
||||
name: string;
|
||||
restartPolicy: string;
|
||||
restartPolicyMaximumRetryCount: number;
|
||||
pidMode: string;
|
||||
composeProject: string;
|
||||
composeService: string;
|
||||
labels: Record<string, string>;
|
||||
}
|
||||
|
||||
const hostSshSessions = new Map<string, HostSshSession>();
|
||||
const microserviceHttpCache = new Map<string, MicroserviceHttpCacheEntry>();
|
||||
const microserviceHttpInFlight = new Map<string, Promise<JsonValue>>();
|
||||
const gatewayMetadata = readGatewayMetadata();
|
||||
const defaultMasterServer = "http://74.48.78.17/";
|
||||
const defaultProviderToken = "unidesk-dev-token-change-me";
|
||||
const microserviceHttpMaxBodyTextLength = 8 * 1024 * 1024;
|
||||
const microserviceForwardRequestHeaders = [
|
||||
"accept",
|
||||
"content-type",
|
||||
"range",
|
||||
"x-auth",
|
||||
"x-requested-with",
|
||||
"destination",
|
||||
"overwrite",
|
||||
"tus-resumable",
|
||||
"upload-concat",
|
||||
"upload-defer-length",
|
||||
"upload-length",
|
||||
"upload-metadata",
|
||||
"upload-offset",
|
||||
] as const;
|
||||
|
||||
function readGatewayMetadataFile(path: string): { name: string; version: string } | null {
|
||||
try {
|
||||
@@ -112,16 +157,24 @@ function readTargetGatewayMetadata(workspace: string): { name: string; version:
|
||||
return readGatewayMetadataFile(`${root}/src/components/provider-gateway/package.json`) ?? gatewayMetadata;
|
||||
}
|
||||
|
||||
function requiredEnv(name: string): string {
|
||||
const value = process.env[name];
|
||||
if (value === undefined || value.length === 0) {
|
||||
throw new Error(`Missing required environment variable: ${name}`);
|
||||
function readEnv(...names: string[]): string | null {
|
||||
for (const name of names) {
|
||||
const value = process.env[name];
|
||||
if (value !== undefined && value.length > 0) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function requiredEnv(name: string, ...aliases: string[]): string {
|
||||
const value = readEnv(name, ...aliases);
|
||||
if (value === null) {
|
||||
throw new Error(`Missing required environment variable: ${[name, ...aliases].join(" or ")}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function readNumberEnv(name: string): number {
|
||||
const raw = requiredEnv(name);
|
||||
function readNumberEnv(name: string, fallback: number, ...aliases: string[]): number {
|
||||
const raw = readEnv(name, ...aliases) ?? String(fallback);
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`);
|
||||
@@ -129,14 +182,12 @@ function readNumberEnv(name: string): number {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readOptionalStringEnv(name: string): string | null {
|
||||
const value = process.env[name];
|
||||
if (value === undefined || value.length === 0) return null;
|
||||
return value;
|
||||
function readOptionalStringEnv(name: string, ...aliases: string[]): string | null {
|
||||
return readEnv(name, ...aliases);
|
||||
}
|
||||
|
||||
function readOptionalNumberEnv(name: string): number | null {
|
||||
const raw = readOptionalStringEnv(name);
|
||||
function readOptionalNumberEnv(name: string, ...aliases: string[]): number | null {
|
||||
const raw = readOptionalStringEnv(name, ...aliases);
|
||||
if (raw === null) return null;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
@@ -145,44 +196,290 @@ function readOptionalNumberEnv(name: string): number | null {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readConfig(): RuntimeConfig {
|
||||
function providerServerUrlFromMaster(rawMaster: string): string {
|
||||
const raw = rawMaster.trim() || defaultMasterServer;
|
||||
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//iu.test(raw) ? raw : `http://${raw}`;
|
||||
const url = new URL(withScheme);
|
||||
if (url.protocol === "ws:" || url.protocol === "wss:") {
|
||||
if (url.pathname === "" || url.pathname === "/") url.pathname = "/ws/provider";
|
||||
return url.toString();
|
||||
}
|
||||
const providerIngressPort = readEnv("PROVIDER_INGRESS_PORT", "UNIDESK_PROVIDER_INGRESS_PORT")
|
||||
?? (url.port && url.port !== "18081" ? url.port : "18082");
|
||||
const protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
||||
const host = url.hostname.includes(":") ? `[${url.hostname}]` : url.hostname;
|
||||
const providerUrl = new URL(`${protocol}//${host}:${providerIngressPort}/ws/provider`);
|
||||
return providerUrl.toString();
|
||||
}
|
||||
|
||||
function readProviderServerUrl(): string {
|
||||
const explicit = readEnv("PROVIDER_SERVER_URL", "UNIDESK_PROVIDER_SERVER_URL");
|
||||
if (explicit !== null) return explicit;
|
||||
return providerServerUrlFromMaster(readEnv("UNIDESK_MASTER_SERVER", "UNIDESK_MASTER_SERVER_URL", "MASTER_SERVER_URL", "MASTER_SERVER_IP") ?? defaultMasterServer);
|
||||
}
|
||||
|
||||
function readProviderId(): string {
|
||||
return requiredEnv("PROVIDER_ID", "UNIDESK_PROVIDER_ID");
|
||||
}
|
||||
|
||||
function safeProviderSlug(value: string): string {
|
||||
return value.replace(/[^a-zA-Z0-9_.-]/g, "-").replace(/^-+|-+$/g, "").toLowerCase() || "provider";
|
||||
}
|
||||
|
||||
function safeReadText(path: string): string {
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function defaultProviderLabels(providerId: string): ProviderLabels {
|
||||
const labels: ProviderLabels = {
|
||||
host: providerId,
|
||||
role: "compute-provider",
|
||||
docker: true,
|
||||
attachMode: "simple",
|
||||
};
|
||||
const kernel = safeReadText("/proc/sys/kernel/osrelease").toLowerCase();
|
||||
if (kernel.includes("microsoft") || kernel.includes("wsl") || process.env.WSL_INTEROP !== undefined) {
|
||||
labels.role = "wsl-provider";
|
||||
labels.wsl = true;
|
||||
}
|
||||
const osRelease = safeReadText("/etc/os-release");
|
||||
const distro = osRelease.match(/^PRETTY_NAME="?([^"\n]+)"?/m)?.[1] ?? osRelease.match(/^ID="?([^"\n]+)"?/m)?.[1] ?? "";
|
||||
if (distro.length > 0) labels.distro = distro;
|
||||
return labels;
|
||||
}
|
||||
|
||||
function readProviderLabels(providerId: string): ProviderLabels {
|
||||
const raw = readEnv("PROVIDER_LABELS_JSON", "UNIDESK_PROVIDER_LABELS_JSON");
|
||||
return raw === null ? defaultProviderLabels(providerId) : parseJsonObject(raw, "PROVIDER_LABELS_JSON");
|
||||
}
|
||||
|
||||
function goTemplateString(value: string): string {
|
||||
return `"${value.replace(/\\/g, "\\\\").replace(/"/g, "\\\"")}"`;
|
||||
}
|
||||
|
||||
function currentContainerId(): string | null {
|
||||
const hostname = (process.env.HOSTNAME ?? safeReadText("/etc/hostname")).trim();
|
||||
return hostname.length > 0 ? hostname : null;
|
||||
}
|
||||
|
||||
function unknownContainerGuardState(error: string): ProviderGatewayContainerGuardState {
|
||||
return {
|
||||
serverUrl: requiredEnv("PROVIDER_SERVER_URL"),
|
||||
token: requiredEnv("PROVIDER_TOKEN"),
|
||||
providerId: requiredEnv("PROVIDER_ID"),
|
||||
providerName: requiredEnv("PROVIDER_NAME"),
|
||||
labels: parseJsonObject(requiredEnv("PROVIDER_LABELS_JSON"), "PROVIDER_LABELS_JSON"),
|
||||
heartbeatIntervalMs: readNumberEnv("HEARTBEAT_INTERVAL_MS"),
|
||||
reconnectBaseMs: readNumberEnv("RECONNECT_BASE_MS"),
|
||||
reconnectMaxMs: readNumberEnv("RECONNECT_MAX_MS"),
|
||||
dockerSocketPath: requiredEnv("DOCKER_SOCKET_PATH"),
|
||||
monitorDiskPath: requiredEnv("MONITOR_DISK_PATH"),
|
||||
upgradeHostProjectRoot: requiredEnv("PROVIDER_UPGRADE_HOST_PROJECT_ROOT"),
|
||||
upgradeWorkspacePath: requiredEnv("PROVIDER_UPGRADE_WORKSPACE_PATH"),
|
||||
upgradeComposeFile: requiredEnv("PROVIDER_UPGRADE_COMPOSE_FILE"),
|
||||
upgradeEnvFile: requiredEnv("PROVIDER_UPGRADE_ENV_FILE"),
|
||||
upgradeComposeProject: requiredEnv("PROVIDER_UPGRADE_COMPOSE_PROJECT"),
|
||||
upgradeService: requiredEnv("PROVIDER_UPGRADE_SERVICE"),
|
||||
upgradeRunnerImage: requiredEnv("PROVIDER_UPGRADE_RUNNER_IMAGE"),
|
||||
hostSshHost: readOptionalStringEnv("HOST_SSH_HOST"),
|
||||
hostSshPort: readOptionalNumberEnv("HOST_SSH_PORT"),
|
||||
hostSshUser: readOptionalStringEnv("HOST_SSH_USER"),
|
||||
hostSshKey: readOptionalStringEnv("HOST_SSH_KEY"),
|
||||
hostRemoteCwd: readOptionalStringEnv("HOST_REMOTE_CWD"),
|
||||
hostLoginShell: readOptionalStringEnv("HOST_LOGIN_SHELL"),
|
||||
logFile: requiredEnv("LOG_FILE"),
|
||||
checkedAt: new Date().toISOString(),
|
||||
containerId: currentContainerId() ?? "",
|
||||
containerName: "",
|
||||
restartPolicy: "",
|
||||
restartPolicyMaximumRetryCount: 0,
|
||||
restartPolicyOk: false,
|
||||
pidMode: "",
|
||||
pidModeOk: false,
|
||||
remediatedRestartPolicy: false,
|
||||
ok: false,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
function stringRecordFromUnknown(value: unknown): Record<string, string> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return {};
|
||||
const output: Record<string, string> = {};
|
||||
for (const [key, raw] of Object.entries(value)) {
|
||||
if (typeof raw === "string") output[key] = raw;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function dockerInspectDetails(value: unknown): DockerContainerInspectDetails | null {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
||||
const row = value as Record<string, unknown>;
|
||||
const hostConfig = typeof row.HostConfig === "object" && row.HostConfig !== null && !Array.isArray(row.HostConfig)
|
||||
? row.HostConfig as Record<string, unknown>
|
||||
: {};
|
||||
const configRow = typeof row.Config === "object" && row.Config !== null && !Array.isArray(row.Config)
|
||||
? row.Config as Record<string, unknown>
|
||||
: {};
|
||||
const restartPolicy = typeof hostConfig.RestartPolicy === "object" && hostConfig.RestartPolicy !== null && !Array.isArray(hostConfig.RestartPolicy)
|
||||
? hostConfig.RestartPolicy as Record<string, unknown>
|
||||
: {};
|
||||
const labels = stringRecordFromUnknown(configRow.Labels);
|
||||
const rawName = typeof row.Name === "string" ? row.Name : "";
|
||||
const rawMaxRetry = restartPolicy.MaximumRetryCount;
|
||||
return {
|
||||
id: typeof row.Id === "string" ? row.Id : "",
|
||||
name: rawName.replace(/^\/+/, ""),
|
||||
restartPolicy: typeof restartPolicy.Name === "string" ? restartPolicy.Name : "",
|
||||
restartPolicyMaximumRetryCount: typeof rawMaxRetry === "number" && Number.isFinite(rawMaxRetry) ? rawMaxRetry : 0,
|
||||
pidMode: typeof hostConfig.PidMode === "string" ? hostConfig.PidMode : "",
|
||||
composeProject: labels["com.docker.compose.project"] ?? "",
|
||||
composeService: labels["com.docker.compose.service"] ?? "",
|
||||
labels,
|
||||
};
|
||||
}
|
||||
|
||||
function inspectCurrentContainerDetails(containerId: string): DockerContainerInspectDetails | null {
|
||||
const result = spawnSync("docker", ["inspect", "--format", "{{json .}}", containerId], {
|
||||
encoding: "utf8",
|
||||
timeout: 2500,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
if (result.status !== 0) return null;
|
||||
try {
|
||||
return dockerInspectDetails(JSON.parse(result.stdout.trim()) as unknown);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function containerGuardStateFromDetails(details: DockerContainerInspectDetails, remediatedRestartPolicy: boolean, error = ""): ProviderGatewayContainerGuardState {
|
||||
const restartPolicyOk = details.restartPolicy === "always";
|
||||
const pidModeOk = details.pidMode === "host";
|
||||
return {
|
||||
checkedAt: new Date().toISOString(),
|
||||
containerId: details.id.slice(0, 12),
|
||||
containerName: details.name,
|
||||
restartPolicy: details.restartPolicy,
|
||||
restartPolicyMaximumRetryCount: details.restartPolicyMaximumRetryCount,
|
||||
restartPolicyOk,
|
||||
pidMode: details.pidMode,
|
||||
pidModeOk,
|
||||
remediatedRestartPolicy,
|
||||
ok: restartPolicyOk && pidModeOk && error.length === 0,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
function refreshSelfContainerGuard(force = false): ProviderGatewayContainerGuardState {
|
||||
const now = Date.now();
|
||||
if (!force && selfContainerGuard !== null && now - selfContainerGuardLastCheckMs < 60_000) return selfContainerGuard;
|
||||
selfContainerGuardLastCheckMs = now;
|
||||
const containerId = currentContainerId();
|
||||
if (containerId === null) {
|
||||
selfContainerGuard = unknownContainerGuardState("unable to determine current container id from hostname");
|
||||
return selfContainerGuard;
|
||||
}
|
||||
if (!existsSync(config.dockerSocketPath)) {
|
||||
selfContainerGuard = unknownContainerGuardState(`docker socket is not mounted at ${config.dockerSocketPath}`);
|
||||
return selfContainerGuard;
|
||||
}
|
||||
|
||||
let details = inspectCurrentContainerDetails(containerId);
|
||||
if (details === null) {
|
||||
selfContainerGuard = unknownContainerGuardState("docker inspect of current provider-gateway container failed");
|
||||
return selfContainerGuard;
|
||||
}
|
||||
|
||||
let remediatedRestartPolicy = false;
|
||||
let error = "";
|
||||
if (details.restartPolicy !== "always") {
|
||||
const updateResult = spawnSync("docker", ["update", "--restart", "always", containerId], {
|
||||
encoding: "utf8",
|
||||
timeout: 2500,
|
||||
maxBuffer: 64 * 1024,
|
||||
});
|
||||
if (updateResult.status === 0) {
|
||||
remediatedRestartPolicy = true;
|
||||
details = inspectCurrentContainerDetails(containerId) ?? details;
|
||||
} else {
|
||||
error = `docker update --restart always failed: ${(updateResult.stderr || updateResult.stdout || "").slice(0, 500)}`;
|
||||
}
|
||||
}
|
||||
|
||||
selfContainerGuard = containerGuardStateFromDetails(details, remediatedRestartPolicy, error);
|
||||
if (remediatedRestartPolicy) {
|
||||
logger("warn", "provider_gateway_restart_policy_remediated", selfContainerGuard as unknown as JsonValue);
|
||||
} else if (!selfContainerGuard.restartPolicyOk) {
|
||||
logger("error", "provider_gateway_restart_policy_guard_failed", selfContainerGuard as unknown as JsonValue);
|
||||
}
|
||||
if (!selfContainerGuard.pidModeOk) {
|
||||
logger("error", "provider_gateway_pid_namespace_guard_failed", selfContainerGuard as unknown as JsonValue);
|
||||
}
|
||||
return selfContainerGuard;
|
||||
}
|
||||
|
||||
function discoverDockerMountSource(destination: string): string | null {
|
||||
const containerId = currentContainerId();
|
||||
if (containerId === null) return null;
|
||||
const template = `{{range .Mounts}}{{if eq .Destination ${goTemplateString(destination)}}{{println .Source}}{{end}}{{end}}`;
|
||||
try {
|
||||
const result = spawnSync("docker", ["inspect", "--format", template, containerId], {
|
||||
encoding: "utf8",
|
||||
timeout: 2000,
|
||||
maxBuffer: 8192,
|
||||
});
|
||||
if (result.status !== 0) return null;
|
||||
const source = (result.stdout ?? "").split("\n").map((line) => line.trim()).find((line) => line.length > 0);
|
||||
return source ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function inferHostSshUser(hostProjectRoot: string): string | null {
|
||||
const homeMatch = hostProjectRoot.match(/^\/home\/([^/]+)/u);
|
||||
if (homeMatch?.[1]) return homeMatch[1];
|
||||
if (hostProjectRoot === "/root" || hostProjectRoot.startsWith("/root/")) return "root";
|
||||
return null;
|
||||
}
|
||||
|
||||
function defaultHostRemoteCwd(user: string | null): string | null {
|
||||
if (user === null) return null;
|
||||
return user === "root" ? "/root" : `/home/${user}`;
|
||||
}
|
||||
|
||||
function readConfig(): RuntimeConfig {
|
||||
const providerId = readProviderId();
|
||||
const workspacePath = readOptionalStringEnv("PROVIDER_UPGRADE_WORKSPACE_PATH") ?? "/workspace";
|
||||
const hostProjectRoot = readOptionalStringEnv("PROVIDER_UPGRADE_HOST_PROJECT_ROOT")
|
||||
?? discoverDockerMountSource(workspacePath)
|
||||
?? workspacePath;
|
||||
const inferredSshUser = inferHostSshUser(hostProjectRoot);
|
||||
const defaultHostSshKey = "/run/host-ssh/id_ed25519";
|
||||
const mountedHostSshKey = existsSync(defaultHostSshKey);
|
||||
const hostSshUser = readOptionalStringEnv("HOST_SSH_USER") ?? (mountedHostSshKey ? inferredSshUser : null);
|
||||
return {
|
||||
serverUrl: readProviderServerUrl(),
|
||||
token: readEnv("PROVIDER_TOKEN", "UNIDESK_PROVIDER_TOKEN") ?? defaultProviderToken,
|
||||
providerId,
|
||||
providerName: readEnv("PROVIDER_NAME", "UNIDESK_PROVIDER_NAME") ?? providerId,
|
||||
labels: readProviderLabels(providerId),
|
||||
heartbeatIntervalMs: readNumberEnv("HEARTBEAT_INTERVAL_MS", 15000, "UNIDESK_HEARTBEAT_INTERVAL_MS"),
|
||||
reconnectBaseMs: readNumberEnv("RECONNECT_BASE_MS", 1000, "UNIDESK_RECONNECT_BASE_MS"),
|
||||
reconnectMaxMs: readNumberEnv("RECONNECT_MAX_MS", 30000, "UNIDESK_RECONNECT_MAX_MS"),
|
||||
dockerSocketPath: readOptionalStringEnv("DOCKER_SOCKET_PATH") ?? "/var/run/docker.sock",
|
||||
monitorDiskPath: readOptionalStringEnv("MONITOR_DISK_PATH", "UNIDESK_MONITOR_DISK_PATH") ?? "/",
|
||||
upgradeHostProjectRoot: hostProjectRoot,
|
||||
upgradeWorkspacePath: workspacePath,
|
||||
upgradeComposeFile: readOptionalStringEnv("PROVIDER_UPGRADE_COMPOSE_FILE") ?? `provider-${providerId}.yml`,
|
||||
upgradeEnvFile: readOptionalStringEnv("PROVIDER_UPGRADE_ENV_FILE") ?? `.state/provider-${providerId}.env`,
|
||||
upgradeComposeProject: readOptionalStringEnv("PROVIDER_UPGRADE_COMPOSE_PROJECT") ?? `unidesk-${safeProviderSlug(providerId)}`,
|
||||
upgradeService: readOptionalStringEnv("PROVIDER_UPGRADE_SERVICE") ?? "provider-gateway",
|
||||
upgradeRunnerImage: readOptionalStringEnv("PROVIDER_UPGRADE_RUNNER_IMAGE") ?? `unidesk_provider-gateway:${safeProviderSlug(providerId)}`,
|
||||
hostSshHost: readOptionalStringEnv("HOST_SSH_HOST") ?? (mountedHostSshKey ? "host.docker.internal" : null),
|
||||
hostSshPort: readOptionalNumberEnv("HOST_SSH_PORT") ?? (mountedHostSshKey ? 22 : null),
|
||||
hostSshUser,
|
||||
hostSshKey: readOptionalStringEnv("HOST_SSH_KEY") ?? (mountedHostSshKey ? defaultHostSshKey : null),
|
||||
hostRemoteCwd: readOptionalStringEnv("HOST_REMOTE_CWD") ?? defaultHostRemoteCwd(hostSshUser),
|
||||
hostLoginShell: readOptionalStringEnv("HOST_LOGIN_SHELL") ?? (hostSshUser === null ? null : "/bin/bash"),
|
||||
logFile: readOptionalStringEnv("LOG_FILE") ?? `/var/log/unidesk/provider-gateway-${providerId}.jsonl`,
|
||||
};
|
||||
}
|
||||
|
||||
function createLogger(service: string, logFile: string) {
|
||||
mkdirSync(dirname(logFile), { recursive: true });
|
||||
const writer = createHourlyJsonlWriter({
|
||||
baseLogFile: logFile,
|
||||
service,
|
||||
maxBytes: logRetentionBytesForService(service),
|
||||
});
|
||||
writer.prune();
|
||||
return (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue): void => {
|
||||
const entry = data === undefined
|
||||
? { ts: new Date().toISOString(), service, level, message }
|
||||
: { ts: new Date().toISOString(), service, level, message, data };
|
||||
const line = `${JSON.stringify(entry)}\n`;
|
||||
try {
|
||||
appendFileSync(logFile, line, "utf8");
|
||||
writer.appendLine(line, new Date(entry.ts));
|
||||
} catch (error) {
|
||||
console.error(JSON.stringify({ ts: new Date().toISOString(), service, level: "error", message: "log_write_failed", data: String(error) }));
|
||||
}
|
||||
@@ -199,6 +496,7 @@ function withToken(rawUrl: string, token: string): string {
|
||||
|
||||
function currentLabels(): ProviderLabels {
|
||||
const hostSshConfigured = isHostSshConfigured();
|
||||
const containerGuard = refreshSelfContainerGuard();
|
||||
return {
|
||||
...config.labels,
|
||||
dockerSocketPresent: existsSync(config.dockerSocketPath),
|
||||
@@ -211,6 +509,17 @@ function currentLabels(): ProviderLabels {
|
||||
providerGatewayStartedAt: startedAt.toISOString(),
|
||||
providerGatewayUpgradePolicy: "always-enabled",
|
||||
providerGatewayMicroserviceHttpCache: true,
|
||||
providerGatewayDockerRestartGuard: true,
|
||||
providerGatewayContainerId: containerGuard.containerId,
|
||||
providerGatewayContainerName: containerGuard.containerName,
|
||||
providerGatewayRestartPolicy: containerGuard.restartPolicy,
|
||||
providerGatewayRestartPolicyOk: containerGuard.restartPolicyOk,
|
||||
providerGatewayRestartPolicyRemediated: containerGuard.remediatedRestartPolicy,
|
||||
providerGatewayPidMode: containerGuard.pidMode,
|
||||
providerGatewayPidModeOk: containerGuard.pidModeOk,
|
||||
providerGatewayRuntimeGuardOk: containerGuard.ok,
|
||||
providerGatewayRuntimeGuardError: containerGuard.error,
|
||||
providerGatewayRuntimeGuardCheckedAt: containerGuard.checkedAt,
|
||||
gatewayUptimeSeconds: Math.floor((Date.now() - startedAt.getTime()) / 1000),
|
||||
};
|
||||
}
|
||||
@@ -321,6 +630,51 @@ async function runDockerCommand(args: string[], timeoutMs = 6000): Promise<{ ok:
|
||||
return runProcessCommand("docker", args, timeoutMs);
|
||||
}
|
||||
|
||||
function containerInspectMapFromJson(stdout: string): Map<string, DockerContainerInspectDetails> {
|
||||
const map = new Map<string, DockerContainerInspectDetails>();
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(stdout) as unknown;
|
||||
} catch {
|
||||
return map;
|
||||
}
|
||||
const rows = Array.isArray(parsed) ? parsed : [parsed];
|
||||
for (const row of rows) {
|
||||
const details = dockerInspectDetails(row);
|
||||
if (details === null) continue;
|
||||
if (details.id.length > 0) {
|
||||
map.set(details.id, details);
|
||||
map.set(details.id.slice(0, 12), details);
|
||||
}
|
||||
if (details.name.length > 0) map.set(details.name, details);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async function inspectContainerDetails(containers: DockerContainerSummary[]): Promise<{ details: Map<string, DockerContainerInspectDetails>; errors: JsonValue[] }> {
|
||||
const details = new Map<string, DockerContainerInspectDetails>();
|
||||
const errors: JsonValue[] = [];
|
||||
const ids = containers.map((container) => container.id).filter((id) => id.length > 0);
|
||||
if (ids.length === 0) return { details, errors };
|
||||
|
||||
const result = await runDockerCommand(["inspect", ...ids], 8000);
|
||||
if (result.ok) {
|
||||
return { details: containerInspectMapFromJson(result.stdout), errors };
|
||||
}
|
||||
|
||||
errors.push({ source: "docker.inspect", exitCode: result.exitCode, stderr: result.stderr.slice(0, 500) });
|
||||
const gatewayContainers = containers.filter((container) => container.name.includes("provider-gateway"));
|
||||
for (const container of gatewayContainers) {
|
||||
const fallback = await runDockerCommand(["inspect", container.id], 3000);
|
||||
if (!fallback.ok) {
|
||||
errors.push({ source: "docker.inspect.provider-gateway", container: container.name, exitCode: fallback.exitCode, stderr: fallback.stderr.slice(0, 300) });
|
||||
continue;
|
||||
}
|
||||
for (const [key, value] of containerInspectMapFromJson(fallback.stdout)) details.set(key, value);
|
||||
}
|
||||
return { details, errors };
|
||||
}
|
||||
|
||||
function stringField(row: Record<string, unknown>, key: string): string {
|
||||
const value = row[key];
|
||||
return typeof value === "string" ? value : value === undefined || value === null ? "" : String(value);
|
||||
@@ -342,7 +696,7 @@ function parseJsonLines(stdout: string, limit: number): Array<Record<string, unk
|
||||
return rows;
|
||||
}
|
||||
|
||||
function toContainer(row: Record<string, unknown>): DockerContainerSummary {
|
||||
function toContainer(row: Record<string, unknown>, inspect: DockerContainerInspectDetails | null = null): DockerContainerSummary {
|
||||
return {
|
||||
id: stringField(row, "ID"),
|
||||
name: stringField(row, "Names"),
|
||||
@@ -354,6 +708,12 @@ function toContainer(row: Record<string, unknown>): DockerContainerSummary {
|
||||
runningFor: stringField(row, "RunningFor"),
|
||||
size: stringField(row, "Size"),
|
||||
networks: stringField(row, "Networks"),
|
||||
restartPolicy: inspect?.restartPolicy ?? "",
|
||||
restartPolicyMaximumRetryCount: inspect?.restartPolicyMaximumRetryCount ?? 0,
|
||||
pidMode: inspect?.pidMode ?? "",
|
||||
composeProject: inspect?.composeProject ?? "",
|
||||
composeService: inspect?.composeService ?? "",
|
||||
labels: inspect?.labels ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -764,7 +1124,16 @@ async function collectDockerStatus(): Promise<DockerStatusSnapshot> {
|
||||
if (!result.ok) errors.push({ source, exitCode: result.exitCode, stderr: result.stderr.slice(0, 500) });
|
||||
}
|
||||
|
||||
const containers = containersResult.ok ? parseJsonLines(containersResult.stdout, 80).map(toContainer) : [];
|
||||
const baseContainers = containersResult.ok ? parseJsonLines(containersResult.stdout, 80).map((row) => toContainer(row)) : [];
|
||||
const inspected = await inspectContainerDetails(baseContainers);
|
||||
for (const error of inspected.errors) errors.push(error);
|
||||
const containers = containersResult.ok
|
||||
? parseJsonLines(containersResult.stdout, 80).map((row) => {
|
||||
const id = stringField(row, "ID");
|
||||
const name = stringField(row, "Names");
|
||||
return toContainer(row, inspected.details.get(id) ?? inspected.details.get(name) ?? null);
|
||||
})
|
||||
: [];
|
||||
const images = imagesResult.ok ? parseJsonLines(imagesResult.stdout, 80).map(toImage) : [];
|
||||
const volumes = volumesResult.ok ? parseJsonLines(volumesResult.stdout, 60).map(toVolume) : [];
|
||||
const networks = networksResult.ok ? parseJsonLines(networksResult.stdout, 60).map(toNetwork) : [];
|
||||
@@ -1236,9 +1605,12 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
`attempt=0`,
|
||||
`validated=0`,
|
||||
`while [ "$attempt" -lt "${validationAttempts}" ]; do logs=$(docker logs ${shellQuote(candidateName)} 2>&1 || true); has_open=0; has_ack=0; has_ok=0; case "$logs" in *${shellQuote(validationNeedleOpen)}*) has_open=1;; esac; case "$logs" in *${shellQuote(validationNeedleAck)}*) has_ack=1;; esac; case "$logs" in *${shellQuote(validationNeedleOk)}*) has_ok=1;; esac; if [ "$has_open" = "1" ] && [ "$has_ack" = "1" ] && [ "$has_ok" = "1" ]; then validated=1; break; fi; candidate_running=$(docker inspect --format '{{.State.Running}}' ${shellQuote(candidateName)} 2>/dev/null || true); if [ "$candidate_running" != "true" ]; then break; fi; attempt=$((attempt + 1)); sleep 2; done`,
|
||||
`if [ "$validated" != "1" ]; then echo "candidate validation failed; old gateway will leave upgrade sleep automatically" >&2; docker logs ${shellQuote(candidateName)} >&2 || true; docker rm -f ${shellQuote(candidateName)} >/dev/null 2>&1 || true; rm -f "$candidate_env_file"; exit 1; fi`,
|
||||
`docker update --restart always ${shellQuote(candidateName)} >/dev/null`,
|
||||
`if [ -n "$old_ids" ]; then docker rm -f $old_ids; fi`,
|
||||
`if [ "$validated" != "1" ]; then echo "candidate validation failed; old gateway will leave upgrade sleep automatically" >&2; docker logs ${shellQuote(candidateName)} >&2 || true; docker rm -f ${shellQuote(candidateName)} >/dev/null 2>&1 || true; rm -f "$candidate_env_file"; exit 1; fi`,
|
||||
`docker update --restart always ${shellQuote(candidateName)} >/dev/null`,
|
||||
`final_restart=$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' ${shellQuote(candidateName)})`,
|
||||
`final_pid_mode=$(docker inspect --format '{{.HostConfig.PidMode}}' ${shellQuote(candidateName)})`,
|
||||
`if [ "$final_restart" != "always" ] || [ "$final_pid_mode" != "host" ]; then echo "candidate runtime guard failed: restart=$final_restart pid=$final_pid_mode" >&2; docker rm -f ${shellQuote(candidateName)} >/dev/null 2>&1 || true; rm -f "$candidate_env_file"; exit 1; fi`,
|
||||
`if [ -n "$old_ids" ]; then docker rm -f $old_ids; fi`,
|
||||
`if [ -n "$old_name" ] && [ "$old_name" != ${shellQuote(candidateName)} ]; then docker rename ${shellQuote(candidateName)} "$old_name" || true; fi`,
|
||||
`rm -f "$candidate_env_file"`,
|
||||
`echo "candidate provider-gateway validated and promoted"`,
|
||||
@@ -1296,12 +1668,15 @@ function upgradePlan(taskId: string): Record<string, JsonValue> {
|
||||
validationTimeoutMs,
|
||||
oldGatewayRestartPolicyBeforeSleep: "always",
|
||||
promoteOnlyAfterCandidateValidation: true,
|
||||
candidateRestartPolicyAfterPromotion: "always",
|
||||
candidateUsesOldContainerMounts: true,
|
||||
candidateUsesOldContainerNetworks: true,
|
||||
candidateUsesOldContainerExtraHosts: true,
|
||||
candidateUsesOldContainerEnvironment: true,
|
||||
candidateUsesHostPidNamespace: true,
|
||||
candidateRestartPolicyAfterPromotion: "always",
|
||||
candidateFinalRestartPolicyValidation: true,
|
||||
candidateUsesOldContainerMounts: true,
|
||||
candidateUsesOldContainerNetworks: true,
|
||||
candidateUsesOldContainerExtraHosts: true,
|
||||
candidateUsesOldContainerEnvironment: true,
|
||||
candidateUsesHostPidNamespace: true,
|
||||
startupSelfHealsRestartPolicy: true,
|
||||
dockerStatusReportsRestartPolicyAndPidMode: true,
|
||||
removeScope: {
|
||||
projectLabel: config.upgradeComposeProject,
|
||||
serviceLabel: config.upgradeService,
|
||||
@@ -1367,7 +1742,7 @@ function assertAllowedMicroserviceBase(rawBaseUrl: string): URL {
|
||||
const baseUrl = new URL(rawBaseUrl);
|
||||
if (baseUrl.protocol !== "http:") throw new Error(`microservice backend only supports http URLs, got ${baseUrl.protocol}`);
|
||||
const host = baseUrl.hostname.toLowerCase();
|
||||
const allowedHosts = new Set(["127.0.0.1", "localhost", "host.docker.internal", "todo-note", "codex-queue"]);
|
||||
const allowedHosts = new Set(["127.0.0.1", "localhost", "host.docker.internal", "todo-note", "code-queue"]);
|
||||
if (!allowedHosts.has(host)) throw new Error(`microservice backend host is not allowed: ${baseUrl.hostname}`);
|
||||
return baseUrl;
|
||||
}
|
||||
@@ -1494,6 +1869,15 @@ function invalidateMicroserviceHttpCache(serviceId: string, targetBaseUrl: strin
|
||||
}
|
||||
}
|
||||
|
||||
function headersFromMicroserviceRequest(requestHeaders: Record<string, JsonValue>): Headers {
|
||||
const headers = new Headers();
|
||||
for (const name of microserviceForwardRequestHeaders) {
|
||||
const value = requestHeaders[name];
|
||||
if (typeof value === "string" && value.length > 0) headers.set(name, value);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function runMicroserviceHttp(payload: Record<string, JsonValue>): Promise<JsonValue> {
|
||||
const rawMethod = String(payload.method || "GET").toUpperCase();
|
||||
const allowedMethods = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"]);
|
||||
@@ -1532,9 +1916,7 @@ async function runMicroserviceHttp(payload: Record<string, JsonValue>): Promise<
|
||||
const requestHeaders = typeof payload.requestHeaders === "object" && payload.requestHeaders !== null && !Array.isArray(payload.requestHeaders)
|
||||
? payload.requestHeaders as Record<string, JsonValue>
|
||||
: {};
|
||||
const headers = new Headers();
|
||||
const contentType = typeof requestHeaders["content-type"] === "string" ? requestHeaders["content-type"] : "";
|
||||
if (contentType.length > 0) headers.set("content-type", contentType);
|
||||
const headers = headersFromMicroserviceRequest(requestHeaders);
|
||||
const requestStartedAt = Date.now();
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
Reference in New Issue
Block a user